@crawlee/basic 4.0.0-beta.27 → 4.0.0-beta.271
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/index.d.ts +11 -2
- package/index.js +11 -1
- package/internals/autoscaling/autoscaled-pool.d.ts +178 -0
- package/internals/autoscaling/autoscaled-pool.js +389 -0
- package/internals/autoscaling/concurrency-system.d.ts +235 -0
- package/internals/autoscaling/concurrency-system.js +357 -0
- package/internals/autoscaling/cpu-load-signal.d.ts +43 -0
- package/internals/autoscaling/cpu-load-signal.js +46 -0
- package/internals/autoscaling/event-loop-load-signal.d.ts +51 -0
- package/internals/autoscaling/event-loop-load-signal.js +60 -0
- package/internals/autoscaling/index.d.ts +9 -0
- package/internals/autoscaling/index.js +9 -0
- package/internals/autoscaling/load-signal.d.ts +98 -0
- package/internals/autoscaling/load-signal.js +103 -0
- package/internals/autoscaling/memory-load-signal.d.ts +45 -0
- package/internals/autoscaling/memory-load-signal.js +104 -0
- package/internals/autoscaling/snapshotter.d.ts +84 -0
- package/internals/autoscaling/snapshotter.js +67 -0
- package/internals/autoscaling/storage-backend-load-signal.d.ts +56 -0
- package/internals/autoscaling/storage-backend-load-signal.js +73 -0
- package/internals/autoscaling/system-status.d.ts +112 -0
- package/internals/autoscaling/system-status.js +139 -0
- package/internals/autoscaling/weighted-avg.d.ts +5 -0
- package/internals/autoscaling/weighted-avg.js +14 -0
- package/internals/basic-crawler.d.ts +383 -369
- package/internals/basic-crawler.js +1468 -890
- package/internals/cookie-utils.d.ts +40 -0
- package/internals/cookie-utils.js +122 -0
- package/internals/crawler-run.d.ts +76 -0
- package/internals/crawler-run.js +144 -0
- package/internals/crawlers/context-pipeline.d.ts +75 -0
- package/internals/crawlers/context-pipeline.js +109 -0
- package/internals/crawlers/crawler-commons.d.ts +216 -0
- package/internals/crawlers/crawler-commons.js +1 -0
- package/internals/crawlers/error-snapshotter.d.ts +29 -0
- package/internals/crawlers/error-snapshotter.js +116 -0
- package/internals/crawlers/error-tracker.d.ts +51 -0
- package/internals/crawlers/error-tracker.js +316 -0
- package/internals/crawlers/index.d.ts +4 -0
- package/internals/crawlers/index.js +3 -0
- package/internals/crawlers/internals/types.d.ts +7 -0
- package/internals/crawlers/internals/types.js +1 -0
- package/internals/crawlers/statistics.d.ts +326 -0
- package/internals/crawlers/statistics.js +559 -0
- package/internals/crawling-request.d.ts +86 -0
- package/internals/crawling-request.js +127 -0
- package/internals/enqueue-links/enqueue-links.d.ts +147 -0
- package/internals/enqueue-links/enqueue-links.js +35 -0
- package/internals/enqueue-links/index.d.ts +3 -0
- package/internals/enqueue-links/index.js +3 -0
- package/internals/enqueue-links/shared.d.ts +107 -0
- package/internals/enqueue-links/shared.js +228 -0
- package/internals/errors.d.ts +73 -0
- package/internals/errors.js +108 -0
- package/internals/http.d.ts +9 -0
- package/internals/http.js +28 -0
- package/internals/request-timeout.d.ts +54 -0
- package/internals/request-timeout.js +98 -0
- package/internals/router.d.ts +299 -0
- package/internals/router.js +312 -0
- package/internals/send-request.d.ts +4 -4
- package/internals/send-request.js +18 -8
- package/internals/session-pool/consts.d.ts +1 -0
- package/internals/session-pool/consts.js +1 -0
- package/internals/session-pool/errors.d.ts +7 -0
- package/internals/session-pool/errors.js +11 -0
- package/internals/session-pool/fingerprint.d.ts +9 -0
- package/internals/session-pool/fingerprint.js +30 -0
- package/internals/session-pool/index.d.ts +4 -0
- package/internals/session-pool/index.js +4 -0
- package/internals/session-pool/session-pool.d.ts +193 -0
- package/internals/session-pool/session-pool.js +453 -0
- package/internals/session-pool/session.d.ts +151 -0
- package/internals/session-pool/session.js +225 -0
- package/internals/sitemap-request-loader.d.ts +158 -0
- package/internals/sitemap-request-loader.js +427 -0
- package/internals/throttling-request-manager.d.ts +229 -0
- package/internals/throttling-request-manager.js +860 -0
- package/package.json +23 -17
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/internals/basic-crawler.d.ts.map +0 -1
- package/internals/basic-crawler.js.map +0 -1
- package/internals/send-request.d.ts.map +0 -1
- package/internals/send-request.js.map +0 -1
|
@@ -1,25 +1,44 @@
|
|
|
1
|
-
import type { AddRequestsBatchedOptions, AddRequestsBatchedResult,
|
|
2
|
-
import {
|
|
3
|
-
import
|
|
1
|
+
import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, CrawleeLogger, DatasetExportOptions, IProxyConfiguration, IRequestLoader, IRequestManager, RequestsLike, Source, StorageIdentifier, StorageWritePolicy } from '@crawlee/core';
|
|
2
|
+
import { Configuration, Request, Dataset, EventManager, RequestQueue } from '@crawlee/core';
|
|
3
|
+
import { BaseHttpClient } from '@crawlee/http-client';
|
|
4
|
+
import type { Awaitable, Dictionary, ISessionPool, SetStatusMessageOptions, StorageBackend } from '@crawlee/types';
|
|
4
5
|
import { RobotsTxtFile } from '@crawlee/utils';
|
|
5
|
-
import type { ReadonlyDeep
|
|
6
|
-
import
|
|
7
|
-
import {
|
|
6
|
+
import type { ReadonlyDeep } from 'type-fest';
|
|
7
|
+
import { z } from 'zod';
|
|
8
|
+
import type { TaskLoopPredicates } from './autoscaling/autoscaled-pool.js';
|
|
9
|
+
import type { ConcurrencySystemOptions, IConcurrencySystem } from './autoscaling/concurrency-system.js';
|
|
10
|
+
import { ConcurrencySystem } from './autoscaling/concurrency-system.js';
|
|
11
|
+
import type { FinalStatistics } from './autoscaling/system-status.js';
|
|
12
|
+
import type { ContextMiddleware } from './crawlers/context-pipeline.js';
|
|
13
|
+
import { ContextPipeline } from './crawlers/context-pipeline.js';
|
|
14
|
+
import type { CrawlingContext, TypedRequestsLike } from './crawlers/crawler-commons.js';
|
|
15
|
+
import type { IStatistics, StatisticState } from './crawlers/statistics.js';
|
|
16
|
+
import type { EnqueueUrlsOptions, SkippedRequestCallback } from './enqueue-links/index.js';
|
|
17
|
+
import type { GetUserDataFromRequest, RouterHandler, RouterRoutes } from './router.js';
|
|
8
18
|
export interface BasicCrawlingContext<UserData extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
|
|
9
19
|
}
|
|
20
|
+
export { navigationDeadlineKey, remainingNavigationWindowMillis } from './request-timeout.js';
|
|
10
21
|
export type RequestHandler<Context extends CrawlingContext = CrawlingContext> = (inputs: Context) => Awaitable<void>;
|
|
11
|
-
|
|
12
|
-
|
|
22
|
+
/**
|
|
23
|
+
* An error handler receives the crawling context and the error that was thrown while processing the request.
|
|
24
|
+
*
|
|
25
|
+
* Unlike the {@link RequestHandler}, an error handler may run before the context pipeline has finished
|
|
26
|
+
* building the full context (e.g. when navigation or session setup fails). Therefore only `BaseContext` is
|
|
27
|
+
* guaranteed to be present, while the extra properties added by the pipeline and `extendContext` (the
|
|
28
|
+
* difference between `BaseContext` and `ExtendedContext`) are only available as a `Partial`.
|
|
29
|
+
*/
|
|
30
|
+
export type ErrorHandler<BaseContext extends CrawlingContext = CrawlingContext, ExtendedContext extends BaseContext = BaseContext> = (inputs: BaseContext & Partial<ExtendedContext>, error: Error) => Awaitable<void>;
|
|
31
|
+
export interface StatusMessageCallbackParams<Context extends CrawlingContext = BasicCrawlingContext, Crawler extends BasicCrawler<any, any, any, any> = BasicCrawler<Context>> {
|
|
13
32
|
state: StatisticState;
|
|
14
33
|
crawler: Crawler;
|
|
15
34
|
previousState: StatisticState;
|
|
16
35
|
message: string;
|
|
17
36
|
}
|
|
18
|
-
export type StatusMessageCallback<Context extends CrawlingContext = BasicCrawlingContext, Crawler extends BasicCrawler<any> = BasicCrawler<Context>> = (params: StatusMessageCallbackParams<Context, Crawler>) => Awaitable<void>;
|
|
37
|
+
export type StatusMessageCallback<Context extends CrawlingContext = BasicCrawlingContext, Crawler extends BasicCrawler<any, any, any, any> = BasicCrawler<Context>> = (params: StatusMessageCallbackParams<Context, Crawler>) => Awaitable<void>;
|
|
19
38
|
export type RequireContextPipeline<DefaultContextType extends CrawlingContext, FinalContextType extends DefaultContextType> = DefaultContextType extends FinalContextType ? {} : {
|
|
20
39
|
contextPipelineBuilder: () => ContextPipeline<CrawlingContext, FinalContextType>;
|
|
21
40
|
};
|
|
22
|
-
export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension> {
|
|
41
|
+
export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> {
|
|
23
42
|
/**
|
|
24
43
|
* User-provided function that performs the logic of the crawler. It is called for each URL to crawl.
|
|
25
44
|
*
|
|
@@ -35,11 +54,21 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
|
|
|
35
54
|
* To make this work, we should **always**
|
|
36
55
|
* let our function throw exceptions rather than catch them.
|
|
37
56
|
* The exceptions are logged to the request using the
|
|
38
|
-
* {@link
|
|
57
|
+
* {@link CrawlingRequest.pushErrorMessage|`request.pushErrorMessage()`} function.
|
|
39
58
|
*/
|
|
40
|
-
requestHandler?: RequestHandler<ExtendedContext>;
|
|
59
|
+
requestHandler?: RouterHandler<ExtendedContext, Routes> | RequestHandler<ExtendedContext>;
|
|
41
60
|
/**
|
|
42
|
-
* Allows the user to extend the crawling context
|
|
61
|
+
* Allows the user to extend the crawling context with custom functionality (helpers, references, etc.).
|
|
62
|
+
*
|
|
63
|
+
* `extendContext` runs *before* navigation, so the returned members are visible to the
|
|
64
|
+
* `preNavigationHooks`, `postNavigationHooks`, and the `requestHandler` alike. As a consequence,
|
|
65
|
+
* the `context` passed to `extendContext` is the pre-navigation {@link CrawlingContext} and does
|
|
66
|
+
* **not** include navigation-dependent members (e.g. `page`, `response`, `$`, `body`). If you need
|
|
67
|
+
* those, use a `postNavigationHook` or the `requestHandler` instead.
|
|
68
|
+
*
|
|
69
|
+
* The second argument is the same {@link CleanupRegistrar|`onCleanup`} registrar used by
|
|
70
|
+
* {@link ContextPipeline.compose}. Use it to register teardown callbacks that run after the
|
|
71
|
+
* request handler or a pipeline failure, in reverse registration order.
|
|
43
72
|
*
|
|
44
73
|
* **Example usage:**
|
|
45
74
|
*
|
|
@@ -59,7 +88,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
|
|
|
59
88
|
* });
|
|
60
89
|
* ```
|
|
61
90
|
*/
|
|
62
|
-
extendContext?:
|
|
91
|
+
extendContext?: ContextMiddleware<CrawlingContext, ContextExtension>;
|
|
63
92
|
/**
|
|
64
93
|
* *Intended for BasicCrawler subclasses*. Prepares a context pipeline that transforms the initial crawling context into the shape given by the `Context` type parameter.
|
|
65
94
|
*
|
|
@@ -68,23 +97,25 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
|
|
|
68
97
|
contextPipelineBuilder?: () => ContextPipeline<CrawlingContext, Context>;
|
|
69
98
|
/**
|
|
70
99
|
* Static list of URLs to be processed.
|
|
71
|
-
*
|
|
72
|
-
*
|
|
73
|
-
*
|
|
100
|
+
*
|
|
101
|
+
* @deprecated Use the `requestManager` option instead. To combine a read-only loader (such as a `RequestList`)
|
|
102
|
+
* with a writable queue, build a tandem with {@link IRequestLoader.toTandem|`requestList.toTandem(requestQueue)`}
|
|
103
|
+
* and pass the result as `requestManager`. When both `requestList` and `requestQueue` are provided, they are
|
|
104
|
+
* combined into a tandem automatically.
|
|
74
105
|
*/
|
|
75
|
-
requestList?:
|
|
106
|
+
requestList?: IRequestLoader;
|
|
76
107
|
/**
|
|
77
108
|
* Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
|
|
78
|
-
*
|
|
79
|
-
*
|
|
80
|
-
* it
|
|
109
|
+
*
|
|
110
|
+
* @deprecated Use the `requestManager` option instead. A `RequestQueue` is itself a request manager, so you can
|
|
111
|
+
* pass it directly as `requestManager`.
|
|
81
112
|
*/
|
|
82
|
-
requestQueue?:
|
|
113
|
+
requestQueue?: RequestQueue;
|
|
83
114
|
/**
|
|
84
|
-
*
|
|
115
|
+
* Manager of requests that should be processed by the crawler. Mutually exclusive with the deprecated
|
|
116
|
+
* `requestQueue` and `requestList` options.
|
|
85
117
|
*
|
|
86
|
-
*
|
|
87
|
-
* If using this, the type of `BasicCrawler.requestQueue` may not be fully compatible with the `RequestProvider` class.
|
|
118
|
+
* If not provided, the crawler will open the default {@link RequestQueue} when it is first needed.
|
|
88
119
|
*/
|
|
89
120
|
requestManager?: IRequestManager;
|
|
90
121
|
/**
|
|
@@ -113,27 +144,21 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
|
|
|
113
144
|
failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
|
|
114
145
|
/**
|
|
115
146
|
* Specifies the maximum number of retries allowed for a request if its processing fails.
|
|
116
|
-
* This includes retries due to navigation errors or errors thrown from user-supplied
|
|
117
|
-
* (`requestHandler`, `preNavigationHooks`, `postNavigationHooks`).
|
|
118
|
-
*
|
|
119
|
-
* This limit does not apply to retries triggered by session rotation
|
|
120
|
-
* (see {@link BasicCrawlerOptions.maxSessionRotations|`maxSessionRotations`}).
|
|
147
|
+
* This includes retries due to navigation errors, session/proxy errors, or errors thrown from user-supplied
|
|
148
|
+
* functions (`requestHandler`, `preNavigationHooks`, `postNavigationHooks`).
|
|
121
149
|
* @default 3
|
|
122
150
|
*/
|
|
123
151
|
maxRequestRetries?: number;
|
|
124
152
|
/**
|
|
125
|
-
* Indicates how much time (in seconds) to wait before crawling another same domain request.
|
|
153
|
+
* Indicates how much time (in seconds) to wait before crawling another same domain request. Subdomains are
|
|
154
|
+
* paced together with the site they belong to.
|
|
155
|
+
*
|
|
156
|
+
* Offered to the crawler's request manager as a `minIntervalEverywhere` {@link PacingSignal}; a manager that
|
|
157
|
+
* already paces every domain it dispatches to takes it, so no domain ends up with two clocks. Otherwise the
|
|
158
|
+
* crawler wraps its request manager in a {@link ThrottlingRequestManager} of its own.
|
|
126
159
|
* @default 0
|
|
127
160
|
*/
|
|
128
161
|
sameDomainDelaySecs?: number;
|
|
129
|
-
/**
|
|
130
|
-
* Maximum number of session rotations per request.
|
|
131
|
-
* The crawler will automatically rotate the session in case of a proxy error or if it gets blocked by the website.
|
|
132
|
-
*
|
|
133
|
-
* The session rotations are not counted towards the {@link BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} limit.
|
|
134
|
-
* @default 10
|
|
135
|
-
*/
|
|
136
|
-
maxSessionRotations?: number;
|
|
137
162
|
/**
|
|
138
163
|
* Maximum number of pages that the crawler will open. The crawl will stop when this limit is reached.
|
|
139
164
|
* This value should always be set in order to prevent infinite loops in misconfigured crawlers.
|
|
@@ -147,46 +172,82 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
|
|
|
147
172
|
*/
|
|
148
173
|
maxCrawlDepth?: number;
|
|
149
174
|
/**
|
|
150
|
-
*
|
|
151
|
-
*
|
|
152
|
-
*
|
|
153
|
-
*
|
|
154
|
-
*
|
|
175
|
+
* Lets you override the predicates that steer the crawler's task loop: `isTaskReadyFunction` (may another request
|
|
176
|
+
* start?) and `isFinishedFunction` (is the crawl over?). The task itself — fetching a request and running it
|
|
177
|
+
* through the pipeline — is owned by the crawler and cannot be overridden.
|
|
178
|
+
*
|
|
179
|
+
* Concurrency is configured elsewhere — through the `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute`
|
|
180
|
+
* shortcuts, or a {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} for finer control.
|
|
155
181
|
*/
|
|
156
|
-
|
|
182
|
+
taskLoopOptions?: TaskLoopPredicates;
|
|
183
|
+
/**
|
|
184
|
+
* A pre-configured concurrency governor — the component that decides whether there is free compute for one more
|
|
185
|
+
* task. Typically a {@link ConcurrencySystem}, though any {@link IConcurrencySystem} is accepted. All
|
|
186
|
+
* scaling configuration (min/max/desired concurrency, scaling ratios, `maxTasksPerMinute`, snapshotter tuning)
|
|
187
|
+
* lives on the instance itself.
|
|
188
|
+
*
|
|
189
|
+
* Inject the *same* instance into several concurrent crawlers to cap their **combined** concurrency against a
|
|
190
|
+
* single budget. Each crawler still builds and drives its own {@link AutoscaledPool}; only the load/scaling
|
|
191
|
+
* accounting is shared.
|
|
192
|
+
*
|
|
193
|
+
* Mutually exclusive with the `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute`
|
|
194
|
+
* shortcuts, which configure the default system this one replaces — combining the two throws.
|
|
195
|
+
*
|
|
196
|
+
* You own a supplied system's lifecycle: `start()` it before `run()` (which throws otherwise) and `stop()` it once
|
|
197
|
+
* every crawler borrowing it has finished. The crawler does neither on your behalf.
|
|
198
|
+
*/
|
|
199
|
+
concurrencySystem?: IConcurrencySystem;
|
|
157
200
|
/**
|
|
158
201
|
* Sets the minimum concurrency (parallelism) for the crawl. Shortcut for the
|
|
159
|
-
*
|
|
202
|
+
* {@link ConcurrencySystemOptions.minConcurrency|`minConcurrency`} option of the crawler's default
|
|
203
|
+
* {@link ConcurrencySystem}.
|
|
160
204
|
* > *WARNING:* If we set this value too high with respect to the available system memory and CPU, our crawler will run extremely slow or crash.
|
|
161
205
|
* If not sure, it's better to keep the default value and the concurrency will scale up automatically.
|
|
162
206
|
*/
|
|
163
207
|
minConcurrency?: number;
|
|
164
208
|
/**
|
|
165
209
|
* Sets the maximum concurrency (parallelism) for the crawl. Shortcut for the
|
|
166
|
-
*
|
|
210
|
+
* {@link ConcurrencySystemOptions.maxConcurrency|`maxConcurrency`} option of the crawler's default
|
|
211
|
+
* {@link ConcurrencySystem}.
|
|
167
212
|
*/
|
|
168
213
|
maxConcurrency?: number;
|
|
214
|
+
/**
|
|
215
|
+
* Sets the concurrency (parallelism) the crawl starts with, before any scaling happens. Shortcut for the
|
|
216
|
+
* {@link ConcurrencySystemOptions.desiredConcurrency|`desiredConcurrency`} option of the crawler's default
|
|
217
|
+
* {@link ConcurrencySystem}. Defaults to `minConcurrency`.
|
|
218
|
+
*/
|
|
219
|
+
initialConcurrency?: number;
|
|
169
220
|
/**
|
|
170
221
|
* The maximum number of requests per minute the crawler should run.
|
|
171
222
|
* By default, this is set to `Infinity`, but we can pass any positive, non-zero integer.
|
|
172
|
-
* Shortcut for the
|
|
223
|
+
* Shortcut for the {@link ConcurrencySystemOptions.maxTasksPerMinute|`maxTasksPerMinute`} option of the
|
|
224
|
+
* crawler's default {@link ConcurrencySystem}.
|
|
173
225
|
*/
|
|
174
226
|
maxRequestsPerMinute?: number;
|
|
175
227
|
/**
|
|
176
228
|
* Allows to keep the crawler alive even if the {@link RequestQueue} gets empty.
|
|
177
229
|
* By default, the `crawler.run()` will resolve once the queue is empty. With `keepAlive: true` it will keep running,
|
|
178
|
-
* waiting for more requests to come.
|
|
230
|
+
* waiting for more requests to come.
|
|
231
|
+
*
|
|
232
|
+
* `crawler.stop()` only stops the crawler from dispatching new requests — it does not wait for or drain
|
|
233
|
+
* requests already sitting in the queue. If this crawler is fed by another crawler or an external producer,
|
|
234
|
+
* calling `stop()` as soon as the producer finishes can abandon requests the producer already enqueued but
|
|
235
|
+
* this crawler had not yet picked up.
|
|
236
|
+
*
|
|
237
|
+
* To shut down only once the queue is genuinely empty and the producer has stopped, override
|
|
238
|
+
* {@link BasicCrawlerOptions.taskLoopOptions|`taskLoopOptions.isFinishedFunction`} to check both
|
|
239
|
+
* `(await requestManager.checkReadiness()).status === 'finished'` and the producer's own running/finished flag.
|
|
179
240
|
*/
|
|
180
241
|
keepAlive?: boolean;
|
|
181
242
|
/**
|
|
182
|
-
*
|
|
183
|
-
*
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
*
|
|
243
|
+
* An existing session pool instance to use. When provided, the crawler will use this pool directly instead of
|
|
244
|
+
* creating a new one, enabling session sharing across multiple crawlers. The crawler will not tear down a shared
|
|
245
|
+
* pool — the caller is responsible for its lifecycle.
|
|
246
|
+
*
|
|
247
|
+
* Accepts the built-in {@link SessionPool} or any object implementing the {@link ISessionPool} interface,
|
|
248
|
+
* so custom session-management strategies can be plugged in.
|
|
188
249
|
*/
|
|
189
|
-
|
|
250
|
+
sessionPool?: ISessionPool;
|
|
190
251
|
/**
|
|
191
252
|
* Defines the length of the interval for calling the `setStatusMessage` in seconds.
|
|
192
253
|
*/
|
|
@@ -208,12 +269,23 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
|
|
|
208
269
|
* ```
|
|
209
270
|
*/
|
|
210
271
|
statusMessageCallback?: StatusMessageCallback;
|
|
272
|
+
/**
|
|
273
|
+
* HTTP status codes that indicate the session should be retired.
|
|
274
|
+
*
|
|
275
|
+
* A 429 from a domain covered by a {@link ThrottlingRequestManager} is handled as a rate limit before
|
|
276
|
+
* this is consulted, so removing 429 here only affects domains that manager does not cover.
|
|
277
|
+
*
|
|
278
|
+
* @default [401, 403, 429]
|
|
279
|
+
*/
|
|
280
|
+
blockedStatusCodes?: number[];
|
|
211
281
|
/**
|
|
212
282
|
* If set to `true`, the crawler will automatically try to bypass any detected bot protection.
|
|
213
283
|
*
|
|
214
284
|
* Currently supports:
|
|
215
285
|
* - [**Cloudflare** Bot Management](https://www.cloudflare.com/products/bot-management/)
|
|
286
|
+
* - [**AWS WAF** Challenge and CAPTCHA](https://docs.aws.amazon.com/waf/latest/developerguide/waf-captcha-and-challenge.html)
|
|
216
287
|
* - [**Google Search** Rate Limiting](https://www.google.com/sorry/)
|
|
288
|
+
* - **Imperva Incapsula**
|
|
217
289
|
*/
|
|
218
290
|
retryOnBlocked?: boolean;
|
|
219
291
|
/**
|
|
@@ -232,31 +304,64 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
|
|
|
232
304
|
* 1. based on robots.txt file,
|
|
233
305
|
* 2. because they don't match enqueueLinks filters,
|
|
234
306
|
* 3. because they are redirected to a URL that doesn't match the enqueueLinks strategy,
|
|
235
|
-
* 4.
|
|
307
|
+
* 4. because the {@link BasicCrawlerOptions.maxRequestsPerCrawl|`maxRequestsPerCrawl`} limit has been reached,
|
|
308
|
+
* 5. or by {@link RestrictedCrawlingContext.skipRequest|`context.skipRequest()`}.
|
|
236
309
|
*/
|
|
237
310
|
onSkippedRequest?: SkippedRequestCallback;
|
|
238
|
-
/** @internal */
|
|
239
|
-
log?: Log;
|
|
240
|
-
/**
|
|
241
|
-
* Enables experimental features of Crawlee, which can alter the behavior of the crawler.
|
|
242
|
-
* WARNING: these options are not guaranteed to be stable and may change or be removed at any time.
|
|
243
|
-
*/
|
|
244
|
-
experiments?: CrawlerExperiments;
|
|
245
311
|
/**
|
|
246
|
-
*
|
|
247
|
-
*
|
|
312
|
+
* A preconfigured statistics instance. When provided, the crawler records into it instead of building its own and
|
|
313
|
+
* will not `reset()` it between `run()` calls. Accepts the built-in {@link Statistics} or any object
|
|
314
|
+
* implementing {@link IStatistics}.
|
|
315
|
+
*
|
|
316
|
+
* Custom fields declared via {@link StatisticsOptions.stateExtension|`stateExtension`} are carried over to
|
|
317
|
+
* {@link BasicCrawler.statistics|`crawler.statistics.state`}:
|
|
318
|
+
*
|
|
319
|
+
* ```ts
|
|
320
|
+
* const statistics = new Statistics({ stateExtension: { defaultState: { productsFound: 0 } } });
|
|
321
|
+
*
|
|
322
|
+
* const crawler = new BasicCrawler({
|
|
323
|
+
* statistics,
|
|
324
|
+
* requestHandler: async () => {
|
|
325
|
+
* statistics.state.productsFound++;
|
|
326
|
+
* },
|
|
327
|
+
* });
|
|
328
|
+
*
|
|
329
|
+
* await crawler.run();
|
|
330
|
+
* // the custom fields are typed on `crawler.statistics` too
|
|
331
|
+
* console.log(crawler.statistics.state.productsFound);
|
|
332
|
+
* ```
|
|
248
333
|
*/
|
|
249
|
-
|
|
334
|
+
statistics?: IStatistics<StatisticStateExtension>;
|
|
250
335
|
/**
|
|
251
336
|
* HTTP client implementation for the `sendRequest` context helper and for plain HTTP crawling.
|
|
252
|
-
* Defaults to
|
|
337
|
+
* Defaults to {@link ImpitHttpClient} when `@crawlee/impit-client` is installed, otherwise {@link FetchHttpClient}.
|
|
253
338
|
*/
|
|
254
339
|
httpClient?: BaseHttpClient;
|
|
255
340
|
/**
|
|
256
341
|
* If set, the crawler will be configured for all connections to use
|
|
257
342
|
* the Proxy URLs provided and rotated according to the configuration.
|
|
258
343
|
*/
|
|
259
|
-
proxyConfiguration?:
|
|
344
|
+
proxyConfiguration?: IProxyConfiguration;
|
|
345
|
+
/**
|
|
346
|
+
* Custom configuration to use for this crawler.
|
|
347
|
+
* If provided, the crawler will use its own ServiceLocator instance instead of the global one.
|
|
348
|
+
*/
|
|
349
|
+
configuration?: Configuration;
|
|
350
|
+
/**
|
|
351
|
+
* Custom storage backend to use for this crawler.
|
|
352
|
+
* If provided, the crawler will use its own ServiceLocator instance instead of the global one.
|
|
353
|
+
*/
|
|
354
|
+
storageBackend?: StorageBackend;
|
|
355
|
+
/**
|
|
356
|
+
* Custom event manager to use for this crawler.
|
|
357
|
+
* If provided, the crawler will use its own ServiceLocator instance instead of the global one.
|
|
358
|
+
*/
|
|
359
|
+
eventManager?: EventManager;
|
|
360
|
+
/**
|
|
361
|
+
* Custom logger to use for this crawler.
|
|
362
|
+
* If provided, the crawler will use its own ServiceLocator instance instead of the global one.
|
|
363
|
+
*/
|
|
364
|
+
logger?: CrawleeLogger;
|
|
260
365
|
/**
|
|
261
366
|
* A unique identifier for the crawler instance. This ID is used to isolate the state returned by
|
|
262
367
|
* {@link BasicCrawler.useState|`crawler.useState()`} from other crawler instances.
|
|
@@ -269,243 +374,153 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
|
|
|
269
374
|
*
|
|
270
375
|
*/
|
|
271
376
|
id?: string;
|
|
272
|
-
}
|
|
273
|
-
/**
|
|
274
|
-
* A set of options that you can toggle to enable experimental features in Crawlee.
|
|
275
|
-
*
|
|
276
|
-
* NOTE: These options will not respect semantic versioning and may be removed or changed at any time. Use at your own risk.
|
|
277
|
-
* If you do use these and encounter issues, please report them to us.
|
|
278
|
-
*/
|
|
279
|
-
export interface CrawlerExperiments {
|
|
280
|
-
/**
|
|
281
|
-
* @deprecated This experiment is now enabled by default, and this flag will be removed in a future release.
|
|
282
|
-
* If you encounter issues due to this change, please:
|
|
283
|
-
* - report it to us: https://github.com/apify/crawlee
|
|
284
|
-
* - set `requestLocking` to `false` in the `experiments` option of the crawler
|
|
285
|
-
*/
|
|
286
|
-
requestLocking?: boolean;
|
|
287
|
-
}
|
|
288
|
-
/**
|
|
289
|
-
* Provides a simple framework for parallel crawling of web pages.
|
|
290
|
-
* The URLs to crawl are fed either from a static list of URLs
|
|
291
|
-
* or from a dynamic queue of URLs enabling recursive crawling of websites.
|
|
292
|
-
*
|
|
293
|
-
* `BasicCrawler` is a low-level tool that requires the user to implement the page
|
|
294
|
-
* download and data extraction functionality themselves.
|
|
295
|
-
* If we want a crawler that already facilitates this functionality,
|
|
296
|
-
* we should consider using {@link CheerioCrawler}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler}.
|
|
297
|
-
*
|
|
298
|
-
* `BasicCrawler` invokes the user-provided {@link BasicCrawlerOptions.requestHandler|`requestHandler`}
|
|
299
|
-
* for each {@link Request} object, which represents a single URL to crawl.
|
|
300
|
-
* The {@link Request} objects are fed from the {@link RequestList} or {@link RequestQueue}
|
|
301
|
-
* instances provided by the {@link BasicCrawlerOptions.requestList|`requestList`} or {@link BasicCrawlerOptions.requestQueue|`requestQueue`}
|
|
302
|
-
* constructor options, respectively. If neither `requestList` nor `requestQueue` options are provided,
|
|
303
|
-
* the crawler will open the default request queue either when the {@link BasicCrawler.addRequests|`crawler.addRequests()`} function is called,
|
|
304
|
-
* or if `requests` parameter (representing the initial requests) of the {@link BasicCrawler.run|`crawler.run()`} function is provided.
|
|
305
|
-
*
|
|
306
|
-
* If both {@link BasicCrawlerOptions.requestList|`requestList`} and {@link BasicCrawlerOptions.requestQueue|`requestQueue`} options are used,
|
|
307
|
-
* the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
|
|
308
|
-
* to the {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
|
|
309
|
-
*
|
|
310
|
-
* The crawler finishes if there are no more {@link Request} objects to crawl.
|
|
311
|
-
*
|
|
312
|
-
* New requests are only dispatched when there is enough free CPU and memory available,
|
|
313
|
-
* using the functionality provided by the {@link AutoscaledPool} class.
|
|
314
|
-
* All {@link AutoscaledPool} configuration options can be passed to the {@link BasicCrawlerOptions.autoscaledPoolOptions|`autoscaledPoolOptions`}
|
|
315
|
-
* parameter of the `BasicCrawler` constructor.
|
|
316
|
-
* For user convenience, the {@link AutoscaledPoolOptions.minConcurrency|`minConcurrency`} and
|
|
317
|
-
* {@link AutoscaledPoolOptions.maxConcurrency|`maxConcurrency`} options of the
|
|
318
|
-
* underlying {@link AutoscaledPool} constructor are available directly in the `BasicCrawler` constructor.
|
|
319
|
-
*
|
|
320
|
-
* **Example usage:**
|
|
321
|
-
*
|
|
322
|
-
* ```javascript
|
|
323
|
-
* import { BasicCrawler, Dataset } from 'crawlee';
|
|
324
|
-
*
|
|
325
|
-
* // Create a crawler instance
|
|
326
|
-
* const crawler = new BasicCrawler({
|
|
327
|
-
* async requestHandler({ request, sendRequest }) {
|
|
328
|
-
* // 'request' contains an instance of the Request class
|
|
329
|
-
* // Here we simply fetch the HTML of the page and store it to a dataset
|
|
330
|
-
* const { body } = await sendRequest({
|
|
331
|
-
* url: request.url,
|
|
332
|
-
* method: request.method,
|
|
333
|
-
* body: request.payload,
|
|
334
|
-
* headers: request.headers,
|
|
335
|
-
* });
|
|
336
|
-
*
|
|
337
|
-
* await Dataset.pushData({
|
|
338
|
-
* url: request.url,
|
|
339
|
-
* html: body,
|
|
340
|
-
* })
|
|
341
|
-
* },
|
|
342
|
-
* });
|
|
343
|
-
*
|
|
344
|
-
* // Enqueue the initial requests and run the crawler
|
|
345
|
-
* await crawler.run([
|
|
346
|
-
* 'http://www.example.com/page-1',
|
|
347
|
-
* 'http://www.example.com/page-2',
|
|
348
|
-
* ]);
|
|
349
|
-
* ```
|
|
350
|
-
* @category Crawlers
|
|
351
|
-
*/
|
|
352
|
-
export declare class BasicCrawler<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension> {
|
|
353
|
-
readonly config: Configuration;
|
|
354
|
-
protected static readonly CRAWLEE_STATE_KEY = "CRAWLEE_STATE";
|
|
355
377
|
/**
|
|
356
|
-
*
|
|
357
|
-
*
|
|
378
|
+
* Makes the storage writes performed while handling a request atomic with respect to the request
|
|
379
|
+
* succeeding: they are recorded in a {@link StorageTransaction} spanning the whole request
|
|
380
|
+
* lifecycle and only applied when the request handler succeeds, so a thrown handler leaves no partial
|
|
381
|
+
* writes behind and a retry does not double-write. Reads within the handler see its own writes.
|
|
382
|
+
*
|
|
383
|
+
* `false` disables the mechanism entirely; an object overrides the per-storage-type
|
|
384
|
+
* {@link StorageWritePolicy} (e.g. `{ requestQueue: 'deferred' }` for all-or-nothing enqueues).
|
|
385
|
+
* {@link withDirectStorageAccess} is the per-call-site escape hatch; `useState()` is deliberately
|
|
386
|
+
* *not* transactional.
|
|
387
|
+
*
|
|
388
|
+
* @default true
|
|
358
389
|
*/
|
|
359
|
-
|
|
390
|
+
transactionalStorage?: boolean | Partial<StorageWritePolicy>;
|
|
360
391
|
/**
|
|
361
|
-
*
|
|
392
|
+
* An array of HTTP response [Status Codes](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status) to be excluded from error consideration.
|
|
393
|
+
* By default, status codes >= 500 trigger errors.
|
|
362
394
|
*/
|
|
363
|
-
|
|
395
|
+
ignoreHttpErrorStatusCodes?: number[];
|
|
364
396
|
/**
|
|
365
|
-
*
|
|
366
|
-
*
|
|
397
|
+
* An array of additional HTTP response [Status Codes](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status) to be treated as errors.
|
|
398
|
+
* By default, status codes >= 500 trigger errors.
|
|
367
399
|
*/
|
|
368
|
-
|
|
400
|
+
additionalHttpErrorStatusCodes?: number[];
|
|
401
|
+
}
|
|
402
|
+
export declare class BasicCrawler<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> {
|
|
403
|
+
#private;
|
|
369
404
|
/**
|
|
370
|
-
*
|
|
371
|
-
*
|
|
372
|
-
* Only available if used by the crawler.
|
|
405
|
+
* The statistics instance collecting the crawler's run statistics - either the injected `statistics` option or a
|
|
406
|
+
* crawler-built default. Typed as {@link IStatistics} so custom implementations can be plugged in.
|
|
373
407
|
*/
|
|
374
|
-
|
|
408
|
+
get statistics(): IStatistics<StatisticStateExtension>;
|
|
375
409
|
/**
|
|
376
|
-
* The main request-handling component of the crawler. It
|
|
410
|
+
* The main request-handling component of the crawler. It manages the requests that the crawler processes,
|
|
411
|
+
* combining any provided request loader and/or queue. It's initialized during the crawler startup or lazily
|
|
412
|
+
* via {@link BasicCrawler.getRequestManager|`getRequestManager()`}.
|
|
377
413
|
*/
|
|
378
|
-
protected requestManager
|
|
414
|
+
protected get requestManager(): IRequestManager | undefined;
|
|
379
415
|
/**
|
|
380
|
-
* A reference to the underlying
|
|
381
|
-
*
|
|
416
|
+
* A reference to the underlying session pool that manages the crawler's {@link Session|sessions}. Typed as
|
|
417
|
+
* {@link ISessionPool} so custom implementations can be plugged in via the `sessionPool` constructor option.
|
|
382
418
|
*/
|
|
383
|
-
sessionPool
|
|
419
|
+
get sessionPool(): ISessionPool;
|
|
420
|
+
/** Whether a {@link BasicCrawler.run|`run()`} is in progress on this instance. */
|
|
421
|
+
get running(): boolean;
|
|
384
422
|
/**
|
|
385
|
-
*
|
|
386
|
-
*
|
|
387
|
-
*
|
|
388
|
-
*
|
|
389
|
-
*
|
|
423
|
+
* The concurrency governor the run in progress is booking its requests against — either the
|
|
424
|
+
* {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} that was injected, or the default the
|
|
425
|
+
* crawler built for itself. Read it for telemetry: `desiredConcurrency`, `currentConcurrency`, `isRunning`.
|
|
426
|
+
*
|
|
427
|
+
* > *NOTE:* `undefined` outside a run, and a crawler-owned default is rebuilt for every run — so read it
|
|
428
|
+
* during a run rather than caching it across runs.
|
|
429
|
+
*
|
|
430
|
+
* {@link IConcurrencySystem} is deliberately read-only. Tuning concurrency *while a crawl is running* means
|
|
431
|
+
* owning the instance: build a {@link ConcurrencySystem} yourself and inject it, then set
|
|
432
|
+
* `minConcurrency`/`maxConcurrency`/`desiredConcurrency` on your own reference.
|
|
390
433
|
*/
|
|
391
|
-
|
|
434
|
+
get concurrencySystem(): IConcurrencySystem | undefined;
|
|
392
435
|
/**
|
|
393
|
-
* A reference to the underlying {@link
|
|
436
|
+
* A reference to the underlying {@link IProxyConfiguration} instance that manages the crawler's proxies.
|
|
394
437
|
* Only available if used by the crawler.
|
|
395
438
|
*/
|
|
396
|
-
proxyConfiguration?:
|
|
439
|
+
readonly proxyConfiguration?: IProxyConfiguration;
|
|
397
440
|
/**
|
|
398
441
|
* Default {@link Router} instance that will be used if we don't specify any {@link BasicCrawlerOptions.requestHandler|`requestHandler`}.
|
|
399
442
|
* See {@link Router.addHandler|`router.addHandler()`} and {@link Router.addDefaultHandler|`router.addDefaultHandler()`}.
|
|
400
443
|
*/
|
|
401
|
-
readonly router: RouterHandler<Context>;
|
|
402
|
-
|
|
403
|
-
private _contextPipeline?;
|
|
444
|
+
readonly router: RouterHandler<Context, Routes>;
|
|
445
|
+
/** @internal */
|
|
404
446
|
get contextPipeline(): ContextPipeline<CrawlingContext, ExtendedContext>;
|
|
405
|
-
|
|
406
|
-
hasFinishedBefore: boolean;
|
|
407
|
-
|
|
408
|
-
protected requestHandler: RequestHandler<ExtendedContext>;
|
|
409
|
-
protected
|
|
410
|
-
protected
|
|
411
|
-
protected
|
|
412
|
-
protected
|
|
413
|
-
protected
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
protected maxSessionRotations: number;
|
|
418
|
-
protected maxRequestsPerCrawl?: number;
|
|
419
|
-
protected handledRequestsCount: number;
|
|
420
|
-
protected statusMessageLoggingInterval: number;
|
|
421
|
-
protected statusMessageCallback?: StatusMessageCallback;
|
|
422
|
-
protected sessionPoolOptions: SessionPoolOptions;
|
|
423
|
-
protected useSessionPool: boolean;
|
|
424
|
-
protected autoscaledPoolOptions: AutoscaledPoolOptions;
|
|
425
|
-
protected events: EventManager;
|
|
426
|
-
protected httpClient: BaseHttpClient;
|
|
427
|
-
protected retryOnBlocked: boolean;
|
|
428
|
-
protected respectRobotsTxtFile: boolean | {
|
|
429
|
-
userAgent?: string;
|
|
430
|
-
};
|
|
431
|
-
protected onSkippedRequest?: SkippedRequestCallback;
|
|
432
|
-
private _closeEvents?;
|
|
433
|
-
private shouldLogMaxProcessedRequestsExceeded;
|
|
434
|
-
private shouldLogMaxEnqueuedRequestsExceeded;
|
|
435
|
-
private experiments;
|
|
436
|
-
private readonly robotsTxtFileCache;
|
|
437
|
-
private _experimentWarnings;
|
|
438
|
-
private readonly crawlerId;
|
|
439
|
-
private readonly hasExplicitId;
|
|
447
|
+
/** Whether a `run()` on this instance has already finished - a repeated one continues where it left off. */
|
|
448
|
+
get hasFinishedBefore(): boolean;
|
|
449
|
+
get log(): CrawleeLogger;
|
|
450
|
+
protected readonly requestHandler: RequestHandler<ExtendedContext>;
|
|
451
|
+
protected readonly internalTimeoutMillis: number;
|
|
452
|
+
protected blockedStatusCodes: Set<number>;
|
|
453
|
+
protected readonly additionalHttpErrorStatusCodes: Set<number>;
|
|
454
|
+
protected readonly httpClient: BaseHttpClient;
|
|
455
|
+
protected readonly retryOnBlocked: boolean;
|
|
456
|
+
/**
|
|
457
|
+
* @internal
|
|
458
|
+
*/
|
|
440
459
|
protected static optionsShape: {
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
485
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
486
|
-
httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
487
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
488
|
-
minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
489
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
490
|
-
maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
491
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
492
|
-
maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
493
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
494
|
-
keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
495
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
496
|
-
log: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
497
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
498
|
-
experiments: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
499
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
500
|
-
statisticsOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
501
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
502
|
-
id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
|
|
460
|
+
contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
461
|
+
extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
462
|
+
requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
463
|
+
requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
464
|
+
requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
465
|
+
requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
466
|
+
requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
467
|
+
errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
468
|
+
failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
469
|
+
maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
470
|
+
sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
471
|
+
maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
472
|
+
maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
473
|
+
taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
474
|
+
concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
475
|
+
sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
476
|
+
proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
477
|
+
statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
478
|
+
statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
479
|
+
additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
480
|
+
ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
481
|
+
blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
482
|
+
retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
|
|
483
|
+
respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
|
|
484
|
+
transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
|
|
485
|
+
requestQueue: z.ZodOptional<z.ZodEnum<{
|
|
486
|
+
deferred: "deferred";
|
|
487
|
+
writeThrough: "writeThrough";
|
|
488
|
+
}>>;
|
|
489
|
+
}, z.core.$strict>]>>;
|
|
490
|
+
onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
491
|
+
httpClient: z.ZodOptional<z.ZodInstanceOf<BaseHttpClient>>;
|
|
492
|
+
configuration: z.ZodOptional<z.ZodInstanceOf<Configuration>>;
|
|
493
|
+
storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
494
|
+
eventManager: z.ZodOptional<z.ZodInstanceOf<EventManager>>;
|
|
495
|
+
logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
496
|
+
minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
497
|
+
maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
498
|
+
initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
499
|
+
maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
500
|
+
keepAlive: z.ZodOptional<z.ZodBoolean>;
|
|
501
|
+
statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
502
|
+
id: z.ZodOptional<z.ZodString>;
|
|
503
503
|
};
|
|
504
504
|
/**
|
|
505
505
|
* All `BasicCrawler` parameters are passed via an options object.
|
|
506
506
|
*/
|
|
507
|
-
constructor(options?: BasicCrawlerOptions<Context, ContextExtension, ExtendedContext> & RequireContextPipeline<CrawlingContext, Context
|
|
508
|
-
|
|
507
|
+
constructor(options?: BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> & RequireContextPipeline<CrawlingContext, Context>);
|
|
508
|
+
/**
|
|
509
|
+
* Builds the crawler-owned default {@link ConcurrencySystem} from the resolved
|
|
510
|
+
* `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute` shortcuts. Not called when a
|
|
511
|
+
* {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} was injected.
|
|
512
|
+
*
|
|
513
|
+
* Subclasses may override this to tune the default system (e.g. {@link HttpCrawler} raises the starting
|
|
514
|
+
* concurrency and relaxes the event loop signal) while still honouring the user's shortcuts.
|
|
515
|
+
*/
|
|
516
|
+
protected createDefaultConcurrencySystem(options: ConcurrencySystemOptions): ConcurrencySystem;
|
|
517
|
+
/**
|
|
518
|
+
* Determines if the given HTTP status code is an error status code given
|
|
519
|
+
* the default behaviour and user-set preferences.
|
|
520
|
+
* @param status
|
|
521
|
+
* @returns `true` if the status code is considered an error, `false` otherwise
|
|
522
|
+
*/
|
|
523
|
+
protected isErrorStatusCode(status: number): boolean;
|
|
509
524
|
/**
|
|
510
525
|
* Checks if the given error is a proxy error by comparing its message to a list of known proxy error messages.
|
|
511
526
|
* Used for retrying requests that failed due to proxy errors.
|
|
@@ -514,21 +529,31 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
|
|
|
514
529
|
*/
|
|
515
530
|
protected isProxyError(error: Error): boolean;
|
|
516
531
|
/**
|
|
532
|
+
* Sets the status message for the current crawler run.
|
|
533
|
+
*
|
|
517
534
|
* This method is periodically called by the crawler, every `statusMessageLoggingInterval` seconds.
|
|
535
|
+
*
|
|
536
|
+
* The message is logged and broadcast via the {@link EventType.STATUS_MESSAGE|`statusMessage`}
|
|
537
|
+
* event. Integrations such as the Apify SDK subscribe to that event and forward the message to
|
|
538
|
+
* their status-reporting backend (e.g. the Apify platform).
|
|
518
539
|
*/
|
|
519
|
-
setStatusMessage(message: string, options?: SetStatusMessageOptions):
|
|
520
|
-
private getPeriodicLogger;
|
|
540
|
+
setStatusMessage(message: string, options?: SetStatusMessageOptions): void;
|
|
521
541
|
/**
|
|
522
|
-
* Runs the crawler. Returns a promise that resolves once
|
|
523
|
-
*
|
|
542
|
+
* Runs the crawler. Returns a promise that resolves once every request has been processed and the crawler's
|
|
543
|
+
* finished-check ({@link BasicCrawlerOptions.taskLoopOptions|`taskLoopOptions.isFinishedFunction`}, or the
|
|
544
|
+
* default "the request manager is empty") reports that the crawl is over.
|
|
524
545
|
*
|
|
525
546
|
* We can use the `requests` parameter to enqueue the initial requests — it is a shortcut for
|
|
526
547
|
* running {@link BasicCrawler.addRequests|`crawler.addRequests()`} before {@link BasicCrawler.run|`crawler.run()`}.
|
|
527
548
|
*
|
|
549
|
+
* Calling `run()` again on the same instance keeps crawling the same request manager - requests the previous
|
|
550
|
+
* run handled (a failed one counts as handled) are not processed again. Purge the queue or open a fresh one
|
|
551
|
+
* if that is what you want.
|
|
552
|
+
*
|
|
528
553
|
* @param [requests] The requests to add.
|
|
529
|
-
* @param [options] Options for the
|
|
554
|
+
* @param [options] Options for adding the initial requests.
|
|
530
555
|
*/
|
|
531
|
-
run(requests?:
|
|
556
|
+
run(requests?: TypedRequestsLike<Routes>, options?: CrawlerRunOptions): Promise<FinalStatistics>;
|
|
532
557
|
/**
|
|
533
558
|
* Gracefully stops the current run of the crawler.
|
|
534
559
|
*
|
|
@@ -536,32 +561,57 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
|
|
|
536
561
|
*
|
|
537
562
|
* To stop the crawler immediately, use {@link BasicCrawler.teardown|`crawler.teardown()`} instead.
|
|
538
563
|
*/
|
|
539
|
-
stop(
|
|
540
|
-
|
|
564
|
+
stop(reason?: string): void;
|
|
565
|
+
/**
|
|
566
|
+
* Stops dispatching new requests, letting the in-progress ones finish. Resolves once they have settled, or rejects
|
|
567
|
+
* after `timeoutSecs` if they take too long. Unlike {@link BasicCrawler.stop|`stop()`}, this does not end the
|
|
568
|
+
* run — {@link BasicCrawler.run|`run()`} stays pending until {@link BasicCrawler.resume|`resume()`}.
|
|
569
|
+
*
|
|
570
|
+
* > *NOTE:* The {@link BasicCrawler.concurrencySystem|concurrency system} keeps monitoring and autoscaling
|
|
571
|
+
* throughout, since a shared one may still be serving other crawlers.
|
|
572
|
+
*/
|
|
573
|
+
pause(timeoutSecs?: number): Promise<void>;
|
|
574
|
+
/**
|
|
575
|
+
* Resumes a run suspended with {@link BasicCrawler.pause|`pause()`}, letting the crawler dispatch requests
|
|
576
|
+
* again. A no-op on a crawler that is not paused.
|
|
577
|
+
*/
|
|
578
|
+
resume(): void;
|
|
579
|
+
/**
|
|
580
|
+
* Returns the crawler's {@link IRequestManager|request manager}, opening the default {@link RequestQueue}
|
|
581
|
+
* if none has been configured or opened yet.
|
|
582
|
+
*/
|
|
583
|
+
getRequestManager(): Promise<IRequestManager>;
|
|
584
|
+
/**
|
|
585
|
+
* @deprecated Use {@link BasicCrawler.getRequestManager|`getRequestManager()`} instead. This returns the
|
|
586
|
+
* crawler's request manager, which is no longer guaranteed to be a {@link RequestQueue}.
|
|
587
|
+
*/
|
|
588
|
+
getRequestQueue(): Promise<IRequestManager>;
|
|
541
589
|
useState<State extends Dictionary = Dictionary>(defaultValue?: State): Promise<State>;
|
|
542
|
-
protected get pendingRequestCountApproximation(): number;
|
|
543
|
-
protected calculateEnqueuedRequestLimit(explicitLimit?: number): number | undefined;
|
|
544
|
-
protected handleSkippedRequest(options: Parameters<SkippedRequestCallback>[0]): Promise<void>;
|
|
545
590
|
/**
|
|
546
591
|
* Adds requests to the queue in batches. By default, it will resolve after the initial batch is added, and continue
|
|
547
592
|
* adding the rest in background. You can configure the batch size via `batchSize` option and the sleep time in between
|
|
548
593
|
* the batches via `waitBetweenBatchesMillis`. If you want to wait for all batches to be added to the queue, you can use
|
|
549
594
|
* the `waitForAllRequestsToBeAdded` promise you get in the response object.
|
|
550
595
|
*
|
|
596
|
+
* Optionally, the requests can be filtered using `include`/`exclude` glob or regexp patterns and an
|
|
597
|
+
* enqueue `strategy` (both AND-ed together, same as {@link CrawlingContext.enqueueLinks|`enqueueLinks`}),
|
|
598
|
+
* relative to `baseUrl`. Unlike `enqueueLinks`, there is no implicit "current page" to anchor the strategy
|
|
599
|
+
* to, so `strategy` defaults to {@link EnqueueStrategy.All|`all`} here.
|
|
600
|
+
*
|
|
551
601
|
* This is an alias for calling `addRequestsBatched()` on the implicit `RequestQueue` for this crawler instance.
|
|
552
602
|
*
|
|
553
603
|
* @param requests The requests to add
|
|
554
604
|
* @param options Options for the request queue
|
|
555
605
|
*/
|
|
556
|
-
addRequests(requests: ReadonlyDeep<
|
|
606
|
+
addRequests(requests: ReadonlyDeep<TypedRequestsLike<Routes>>, options?: CrawlerAddRequestsOptions): Promise<CrawlerAddRequestsResult>;
|
|
557
607
|
/**
|
|
558
608
|
* Pushes data to the specified {@link Dataset}, or the default crawler {@link Dataset} by calling {@link Dataset.pushData}.
|
|
559
609
|
*/
|
|
560
|
-
pushData(data: Parameters<Dataset['pushData']>[0],
|
|
610
|
+
pushData(data: Parameters<Dataset['pushData']>[0], datasetIdentifier?: string | StorageIdentifier): Promise<void>;
|
|
561
611
|
/**
|
|
562
612
|
* Retrieves the specified {@link Dataset}, or the default crawler {@link Dataset}.
|
|
563
613
|
*/
|
|
564
|
-
getDataset(
|
|
614
|
+
getDataset(identifier?: string | StorageIdentifier): Promise<Dataset>;
|
|
565
615
|
/**
|
|
566
616
|
* Retrieves data from the default crawler {@link Dataset} by calling {@link Dataset.getData}.
|
|
567
617
|
*/
|
|
@@ -574,112 +624,76 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
|
|
|
574
624
|
/**
|
|
575
625
|
* Initializes the crawler.
|
|
576
626
|
*/
|
|
577
|
-
protected
|
|
578
|
-
protected runRequestHandler(crawlingContext: CrawlingContext): Promise<void>;
|
|
627
|
+
protected init(): Promise<void>;
|
|
579
628
|
/**
|
|
580
|
-
*
|
|
629
|
+
* The navigation timeout (pre-navigation hooks, navigation, and post-navigation hooks) in milliseconds, used
|
|
630
|
+
* to size the internal request timeout. `BasicCrawler` has no navigation phase, so this is 0; the HTTP and
|
|
631
|
+
* browser crawlers override it with their `navigationTimeoutSecs`.
|
|
581
632
|
*/
|
|
582
|
-
protected
|
|
583
|
-
|
|
584
|
-
protected getRobotsTxtFileForUrl(url: string): Promise<RobotsTxtFile | undefined>;
|
|
585
|
-
protected _pauseOnMigration(): Promise<void>;
|
|
633
|
+
protected getNavigationTimeoutMillis(): number;
|
|
634
|
+
protected runRequestHandler(crawlingContext: ExtendedContext): Promise<void>;
|
|
586
635
|
/**
|
|
587
|
-
*
|
|
636
|
+
* Runs `callback` inside a {@link StorageTransaction}, unless transactional storage is disabled.
|
|
637
|
+
* Deliberately does **not** commit on return - the pipeline hands failures to `handleRequestFailure`, so a
|
|
638
|
+
* normal return says nothing about success. `handleRequest` commits on success.
|
|
588
639
|
*/
|
|
589
|
-
private
|
|
640
|
+
private runInStorageTransaction;
|
|
590
641
|
/**
|
|
591
|
-
*
|
|
592
|
-
*/
|
|
593
|
-
protected _fetchNextRequest(): Promise<Request<Dictionary> | null>;
|
|
594
|
-
/**
|
|
595
|
-
* Delays processing of the request based on the `sameDomainDelaySecs` option,
|
|
596
|
-
* adding it back to the queue after the timeout passes. Returns `true` if the request
|
|
597
|
-
* should be ignored and will be reclaimed to the queue once ready.
|
|
598
|
-
*/
|
|
599
|
-
protected delayRequest(request: Request, source: IRequestList | RequestProvider | IRequestManager): boolean;
|
|
600
|
-
/**
|
|
601
|
-
* Wrapper around requestHandler that fetches requests from RequestList/RequestQueue
|
|
602
|
-
* then retries them in a case of an error, etc.
|
|
642
|
+
* Handles blocked request
|
|
603
643
|
*/
|
|
604
|
-
protected
|
|
644
|
+
protected throwOnBlockedRequest(statusCode: number): void;
|
|
605
645
|
/**
|
|
606
|
-
*
|
|
607
|
-
*
|
|
608
|
-
*
|
|
609
|
-
*
|
|
610
|
-
*
|
|
646
|
+
* Records an HTTP 429 against the URL's domain so the request manager can hold the retry back.
|
|
647
|
+
*
|
|
648
|
+
* @param retryAfterHeader The raw `Retry-After` response header, if the server sent one.
|
|
649
|
+
* @returns `true` if the manager took responsibility for the delay, in which case the caller should throw
|
|
650
|
+
* {@link RequestThrottledError} rather than treating the response as a blocked session.
|
|
611
651
|
*/
|
|
612
|
-
protected
|
|
652
|
+
protected recordDomainRateLimit(url: string, retryAfterHeader?: string | null): boolean;
|
|
653
|
+
/** @internal */
|
|
654
|
+
protected getRobotsTxtFileForUrl(url: string): Promise<RobotsTxtFile | undefined>;
|
|
655
|
+
/** Runs the request handler and marks the request as handled. Failures are left to `handleRequestFailure`. */
|
|
656
|
+
private handleRequest;
|
|
613
657
|
/**
|
|
614
658
|
* Generator function that yields requests injected with the given crawl depth.
|
|
615
659
|
* @internal
|
|
616
660
|
*/
|
|
617
661
|
protected addCrawlDepthRequestGenerator(requests: RequestsLike, newRequestDepth: number): AsyncGenerator<Source, void, undefined>;
|
|
618
|
-
/**
|
|
619
|
-
* Run async callback with given timeout and retry. Returns the result of the callback.
|
|
620
|
-
* @ignore
|
|
621
|
-
*/
|
|
622
|
-
protected _timeoutAndRetry<T>(handler: () => Promise<T>, timeout: number, error: Error | string, maxRetries?: number, retried?: number): Promise<T>;
|
|
623
|
-
/**
|
|
624
|
-
* Returns true if either RequestList or RequestQueue have a request ready for processing.
|
|
625
|
-
*/
|
|
626
|
-
protected _isTaskReadyFunction(): Promise<boolean>;
|
|
627
|
-
/**
|
|
628
|
-
* Returns true if both RequestList and RequestQueue have all requests finished.
|
|
629
|
-
*/
|
|
630
|
-
protected _defaultIsFinishedFunction(): Promise<boolean>;
|
|
631
|
-
private _rotateSession;
|
|
632
|
-
/**
|
|
633
|
-
* Unwraps errors thrown by the context pipeline to get the actual user error.
|
|
634
|
-
* RequestHandlerError and ContextPipelineInitializationError wrap the actual error.
|
|
635
|
-
*/
|
|
636
|
-
private unwrapError;
|
|
637
662
|
/**
|
|
638
663
|
* Handles errors thrown by user provided requestHandler()
|
|
639
664
|
*/
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
protected _handleFailedRequestHandler(crawlingContext: CrawlingContext, error: Error): Promise<void>;
|
|
665
|
+
private requestFunctionErrorHandler;
|
|
666
|
+
private handleFailedRequestHandler;
|
|
643
667
|
/**
|
|
644
668
|
* Resolves the most verbose error message from a thrown error
|
|
645
669
|
* @param error The error received
|
|
646
670
|
* @returns The message to be logged
|
|
647
671
|
*/
|
|
648
|
-
protected
|
|
649
|
-
protected _canRequestBeRetried(request: Request, error: Error): boolean;
|
|
672
|
+
protected getMessageFromError(error: Error, forceStack?: boolean): string;
|
|
650
673
|
/**
|
|
651
|
-
*
|
|
652
|
-
*/
|
|
653
|
-
protected _loadHandledRequestCount(): Promise<void>;
|
|
654
|
-
protected _executeHooks<HookLike extends (...args: any[]) => Awaitable<void>>(hooks: HookLike[], ...args: Parameters<HookLike>): Promise<void>;
|
|
655
|
-
/**
|
|
656
|
-
* Stops the crawler immediately.
|
|
674
|
+
* Ends the run in progress immediately, without waiting for the requests already in flight.
|
|
657
675
|
*
|
|
658
|
-
* This
|
|
676
|
+
* This runs at the end of every {@link BasicCrawler.run|`run()`}, so what it releases is per-run only, and
|
|
677
|
+
* a repeated `run()` has to find everything it needs either intact or re-establishable. What outlives a run is
|
|
678
|
+
* released by {@link BasicCrawler.destroy|`destroy()`} instead.
|
|
659
679
|
*
|
|
660
680
|
* To stop the crawler gracefully (waiting for all running requests to finish), use {@link BasicCrawler.stop|`crawler.stop()`} instead.
|
|
661
681
|
*/
|
|
662
682
|
teardown(): Promise<void>;
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
683
|
+
/**
|
|
684
|
+
* Releases what the crawler owns beyond a single run — in the browser crawlers, the browser pool. A finished
|
|
685
|
+
* `run()` has already released everything a run owns, so this is only needed before dropping a crawler you
|
|
686
|
+
* could otherwise have run again.
|
|
687
|
+
*/
|
|
688
|
+
destroy(): Promise<void>;
|
|
689
|
+
[Symbol.asyncDispose](): Promise<void>;
|
|
690
|
+
protected getCookieHeaderFromRequest(request: Request): string;
|
|
671
691
|
}
|
|
672
|
-
export interface CrawlerAddRequestsOptions extends AddRequestsBatchedOptions {
|
|
692
|
+
export interface CrawlerAddRequestsOptions extends AddRequestsBatchedOptions, EnqueueUrlsOptions {
|
|
673
693
|
}
|
|
674
694
|
export interface CrawlerAddRequestsResult extends AddRequestsBatchedResult {
|
|
675
695
|
}
|
|
676
696
|
export interface CrawlerRunOptions extends CrawlerAddRequestsOptions {
|
|
677
|
-
/**
|
|
678
|
-
* Whether to purge the RequestQueue before running the crawler again. Defaults to true, so it is possible to reprocess failed requests.
|
|
679
|
-
* When disabled, only new requests will be considered. Note that even a failed request is considered as handled.
|
|
680
|
-
* @default true
|
|
681
|
-
*/
|
|
682
|
-
purgeRequestQueue?: boolean;
|
|
683
697
|
}
|
|
684
698
|
/**
|
|
685
699
|
* Creates new {@link Router} instance that works based on request labels.
|
|
@@ -705,5 +719,5 @@ export interface CrawlerRunOptions extends CrawlerAddRequestsOptions {
|
|
|
705
719
|
* await crawler.run();
|
|
706
720
|
* ```
|
|
707
721
|
*/
|
|
708
|
-
export declare function createBasicRouter<Context extends BasicCrawlingContext = BasicCrawlingContext,
|
|
709
|
-
|
|
722
|
+
export declare function createBasicRouter<Context extends BasicCrawlingContext = BasicCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
|
|
723
|
+
export declare function createBasicRouter<Context extends BasicCrawlingContext = BasicCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
|