@crawlee/basic 4.0.0-beta.27 → 4.0.0-beta.271

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/README.md +15 -15
  2. package/index.d.ts +11 -2
  3. package/index.js +11 -1
  4. package/internals/autoscaling/autoscaled-pool.d.ts +178 -0
  5. package/internals/autoscaling/autoscaled-pool.js +389 -0
  6. package/internals/autoscaling/concurrency-system.d.ts +235 -0
  7. package/internals/autoscaling/concurrency-system.js +357 -0
  8. package/internals/autoscaling/cpu-load-signal.d.ts +43 -0
  9. package/internals/autoscaling/cpu-load-signal.js +46 -0
  10. package/internals/autoscaling/event-loop-load-signal.d.ts +51 -0
  11. package/internals/autoscaling/event-loop-load-signal.js +60 -0
  12. package/internals/autoscaling/index.d.ts +9 -0
  13. package/internals/autoscaling/index.js +9 -0
  14. package/internals/autoscaling/load-signal.d.ts +98 -0
  15. package/internals/autoscaling/load-signal.js +103 -0
  16. package/internals/autoscaling/memory-load-signal.d.ts +45 -0
  17. package/internals/autoscaling/memory-load-signal.js +104 -0
  18. package/internals/autoscaling/snapshotter.d.ts +84 -0
  19. package/internals/autoscaling/snapshotter.js +67 -0
  20. package/internals/autoscaling/storage-backend-load-signal.d.ts +56 -0
  21. package/internals/autoscaling/storage-backend-load-signal.js +73 -0
  22. package/internals/autoscaling/system-status.d.ts +112 -0
  23. package/internals/autoscaling/system-status.js +139 -0
  24. package/internals/autoscaling/weighted-avg.d.ts +5 -0
  25. package/internals/autoscaling/weighted-avg.js +14 -0
  26. package/internals/basic-crawler.d.ts +383 -369
  27. package/internals/basic-crawler.js +1468 -890
  28. package/internals/cookie-utils.d.ts +40 -0
  29. package/internals/cookie-utils.js +122 -0
  30. package/internals/crawler-run.d.ts +76 -0
  31. package/internals/crawler-run.js +144 -0
  32. package/internals/crawlers/context-pipeline.d.ts +75 -0
  33. package/internals/crawlers/context-pipeline.js +109 -0
  34. package/internals/crawlers/crawler-commons.d.ts +216 -0
  35. package/internals/crawlers/crawler-commons.js +1 -0
  36. package/internals/crawlers/error-snapshotter.d.ts +29 -0
  37. package/internals/crawlers/error-snapshotter.js +116 -0
  38. package/internals/crawlers/error-tracker.d.ts +51 -0
  39. package/internals/crawlers/error-tracker.js +316 -0
  40. package/internals/crawlers/index.d.ts +4 -0
  41. package/internals/crawlers/index.js +3 -0
  42. package/internals/crawlers/internals/types.d.ts +7 -0
  43. package/internals/crawlers/internals/types.js +1 -0
  44. package/internals/crawlers/statistics.d.ts +326 -0
  45. package/internals/crawlers/statistics.js +559 -0
  46. package/internals/crawling-request.d.ts +86 -0
  47. package/internals/crawling-request.js +127 -0
  48. package/internals/enqueue-links/enqueue-links.d.ts +147 -0
  49. package/internals/enqueue-links/enqueue-links.js +35 -0
  50. package/internals/enqueue-links/index.d.ts +3 -0
  51. package/internals/enqueue-links/index.js +3 -0
  52. package/internals/enqueue-links/shared.d.ts +107 -0
  53. package/internals/enqueue-links/shared.js +228 -0
  54. package/internals/errors.d.ts +73 -0
  55. package/internals/errors.js +108 -0
  56. package/internals/http.d.ts +9 -0
  57. package/internals/http.js +28 -0
  58. package/internals/request-timeout.d.ts +54 -0
  59. package/internals/request-timeout.js +98 -0
  60. package/internals/router.d.ts +299 -0
  61. package/internals/router.js +312 -0
  62. package/internals/send-request.d.ts +4 -4
  63. package/internals/send-request.js +18 -8
  64. package/internals/session-pool/consts.d.ts +1 -0
  65. package/internals/session-pool/consts.js +1 -0
  66. package/internals/session-pool/errors.d.ts +7 -0
  67. package/internals/session-pool/errors.js +11 -0
  68. package/internals/session-pool/fingerprint.d.ts +9 -0
  69. package/internals/session-pool/fingerprint.js +30 -0
  70. package/internals/session-pool/index.d.ts +4 -0
  71. package/internals/session-pool/index.js +4 -0
  72. package/internals/session-pool/session-pool.d.ts +193 -0
  73. package/internals/session-pool/session-pool.js +453 -0
  74. package/internals/session-pool/session.d.ts +151 -0
  75. package/internals/session-pool/session.js +225 -0
  76. package/internals/sitemap-request-loader.d.ts +158 -0
  77. package/internals/sitemap-request-loader.js +427 -0
  78. package/internals/throttling-request-manager.d.ts +229 -0
  79. package/internals/throttling-request-manager.js +860 -0
  80. package/package.json +23 -17
  81. package/index.d.ts.map +0 -1
  82. package/index.js.map +0 -1
  83. package/internals/basic-crawler.d.ts.map +0 -1
  84. package/internals/basic-crawler.js.map +0 -1
  85. package/internals/send-request.d.ts.map +0 -1
  86. package/internals/send-request.js.map +0 -1
@@ -1,25 +1,44 @@
1
- import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, AutoscaledPoolOptions, CrawlingContext, DatasetExportOptions, EnqueueLinksOptions, EventManager, FinalStatistics, GetUserDataFromRequest, IRequestList, IRequestManager, ProxyConfiguration, Request, RequestsLike, RouterHandler, RouterRoutes, Session, SessionPoolOptions, SkippedRequestCallback, Source, StatisticsOptions, StatisticState } from '@crawlee/core';
2
- import { AutoscaledPool, Configuration, ContextPipeline, Dataset, RequestProvider, SessionPool, Statistics } from '@crawlee/core';
3
- import type { Awaitable, BaseHttpClient, BatchAddRequestsResult, Dictionary, ProxyInfo, SetStatusMessageOptions } from '@crawlee/types';
1
+ import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, CrawleeLogger, DatasetExportOptions, IProxyConfiguration, IRequestLoader, IRequestManager, RequestsLike, Source, StorageIdentifier, StorageWritePolicy } from '@crawlee/core';
2
+ import { Configuration, Request, Dataset, EventManager, RequestQueue } from '@crawlee/core';
3
+ import { BaseHttpClient } from '@crawlee/http-client';
4
+ import type { Awaitable, Dictionary, ISessionPool, SetStatusMessageOptions, StorageBackend } from '@crawlee/types';
4
5
  import { RobotsTxtFile } from '@crawlee/utils';
5
- import type { ReadonlyDeep, SetRequired } from 'type-fest';
6
- import type { Log } from '@apify/log';
7
- import { TimeoutError } from '@apify/timeout';
6
+ import type { ReadonlyDeep } from 'type-fest';
7
+ import { z } from 'zod';
8
+ import type { TaskLoopPredicates } from './autoscaling/autoscaled-pool.js';
9
+ import type { ConcurrencySystemOptions, IConcurrencySystem } from './autoscaling/concurrency-system.js';
10
+ import { ConcurrencySystem } from './autoscaling/concurrency-system.js';
11
+ import type { FinalStatistics } from './autoscaling/system-status.js';
12
+ import type { ContextMiddleware } from './crawlers/context-pipeline.js';
13
+ import { ContextPipeline } from './crawlers/context-pipeline.js';
14
+ import type { CrawlingContext, TypedRequestsLike } from './crawlers/crawler-commons.js';
15
+ import type { IStatistics, StatisticState } from './crawlers/statistics.js';
16
+ import type { EnqueueUrlsOptions, SkippedRequestCallback } from './enqueue-links/index.js';
17
+ import type { GetUserDataFromRequest, RouterHandler, RouterRoutes } from './router.js';
8
18
  export interface BasicCrawlingContext<UserData extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
9
19
  }
20
+ export { navigationDeadlineKey, remainingNavigationWindowMillis } from './request-timeout.js';
10
21
  export type RequestHandler<Context extends CrawlingContext = CrawlingContext> = (inputs: Context) => Awaitable<void>;
11
- export type ErrorHandler<Context extends CrawlingContext = CrawlingContext, ExtendedContext extends Context = Context> = (inputs: Context & Partial<ExtendedContext>, error: Error) => Awaitable<void>;
12
- export interface StatusMessageCallbackParams<Context extends CrawlingContext = BasicCrawlingContext, Crawler extends BasicCrawler<any> = BasicCrawler<Context>> {
22
+ /**
23
+ * An error handler receives the crawling context and the error that was thrown while processing the request.
24
+ *
25
+ * Unlike the {@link RequestHandler}, an error handler may run before the context pipeline has finished
26
+ * building the full context (e.g. when navigation or session setup fails). Therefore only `BaseContext` is
27
+ * guaranteed to be present, while the extra properties added by the pipeline and `extendContext` (the
28
+ * difference between `BaseContext` and `ExtendedContext`) are only available as a `Partial`.
29
+ */
30
+ export type ErrorHandler<BaseContext extends CrawlingContext = CrawlingContext, ExtendedContext extends BaseContext = BaseContext> = (inputs: BaseContext & Partial<ExtendedContext>, error: Error) => Awaitable<void>;
31
+ export interface StatusMessageCallbackParams<Context extends CrawlingContext = BasicCrawlingContext, Crawler extends BasicCrawler<any, any, any, any> = BasicCrawler<Context>> {
13
32
  state: StatisticState;
14
33
  crawler: Crawler;
15
34
  previousState: StatisticState;
16
35
  message: string;
17
36
  }
18
- export type StatusMessageCallback<Context extends CrawlingContext = BasicCrawlingContext, Crawler extends BasicCrawler<any> = BasicCrawler<Context>> = (params: StatusMessageCallbackParams<Context, Crawler>) => Awaitable<void>;
37
+ export type StatusMessageCallback<Context extends CrawlingContext = BasicCrawlingContext, Crawler extends BasicCrawler<any, any, any, any> = BasicCrawler<Context>> = (params: StatusMessageCallbackParams<Context, Crawler>) => Awaitable<void>;
19
38
  export type RequireContextPipeline<DefaultContextType extends CrawlingContext, FinalContextType extends DefaultContextType> = DefaultContextType extends FinalContextType ? {} : {
20
39
  contextPipelineBuilder: () => ContextPipeline<CrawlingContext, FinalContextType>;
21
40
  };
22
- export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension> {
41
+ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> {
23
42
  /**
24
43
  * User-provided function that performs the logic of the crawler. It is called for each URL to crawl.
25
44
  *
@@ -35,11 +54,21 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
35
54
  * To make this work, we should **always**
36
55
  * let our function throw exceptions rather than catch them.
37
56
  * The exceptions are logged to the request using the
38
- * {@link Request.pushErrorMessage|`Request.pushErrorMessage()`} function.
57
+ * {@link CrawlingRequest.pushErrorMessage|`request.pushErrorMessage()`} function.
39
58
  */
40
- requestHandler?: RequestHandler<ExtendedContext>;
59
+ requestHandler?: RouterHandler<ExtendedContext, Routes> | RequestHandler<ExtendedContext>;
41
60
  /**
42
- * Allows the user to extend the crawling context passed to the request handler with custom functionality.
61
+ * Allows the user to extend the crawling context with custom functionality (helpers, references, etc.).
62
+ *
63
+ * `extendContext` runs *before* navigation, so the returned members are visible to the
64
+ * `preNavigationHooks`, `postNavigationHooks`, and the `requestHandler` alike. As a consequence,
65
+ * the `context` passed to `extendContext` is the pre-navigation {@link CrawlingContext} and does
66
+ * **not** include navigation-dependent members (e.g. `page`, `response`, `$`, `body`). If you need
67
+ * those, use a `postNavigationHook` or the `requestHandler` instead.
68
+ *
69
+ * The second argument is the same {@link CleanupRegistrar|`onCleanup`} registrar used by
70
+ * {@link ContextPipeline.compose}. Use it to register teardown callbacks that run after the
71
+ * request handler or a pipeline failure, in reverse registration order.
43
72
  *
44
73
  * **Example usage:**
45
74
  *
@@ -59,7 +88,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
59
88
  * });
60
89
  * ```
61
90
  */
62
- extendContext?: (context: Context) => Awaitable<ContextExtension>;
91
+ extendContext?: ContextMiddleware<CrawlingContext, ContextExtension>;
63
92
  /**
64
93
  * *Intended for BasicCrawler subclasses*. Prepares a context pipeline that transforms the initial crawling context into the shape given by the `Context` type parameter.
65
94
  *
@@ -68,23 +97,25 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
68
97
  contextPipelineBuilder?: () => ContextPipeline<CrawlingContext, Context>;
69
98
  /**
70
99
  * Static list of URLs to be processed.
71
- * If not provided, the crawler will open the default request queue when the {@link BasicCrawler.addRequests|`crawler.addRequests()`} function is called.
72
- * > Alternatively, `requests` parameter of {@link BasicCrawler.run|`crawler.run()`} could be used to enqueue the initial requests -
73
- * it is a shortcut for running `crawler.addRequests()` before the `crawler.run()`.
100
+ *
101
+ * @deprecated Use the `requestManager` option instead. To combine a read-only loader (such as a `RequestList`)
102
+ * with a writable queue, build a tandem with {@link IRequestLoader.toTandem|`requestList.toTandem(requestQueue)`}
103
+ * and pass the result as `requestManager`. When both `requestList` and `requestQueue` are provided, they are
104
+ * combined into a tandem automatically.
74
105
  */
75
- requestList?: IRequestList;
106
+ requestList?: IRequestLoader;
76
107
  /**
77
108
  * Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
78
- * If not provided, the crawler will open the default request queue when the {@link BasicCrawler.addRequests|`crawler.addRequests()`} function is called.
79
- * > Alternatively, `requests` parameter of {@link BasicCrawler.run|`crawler.run()`} could be used to enqueue the initial requests -
80
- * it is a shortcut for running `crawler.addRequests()` before the `crawler.run()`.
109
+ *
110
+ * @deprecated Use the `requestManager` option instead. A `RequestQueue` is itself a request manager, so you can
111
+ * pass it directly as `requestManager`.
81
112
  */
82
- requestQueue?: RequestProvider;
113
+ requestQueue?: RequestQueue;
83
114
  /**
84
- * Allows explicitly configuring a request manager. Mutually exclusive with the `requestQueue` and `requestList` options.
115
+ * Manager of requests that should be processed by the crawler. Mutually exclusive with the deprecated
116
+ * `requestQueue` and `requestList` options.
85
117
  *
86
- * This enables explicitly configuring the crawler to use `RequestManagerTandem`, for instance.
87
- * If using this, the type of `BasicCrawler.requestQueue` may not be fully compatible with the `RequestProvider` class.
118
+ * If not provided, the crawler will open the default {@link RequestQueue} when it is first needed.
88
119
  */
89
120
  requestManager?: IRequestManager;
90
121
  /**
@@ -113,27 +144,21 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
113
144
  failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
114
145
  /**
115
146
  * Specifies the maximum number of retries allowed for a request if its processing fails.
116
- * This includes retries due to navigation errors or errors thrown from user-supplied functions
117
- * (`requestHandler`, `preNavigationHooks`, `postNavigationHooks`).
118
- *
119
- * This limit does not apply to retries triggered by session rotation
120
- * (see {@link BasicCrawlerOptions.maxSessionRotations|`maxSessionRotations`}).
147
+ * This includes retries due to navigation errors, session/proxy errors, or errors thrown from user-supplied
148
+ * functions (`requestHandler`, `preNavigationHooks`, `postNavigationHooks`).
121
149
  * @default 3
122
150
  */
123
151
  maxRequestRetries?: number;
124
152
  /**
125
- * Indicates how much time (in seconds) to wait before crawling another same domain request.
153
+ * Indicates how much time (in seconds) to wait before crawling another same domain request. Subdomains are
154
+ * paced together with the site they belong to.
155
+ *
156
+ * Offered to the crawler's request manager as a `minIntervalEverywhere` {@link PacingSignal}; a manager that
157
+ * already paces every domain it dispatches to takes it, so no domain ends up with two clocks. Otherwise the
158
+ * crawler wraps its request manager in a {@link ThrottlingRequestManager} of its own.
126
159
  * @default 0
127
160
  */
128
161
  sameDomainDelaySecs?: number;
129
- /**
130
- * Maximum number of session rotations per request.
131
- * The crawler will automatically rotate the session in case of a proxy error or if it gets blocked by the website.
132
- *
133
- * The session rotations are not counted towards the {@link BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} limit.
134
- * @default 10
135
- */
136
- maxSessionRotations?: number;
137
162
  /**
138
163
  * Maximum number of pages that the crawler will open. The crawl will stop when this limit is reached.
139
164
  * This value should always be set in order to prevent infinite loops in misconfigured crawlers.
@@ -147,46 +172,82 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
147
172
  */
148
173
  maxCrawlDepth?: number;
149
174
  /**
150
- * Custom options passed to the underlying {@link AutoscaledPool} constructor.
151
- * > *NOTE:* The {@link AutoscaledPoolOptions.runTaskFunction|`runTaskFunction`}
152
- * option is provided by the crawler and cannot be overridden.
153
- * However, we can provide custom implementations of {@link AutoscaledPoolOptions.isFinishedFunction|`isFinishedFunction`}
154
- * and {@link AutoscaledPoolOptions.isTaskReadyFunction|`isTaskReadyFunction`}.
175
+ * Lets you override the predicates that steer the crawler's task loop: `isTaskReadyFunction` (may another request
176
+ * start?) and `isFinishedFunction` (is the crawl over?). The task itself — fetching a request and running it
177
+ * through the pipeline — is owned by the crawler and cannot be overridden.
178
+ *
179
+ * Concurrency is configured elsewhere — through the `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute`
180
+ * shortcuts, or a {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} for finer control.
155
181
  */
156
- autoscaledPoolOptions?: AutoscaledPoolOptions;
182
+ taskLoopOptions?: TaskLoopPredicates;
183
+ /**
184
+ * A pre-configured concurrency governor — the component that decides whether there is free compute for one more
185
+ * task. Typically a {@link ConcurrencySystem}, though any {@link IConcurrencySystem} is accepted. All
186
+ * scaling configuration (min/max/desired concurrency, scaling ratios, `maxTasksPerMinute`, snapshotter tuning)
187
+ * lives on the instance itself.
188
+ *
189
+ * Inject the *same* instance into several concurrent crawlers to cap their **combined** concurrency against a
190
+ * single budget. Each crawler still builds and drives its own {@link AutoscaledPool}; only the load/scaling
191
+ * accounting is shared.
192
+ *
193
+ * Mutually exclusive with the `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute`
194
+ * shortcuts, which configure the default system this one replaces — combining the two throws.
195
+ *
196
+ * You own a supplied system's lifecycle: `start()` it before `run()` (which throws otherwise) and `stop()` it once
197
+ * every crawler borrowing it has finished. The crawler does neither on your behalf.
198
+ */
199
+ concurrencySystem?: IConcurrencySystem;
157
200
  /**
158
201
  * Sets the minimum concurrency (parallelism) for the crawl. Shortcut for the
159
- * AutoscaledPool {@link AutoscaledPoolOptions.minConcurrency|`minConcurrency`} option.
202
+ * {@link ConcurrencySystemOptions.minConcurrency|`minConcurrency`} option of the crawler's default
203
+ * {@link ConcurrencySystem}.
160
204
  * > *WARNING:* If we set this value too high with respect to the available system memory and CPU, our crawler will run extremely slow or crash.
161
205
  * If not sure, it's better to keep the default value and the concurrency will scale up automatically.
162
206
  */
163
207
  minConcurrency?: number;
164
208
  /**
165
209
  * Sets the maximum concurrency (parallelism) for the crawl. Shortcut for the
166
- * AutoscaledPool {@link AutoscaledPoolOptions.maxConcurrency|`maxConcurrency`} option.
210
+ * {@link ConcurrencySystemOptions.maxConcurrency|`maxConcurrency`} option of the crawler's default
211
+ * {@link ConcurrencySystem}.
167
212
  */
168
213
  maxConcurrency?: number;
214
+ /**
215
+ * Sets the concurrency (parallelism) the crawl starts with, before any scaling happens. Shortcut for the
216
+ * {@link ConcurrencySystemOptions.desiredConcurrency|`desiredConcurrency`} option of the crawler's default
217
+ * {@link ConcurrencySystem}. Defaults to `minConcurrency`.
218
+ */
219
+ initialConcurrency?: number;
169
220
  /**
170
221
  * The maximum number of requests per minute the crawler should run.
171
222
  * By default, this is set to `Infinity`, but we can pass any positive, non-zero integer.
172
- * Shortcut for the AutoscaledPool {@link AutoscaledPoolOptions.maxTasksPerMinute|`maxTasksPerMinute`} option.
223
+ * Shortcut for the {@link ConcurrencySystemOptions.maxTasksPerMinute|`maxTasksPerMinute`} option of the
224
+ * crawler's default {@link ConcurrencySystem}.
173
225
  */
174
226
  maxRequestsPerMinute?: number;
175
227
  /**
176
228
  * Allows to keep the crawler alive even if the {@link RequestQueue} gets empty.
177
229
  * By default, the `crawler.run()` will resolve once the queue is empty. With `keepAlive: true` it will keep running,
178
- * waiting for more requests to come. Use `crawler.stop()` to exit the crawler gracefully, or `crawler.teardown()` to stop it immediately.
230
+ * waiting for more requests to come.
231
+ *
232
+ * `crawler.stop()` only stops the crawler from dispatching new requests — it does not wait for or drain
233
+ * requests already sitting in the queue. If this crawler is fed by another crawler or an external producer,
234
+ * calling `stop()` as soon as the producer finishes can abandon requests the producer already enqueued but
235
+ * this crawler had not yet picked up.
236
+ *
237
+ * To shut down only once the queue is genuinely empty and the producer has stopped, override
238
+ * {@link BasicCrawlerOptions.taskLoopOptions|`taskLoopOptions.isFinishedFunction`} to check both
239
+ * `(await requestManager.checkReadiness()).status === 'finished'` and the producer's own running/finished flag.
179
240
  */
180
241
  keepAlive?: boolean;
181
242
  /**
182
- * Basic crawler will initialize the {@link SessionPool} with the corresponding {@link SessionPoolOptions|`sessionPoolOptions`}.
183
- * The session instance will be than available in the {@link BasicCrawlerOptions.requestHandler|`requestHandler`}.
184
- */
185
- useSessionPool?: boolean;
186
- /**
187
- * The configuration options for {@link SessionPool} to use.
243
+ * An existing session pool instance to use. When provided, the crawler will use this pool directly instead of
244
+ * creating a new one, enabling session sharing across multiple crawlers. The crawler will not tear down a shared
245
+ * pool — the caller is responsible for its lifecycle.
246
+ *
247
+ * Accepts the built-in {@link SessionPool} or any object implementing the {@link ISessionPool} interface,
248
+ * so custom session-management strategies can be plugged in.
188
249
  */
189
- sessionPoolOptions?: SessionPoolOptions;
250
+ sessionPool?: ISessionPool;
190
251
  /**
191
252
  * Defines the length of the interval for calling the `setStatusMessage` in seconds.
192
253
  */
@@ -208,12 +269,23 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
208
269
  * ```
209
270
  */
210
271
  statusMessageCallback?: StatusMessageCallback;
272
+ /**
273
+ * HTTP status codes that indicate the session should be retired.
274
+ *
275
+ * A 429 from a domain covered by a {@link ThrottlingRequestManager} is handled as a rate limit before
276
+ * this is consulted, so removing 429 here only affects domains that manager does not cover.
277
+ *
278
+ * @default [401, 403, 429]
279
+ */
280
+ blockedStatusCodes?: number[];
211
281
  /**
212
282
  * If set to `true`, the crawler will automatically try to bypass any detected bot protection.
213
283
  *
214
284
  * Currently supports:
215
285
  * - [**Cloudflare** Bot Management](https://www.cloudflare.com/products/bot-management/)
286
+ * - [**AWS WAF** Challenge and CAPTCHA](https://docs.aws.amazon.com/waf/latest/developerguide/waf-captcha-and-challenge.html)
216
287
  * - [**Google Search** Rate Limiting](https://www.google.com/sorry/)
288
+ * - **Imperva Incapsula**
217
289
  */
218
290
  retryOnBlocked?: boolean;
219
291
  /**
@@ -232,31 +304,64 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
232
304
  * 1. based on robots.txt file,
233
305
  * 2. because they don't match enqueueLinks filters,
234
306
  * 3. because they are redirected to a URL that doesn't match the enqueueLinks strategy,
235
- * 4. or because the {@link BasicCrawlerOptions.maxRequestsPerCrawl|`maxRequestsPerCrawl`} limit has been reached
307
+ * 4. because the {@link BasicCrawlerOptions.maxRequestsPerCrawl|`maxRequestsPerCrawl`} limit has been reached,
308
+ * 5. or by {@link RestrictedCrawlingContext.skipRequest|`context.skipRequest()`}.
236
309
  */
237
310
  onSkippedRequest?: SkippedRequestCallback;
238
- /** @internal */
239
- log?: Log;
240
- /**
241
- * Enables experimental features of Crawlee, which can alter the behavior of the crawler.
242
- * WARNING: these options are not guaranteed to be stable and may change or be removed at any time.
243
- */
244
- experiments?: CrawlerExperiments;
245
311
  /**
246
- * Customize the way statistics collecting works, such as logging interval or
247
- * whether to output them to the Key-Value store.
312
+ * A preconfigured statistics instance. When provided, the crawler records into it instead of building its own and
313
+ * will not `reset()` it between `run()` calls. Accepts the built-in {@link Statistics} or any object
314
+ * implementing {@link IStatistics}.
315
+ *
316
+ * Custom fields declared via {@link StatisticsOptions.stateExtension|`stateExtension`} are carried over to
317
+ * {@link BasicCrawler.statistics|`crawler.statistics.state`}:
318
+ *
319
+ * ```ts
320
+ * const statistics = new Statistics({ stateExtension: { defaultState: { productsFound: 0 } } });
321
+ *
322
+ * const crawler = new BasicCrawler({
323
+ * statistics,
324
+ * requestHandler: async () => {
325
+ * statistics.state.productsFound++;
326
+ * },
327
+ * });
328
+ *
329
+ * await crawler.run();
330
+ * // the custom fields are typed on `crawler.statistics` too
331
+ * console.log(crawler.statistics.state.productsFound);
332
+ * ```
248
333
  */
249
- statisticsOptions?: StatisticsOptions;
334
+ statistics?: IStatistics<StatisticStateExtension>;
250
335
  /**
251
336
  * HTTP client implementation for the `sendRequest` context helper and for plain HTTP crawling.
252
- * Defaults to a new instance of {@link GotScrapingHttpClient}
337
+ * Defaults to {@link ImpitHttpClient} when `@crawlee/impit-client` is installed, otherwise {@link FetchHttpClient}.
253
338
  */
254
339
  httpClient?: BaseHttpClient;
255
340
  /**
256
341
  * If set, the crawler will be configured for all connections to use
257
342
  * the Proxy URLs provided and rotated according to the configuration.
258
343
  */
259
- proxyConfiguration?: ProxyConfiguration;
344
+ proxyConfiguration?: IProxyConfiguration;
345
+ /**
346
+ * Custom configuration to use for this crawler.
347
+ * If provided, the crawler will use its own ServiceLocator instance instead of the global one.
348
+ */
349
+ configuration?: Configuration;
350
+ /**
351
+ * Custom storage backend to use for this crawler.
352
+ * If provided, the crawler will use its own ServiceLocator instance instead of the global one.
353
+ */
354
+ storageBackend?: StorageBackend;
355
+ /**
356
+ * Custom event manager to use for this crawler.
357
+ * If provided, the crawler will use its own ServiceLocator instance instead of the global one.
358
+ */
359
+ eventManager?: EventManager;
360
+ /**
361
+ * Custom logger to use for this crawler.
362
+ * If provided, the crawler will use its own ServiceLocator instance instead of the global one.
363
+ */
364
+ logger?: CrawleeLogger;
260
365
  /**
261
366
  * A unique identifier for the crawler instance. This ID is used to isolate the state returned by
262
367
  * {@link BasicCrawler.useState|`crawler.useState()`} from other crawler instances.
@@ -269,243 +374,153 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
269
374
  *
270
375
  */
271
376
  id?: string;
272
- }
273
- /**
274
- * A set of options that you can toggle to enable experimental features in Crawlee.
275
- *
276
- * NOTE: These options will not respect semantic versioning and may be removed or changed at any time. Use at your own risk.
277
- * If you do use these and encounter issues, please report them to us.
278
- */
279
- export interface CrawlerExperiments {
280
- /**
281
- * @deprecated This experiment is now enabled by default, and this flag will be removed in a future release.
282
- * If you encounter issues due to this change, please:
283
- * - report it to us: https://github.com/apify/crawlee
284
- * - set `requestLocking` to `false` in the `experiments` option of the crawler
285
- */
286
- requestLocking?: boolean;
287
- }
288
- /**
289
- * Provides a simple framework for parallel crawling of web pages.
290
- * The URLs to crawl are fed either from a static list of URLs
291
- * or from a dynamic queue of URLs enabling recursive crawling of websites.
292
- *
293
- * `BasicCrawler` is a low-level tool that requires the user to implement the page
294
- * download and data extraction functionality themselves.
295
- * If we want a crawler that already facilitates this functionality,
296
- * we should consider using {@link CheerioCrawler}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler}.
297
- *
298
- * `BasicCrawler` invokes the user-provided {@link BasicCrawlerOptions.requestHandler|`requestHandler`}
299
- * for each {@link Request} object, which represents a single URL to crawl.
300
- * The {@link Request} objects are fed from the {@link RequestList} or {@link RequestQueue}
301
- * instances provided by the {@link BasicCrawlerOptions.requestList|`requestList`} or {@link BasicCrawlerOptions.requestQueue|`requestQueue`}
302
- * constructor options, respectively. If neither `requestList` nor `requestQueue` options are provided,
303
- * the crawler will open the default request queue either when the {@link BasicCrawler.addRequests|`crawler.addRequests()`} function is called,
304
- * or if `requests` parameter (representing the initial requests) of the {@link BasicCrawler.run|`crawler.run()`} function is provided.
305
- *
306
- * If both {@link BasicCrawlerOptions.requestList|`requestList`} and {@link BasicCrawlerOptions.requestQueue|`requestQueue`} options are used,
307
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
308
- * to the {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
309
- *
310
- * The crawler finishes if there are no more {@link Request} objects to crawl.
311
- *
312
- * New requests are only dispatched when there is enough free CPU and memory available,
313
- * using the functionality provided by the {@link AutoscaledPool} class.
314
- * All {@link AutoscaledPool} configuration options can be passed to the {@link BasicCrawlerOptions.autoscaledPoolOptions|`autoscaledPoolOptions`}
315
- * parameter of the `BasicCrawler` constructor.
316
- * For user convenience, the {@link AutoscaledPoolOptions.minConcurrency|`minConcurrency`} and
317
- * {@link AutoscaledPoolOptions.maxConcurrency|`maxConcurrency`} options of the
318
- * underlying {@link AutoscaledPool} constructor are available directly in the `BasicCrawler` constructor.
319
- *
320
- * **Example usage:**
321
- *
322
- * ```javascript
323
- * import { BasicCrawler, Dataset } from 'crawlee';
324
- *
325
- * // Create a crawler instance
326
- * const crawler = new BasicCrawler({
327
- * async requestHandler({ request, sendRequest }) {
328
- * // 'request' contains an instance of the Request class
329
- * // Here we simply fetch the HTML of the page and store it to a dataset
330
- * const { body } = await sendRequest({
331
- * url: request.url,
332
- * method: request.method,
333
- * body: request.payload,
334
- * headers: request.headers,
335
- * });
336
- *
337
- * await Dataset.pushData({
338
- * url: request.url,
339
- * html: body,
340
- * })
341
- * },
342
- * });
343
- *
344
- * // Enqueue the initial requests and run the crawler
345
- * await crawler.run([
346
- * 'http://www.example.com/page-1',
347
- * 'http://www.example.com/page-2',
348
- * ]);
349
- * ```
350
- * @category Crawlers
351
- */
352
- export declare class BasicCrawler<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension> {
353
- readonly config: Configuration;
354
- protected static readonly CRAWLEE_STATE_KEY = "CRAWLEE_STATE";
355
377
  /**
356
- * Tracks crawler instances that accessed shared state without having an explicit id.
357
- * Used to detect and warn about multiple crawlers sharing the same state.
378
+ * Makes the storage writes performed while handling a request atomic with respect to the request
379
+ * succeeding: they are recorded in a {@link StorageTransaction} spanning the whole request
380
+ * lifecycle and only applied when the request handler succeeds, so a thrown handler leaves no partial
381
+ * writes behind and a retry does not double-write. Reads within the handler see its own writes.
382
+ *
383
+ * `false` disables the mechanism entirely; an object overrides the per-storage-type
384
+ * {@link StorageWritePolicy} (e.g. `{ requestQueue: 'deferred' }` for all-or-nothing enqueues).
385
+ * {@link withDirectStorageAccess} is the per-call-site escape hatch; `useState()` is deliberately
386
+ * *not* transactional.
387
+ *
388
+ * @default true
358
389
  */
359
- private static useStateCrawlerIds;
390
+ transactionalStorage?: boolean | Partial<StorageWritePolicy>;
360
391
  /**
361
- * A reference to the underlying {@link Statistics} class that collects and logs run statistics for requests.
392
+ * An array of HTTP response [Status Codes](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status) to be excluded from error consideration.
393
+ * By default, status codes >= 500 trigger errors.
362
394
  */
363
- readonly stats: Statistics;
395
+ ignoreHttpErrorStatusCodes?: number[];
364
396
  /**
365
- * A reference to the underlying {@link RequestList} class that manages the crawler's {@link Request|requests}.
366
- * Only available if used by the crawler.
397
+ * An array of additional HTTP response [Status Codes](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status) to be treated as errors.
398
+ * By default, status codes >= 500 trigger errors.
367
399
  */
368
- requestList?: IRequestList;
400
+ additionalHttpErrorStatusCodes?: number[];
401
+ }
402
+ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> {
403
+ #private;
369
404
  /**
370
- * Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
371
- * A reference to the underlying {@link RequestQueue} class that manages the crawler's {@link Request|requests}.
372
- * Only available if used by the crawler.
405
+ * The statistics instance collecting the crawler's run statistics - either the injected `statistics` option or a
406
+ * crawler-built default. Typed as {@link IStatistics} so custom implementations can be plugged in.
373
407
  */
374
- requestQueue?: RequestProvider;
408
+ get statistics(): IStatistics<StatisticStateExtension>;
375
409
  /**
376
- * The main request-handling component of the crawler. It's initialized during the crawler startup.
410
+ * The main request-handling component of the crawler. It manages the requests that the crawler processes,
411
+ * combining any provided request loader and/or queue. It's initialized during the crawler startup or lazily
412
+ * via {@link BasicCrawler.getRequestManager|`getRequestManager()`}.
377
413
  */
378
- protected requestManager?: IRequestManager;
414
+ protected get requestManager(): IRequestManager | undefined;
379
415
  /**
380
- * A reference to the underlying {@link SessionPool} class that manages the crawler's {@link Session|sessions}.
381
- * Only available if used by the crawler.
416
+ * A reference to the underlying session pool that manages the crawler's {@link Session|sessions}. Typed as
417
+ * {@link ISessionPool} so custom implementations can be plugged in via the `sessionPool` constructor option.
382
418
  */
383
- sessionPool?: SessionPool;
419
+ get sessionPool(): ISessionPool;
420
+ /** Whether a {@link BasicCrawler.run|`run()`} is in progress on this instance. */
421
+ get running(): boolean;
384
422
  /**
385
- * A reference to the underlying {@link AutoscaledPool} class that manages the concurrency of the crawler.
386
- * > *NOTE:* This property is only initialized after calling the {@link BasicCrawler.run|`crawler.run()`} function.
387
- * We can use it to change the concurrency settings on the fly,
388
- * to pause the crawler by calling {@link AutoscaledPool.pause|`autoscaledPool.pause()`}
389
- * or to abort it by calling {@link AutoscaledPool.abort|`autoscaledPool.abort()`}.
423
+ * The concurrency governor the run in progress is booking its requests against — either the
424
+ * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} that was injected, or the default the
425
+ * crawler built for itself. Read it for telemetry: `desiredConcurrency`, `currentConcurrency`, `isRunning`.
426
+ *
427
+ * > *NOTE:* `undefined` outside a run, and a crawler-owned default is rebuilt for every run — so read it
428
+ * during a run rather than caching it across runs.
429
+ *
430
+ * {@link IConcurrencySystem} is deliberately read-only. Tuning concurrency *while a crawl is running* means
431
+ * owning the instance: build a {@link ConcurrencySystem} yourself and inject it, then set
432
+ * `minConcurrency`/`maxConcurrency`/`desiredConcurrency` on your own reference.
390
433
  */
391
- autoscaledPool?: AutoscaledPool;
434
+ get concurrencySystem(): IConcurrencySystem | undefined;
392
435
  /**
393
- * A reference to the underlying {@link ProxyConfiguration} class that manages the crawler's proxies.
436
+ * A reference to the underlying {@link IProxyConfiguration} instance that manages the crawler's proxies.
394
437
  * Only available if used by the crawler.
395
438
  */
396
- proxyConfiguration?: ProxyConfiguration;
439
+ readonly proxyConfiguration?: IProxyConfiguration;
397
440
  /**
398
441
  * Default {@link Router} instance that will be used if we don't specify any {@link BasicCrawlerOptions.requestHandler|`requestHandler`}.
399
442
  * See {@link Router.addHandler|`router.addHandler()`} and {@link Router.addDefaultHandler|`router.addDefaultHandler()`}.
400
443
  */
401
- readonly router: RouterHandler<Context>;
402
- private contextPipelineBuilder;
403
- private _contextPipeline?;
444
+ readonly router: RouterHandler<Context, Routes>;
445
+ /** @internal */
404
446
  get contextPipeline(): ContextPipeline<CrawlingContext, ExtendedContext>;
405
- running: boolean;
406
- hasFinishedBefore: boolean;
407
- readonly log: Log;
408
- protected requestHandler: RequestHandler<ExtendedContext>;
409
- protected errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
410
- protected failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
411
- protected requestHandlerTimeoutMillis: number;
412
- protected internalTimeoutMillis: number;
413
- protected maxRequestRetries: number;
414
- protected maxCrawlDepth?: number;
415
- protected sameDomainDelayMillis: number;
416
- protected domainAccessedTime: Map<string, number>;
417
- protected maxSessionRotations: number;
418
- protected maxRequestsPerCrawl?: number;
419
- protected handledRequestsCount: number;
420
- protected statusMessageLoggingInterval: number;
421
- protected statusMessageCallback?: StatusMessageCallback;
422
- protected sessionPoolOptions: SessionPoolOptions;
423
- protected useSessionPool: boolean;
424
- protected autoscaledPoolOptions: AutoscaledPoolOptions;
425
- protected events: EventManager;
426
- protected httpClient: BaseHttpClient;
427
- protected retryOnBlocked: boolean;
428
- protected respectRobotsTxtFile: boolean | {
429
- userAgent?: string;
430
- };
431
- protected onSkippedRequest?: SkippedRequestCallback;
432
- private _closeEvents?;
433
- private shouldLogMaxProcessedRequestsExceeded;
434
- private shouldLogMaxEnqueuedRequestsExceeded;
435
- private experiments;
436
- private readonly robotsTxtFileCache;
437
- private _experimentWarnings;
438
- private readonly crawlerId;
439
- private readonly hasExplicitId;
447
+ /** Whether a `run()` on this instance has already finished - a repeated one continues where it left off. */
448
+ get hasFinishedBefore(): boolean;
449
+ get log(): CrawleeLogger;
450
+ protected readonly requestHandler: RequestHandler<ExtendedContext>;
451
+ protected readonly internalTimeoutMillis: number;
452
+ protected blockedStatusCodes: Set<number>;
453
+ protected readonly additionalHttpErrorStatusCodes: Set<number>;
454
+ protected readonly httpClient: BaseHttpClient;
455
+ protected readonly retryOnBlocked: boolean;
456
+ /**
457
+ * @internal
458
+ */
440
459
  protected static optionsShape: {
441
- // @ts-ignore optional peer dependency or compatibility with es2022
442
- contextPipelineBuilder: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
443
- // @ts-ignore optional peer dependency or compatibility with es2022
444
- extendContext: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
445
- // @ts-ignore optional peer dependency or compatibility with es2022
446
- requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
447
- // @ts-ignore optional peer dependency or compatibility with es2022
448
- requestQueue: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
449
- // @ts-ignore optional peer dependency or compatibility with es2022
450
- requestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
451
- // @ts-ignore optional peer dependency or compatibility with es2022
452
- requestHandlerTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
453
- // @ts-ignore optional peer dependency or compatibility with es2022
454
- errorHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
455
- // @ts-ignore optional peer dependency or compatibility with es2022
456
- failedRequestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
457
- // @ts-ignore optional peer dependency or compatibility with es2022
458
- maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
459
- // @ts-ignore optional peer dependency or compatibility with es2022
460
- sameDomainDelaySecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
461
- // @ts-ignore optional peer dependency or compatibility with es2022
462
- maxSessionRotations: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
463
- // @ts-ignore optional peer dependency or compatibility with es2022
464
- maxRequestsPerCrawl: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
465
- // @ts-ignore optional peer dependency or compatibility with es2022
466
- maxCrawlDepth: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
467
- // @ts-ignore optional peer dependency or compatibility with es2022
468
- autoscaledPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
469
- // @ts-ignore optional peer dependency or compatibility with es2022
470
- sessionPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
471
- // @ts-ignore optional peer dependency or compatibility with es2022
472
- useSessionPool: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
473
- // @ts-ignore optional peer dependency or compatibility with es2022
474
- proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
475
- // @ts-ignore optional peer dependency or compatibility with es2022
476
- statusMessageLoggingInterval: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
477
- // @ts-ignore optional peer dependency or compatibility with es2022
478
- statusMessageCallback: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
479
- // @ts-ignore optional peer dependency or compatibility with es2022
480
- retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
481
- // @ts-ignore optional peer dependency or compatibility with es2022
482
- respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
483
- // @ts-ignore optional peer dependency or compatibility with es2022
484
- onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
485
- // @ts-ignore optional peer dependency or compatibility with es2022
486
- httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
487
- // @ts-ignore optional peer dependency or compatibility with es2022
488
- minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
489
- // @ts-ignore optional peer dependency or compatibility with es2022
490
- maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
491
- // @ts-ignore optional peer dependency or compatibility with es2022
492
- maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
493
- // @ts-ignore optional peer dependency or compatibility with es2022
494
- keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
495
- // @ts-ignore optional peer dependency or compatibility with es2022
496
- log: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
497
- // @ts-ignore optional peer dependency or compatibility with es2022
498
- experiments: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
499
- // @ts-ignore optional peer dependency or compatibility with es2022
500
- statisticsOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
501
- // @ts-ignore optional peer dependency or compatibility with es2022
502
- id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
460
+ contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
461
+ extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
462
+ requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
463
+ requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
464
+ requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
465
+ requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
466
+ requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
467
+ errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
468
+ failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
469
+ maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
470
+ sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
471
+ maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
472
+ maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
473
+ taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
474
+ concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
475
+ sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
476
+ proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
477
+ statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
478
+ statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
479
+ additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
480
+ ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
481
+ blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
482
+ retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
483
+ respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
484
+ transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
485
+ requestQueue: z.ZodOptional<z.ZodEnum<{
486
+ deferred: "deferred";
487
+ writeThrough: "writeThrough";
488
+ }>>;
489
+ }, z.core.$strict>]>>;
490
+ onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
491
+ httpClient: z.ZodOptional<z.ZodInstanceOf<BaseHttpClient>>;
492
+ configuration: z.ZodOptional<z.ZodInstanceOf<Configuration>>;
493
+ storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
494
+ eventManager: z.ZodOptional<z.ZodInstanceOf<EventManager>>;
495
+ logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
496
+ minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
497
+ maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
498
+ initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
499
+ maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
500
+ keepAlive: z.ZodOptional<z.ZodBoolean>;
501
+ statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
502
+ id: z.ZodOptional<z.ZodString>;
503
503
  };
504
504
  /**
505
505
  * All `BasicCrawler` parameters are passed via an options object.
506
506
  */
507
- constructor(options?: BasicCrawlerOptions<Context, ContextExtension, ExtendedContext> & RequireContextPipeline<CrawlingContext, Context>, // cast because the constructor logic handles missing `contextPipelineBuilder` - the type is just for DX
508
- config?: Configuration);
507
+ constructor(options?: BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> & RequireContextPipeline<CrawlingContext, Context>);
508
+ /**
509
+ * Builds the crawler-owned default {@link ConcurrencySystem} from the resolved
510
+ * `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute` shortcuts. Not called when a
511
+ * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} was injected.
512
+ *
513
+ * Subclasses may override this to tune the default system (e.g. {@link HttpCrawler} raises the starting
514
+ * concurrency and relaxes the event loop signal) while still honouring the user's shortcuts.
515
+ */
516
+ protected createDefaultConcurrencySystem(options: ConcurrencySystemOptions): ConcurrencySystem;
517
+ /**
518
+ * Determines if the given HTTP status code is an error status code given
519
+ * the default behaviour and user-set preferences.
520
+ * @param status
521
+ * @returns `true` if the status code is considered an error, `false` otherwise
522
+ */
523
+ protected isErrorStatusCode(status: number): boolean;
509
524
  /**
510
525
  * Checks if the given error is a proxy error by comparing its message to a list of known proxy error messages.
511
526
  * Used for retrying requests that failed due to proxy errors.
@@ -514,21 +529,31 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
514
529
  */
515
530
  protected isProxyError(error: Error): boolean;
516
531
  /**
532
+ * Sets the status message for the current crawler run.
533
+ *
517
534
  * This method is periodically called by the crawler, every `statusMessageLoggingInterval` seconds.
535
+ *
536
+ * The message is logged and broadcast via the {@link EventType.STATUS_MESSAGE|`statusMessage`}
537
+ * event. Integrations such as the Apify SDK subscribe to that event and forward the message to
538
+ * their status-reporting backend (e.g. the Apify platform).
518
539
  */
519
- setStatusMessage(message: string, options?: SetStatusMessageOptions): Promise<void>;
520
- private getPeriodicLogger;
540
+ setStatusMessage(message: string, options?: SetStatusMessageOptions): void;
521
541
  /**
522
- * Runs the crawler. Returns a promise that resolves once all the requests are processed
523
- * and `autoscaledPool.isFinished` returns `true`.
542
+ * Runs the crawler. Returns a promise that resolves once every request has been processed and the crawler's
543
+ * finished-check ({@link BasicCrawlerOptions.taskLoopOptions|`taskLoopOptions.isFinishedFunction`}, or the
544
+ * default "the request manager is empty") reports that the crawl is over.
524
545
  *
525
546
  * We can use the `requests` parameter to enqueue the initial requests — it is a shortcut for
526
547
  * running {@link BasicCrawler.addRequests|`crawler.addRequests()`} before {@link BasicCrawler.run|`crawler.run()`}.
527
548
  *
549
+ * Calling `run()` again on the same instance keeps crawling the same request manager - requests the previous
550
+ * run handled (a failed one counts as handled) are not processed again. Purge the queue or open a fresh one
551
+ * if that is what you want.
552
+ *
528
553
  * @param [requests] The requests to add.
529
- * @param [options] Options for the request queue.
554
+ * @param [options] Options for adding the initial requests.
530
555
  */
531
- run(requests?: RequestsLike, options?: CrawlerRunOptions): Promise<FinalStatistics>;
556
+ run(requests?: TypedRequestsLike<Routes>, options?: CrawlerRunOptions): Promise<FinalStatistics>;
532
557
  /**
533
558
  * Gracefully stops the current run of the crawler.
534
559
  *
@@ -536,32 +561,57 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
536
561
  *
537
562
  * To stop the crawler immediately, use {@link BasicCrawler.teardown|`crawler.teardown()`} instead.
538
563
  */
539
- stop(message?: string): void;
540
- getRequestQueue(): Promise<RequestProvider>;
564
+ stop(reason?: string): void;
565
+ /**
566
+ * Stops dispatching new requests, letting the in-progress ones finish. Resolves once they have settled, or rejects
567
+ * after `timeoutSecs` if they take too long. Unlike {@link BasicCrawler.stop|`stop()`}, this does not end the
568
+ * run — {@link BasicCrawler.run|`run()`} stays pending until {@link BasicCrawler.resume|`resume()`}.
569
+ *
570
+ * > *NOTE:* The {@link BasicCrawler.concurrencySystem|concurrency system} keeps monitoring and autoscaling
571
+ * throughout, since a shared one may still be serving other crawlers.
572
+ */
573
+ pause(timeoutSecs?: number): Promise<void>;
574
+ /**
575
+ * Resumes a run suspended with {@link BasicCrawler.pause|`pause()`}, letting the crawler dispatch requests
576
+ * again. A no-op on a crawler that is not paused.
577
+ */
578
+ resume(): void;
579
+ /**
580
+ * Returns the crawler's {@link IRequestManager|request manager}, opening the default {@link RequestQueue}
581
+ * if none has been configured or opened yet.
582
+ */
583
+ getRequestManager(): Promise<IRequestManager>;
584
+ /**
585
+ * @deprecated Use {@link BasicCrawler.getRequestManager|`getRequestManager()`} instead. This returns the
586
+ * crawler's request manager, which is no longer guaranteed to be a {@link RequestQueue}.
587
+ */
588
+ getRequestQueue(): Promise<IRequestManager>;
541
589
  useState<State extends Dictionary = Dictionary>(defaultValue?: State): Promise<State>;
542
- protected get pendingRequestCountApproximation(): number;
543
- protected calculateEnqueuedRequestLimit(explicitLimit?: number): number | undefined;
544
- protected handleSkippedRequest(options: Parameters<SkippedRequestCallback>[0]): Promise<void>;
545
590
  /**
546
591
  * Adds requests to the queue in batches. By default, it will resolve after the initial batch is added, and continue
547
592
  * adding the rest in background. You can configure the batch size via `batchSize` option and the sleep time in between
548
593
  * the batches via `waitBetweenBatchesMillis`. If you want to wait for all batches to be added to the queue, you can use
549
594
  * the `waitForAllRequestsToBeAdded` promise you get in the response object.
550
595
  *
596
+ * Optionally, the requests can be filtered using `include`/`exclude` glob or regexp patterns and an
597
+ * enqueue `strategy` (both AND-ed together, same as {@link CrawlingContext.enqueueLinks|`enqueueLinks`}),
598
+ * relative to `baseUrl`. Unlike `enqueueLinks`, there is no implicit "current page" to anchor the strategy
599
+ * to, so `strategy` defaults to {@link EnqueueStrategy.All|`all`} here.
600
+ *
551
601
  * This is an alias for calling `addRequestsBatched()` on the implicit `RequestQueue` for this crawler instance.
552
602
  *
553
603
  * @param requests The requests to add
554
604
  * @param options Options for the request queue
555
605
  */
556
- addRequests(requests: ReadonlyDeep<RequestsLike>, options?: CrawlerAddRequestsOptions): Promise<CrawlerAddRequestsResult>;
606
+ addRequests(requests: ReadonlyDeep<TypedRequestsLike<Routes>>, options?: CrawlerAddRequestsOptions): Promise<CrawlerAddRequestsResult>;
557
607
  /**
558
608
  * Pushes data to the specified {@link Dataset}, or the default crawler {@link Dataset} by calling {@link Dataset.pushData}.
559
609
  */
560
- pushData(data: Parameters<Dataset['pushData']>[0], datasetIdOrName?: string): Promise<void>;
610
+ pushData(data: Parameters<Dataset['pushData']>[0], datasetIdentifier?: string | StorageIdentifier): Promise<void>;
561
611
  /**
562
612
  * Retrieves the specified {@link Dataset}, or the default crawler {@link Dataset}.
563
613
  */
564
- getDataset(idOrName?: string): Promise<Dataset>;
614
+ getDataset(identifier?: string | StorageIdentifier): Promise<Dataset>;
565
615
  /**
566
616
  * Retrieves data from the default crawler {@link Dataset} by calling {@link Dataset.getData}.
567
617
  */
@@ -574,112 +624,76 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
574
624
  /**
575
625
  * Initializes the crawler.
576
626
  */
577
- protected _init(): Promise<void>;
578
- protected runRequestHandler(crawlingContext: CrawlingContext): Promise<void>;
627
+ protected init(): Promise<void>;
579
628
  /**
580
- * Handles blocked request
629
+ * The navigation timeout (pre-navigation hooks, navigation, and post-navigation hooks) in milliseconds, used
630
+ * to size the internal request timeout. `BasicCrawler` has no navigation phase, so this is 0; the HTTP and
631
+ * browser crawlers override it with their `navigationTimeoutSecs`.
581
632
  */
582
- protected _throwOnBlockedRequest(session: Session, statusCode: number): void;
583
- private isAllowedBasedOnRobotsTxtFile;
584
- protected getRobotsTxtFileForUrl(url: string): Promise<RobotsTxtFile | undefined>;
585
- protected _pauseOnMigration(): Promise<void>;
633
+ protected getNavigationTimeoutMillis(): number;
634
+ protected runRequestHandler(crawlingContext: ExtendedContext): Promise<void>;
586
635
  /**
587
- * Initializes the RequestManager based on the configured requestList and requestQueue.
636
+ * Runs `callback` inside a {@link StorageTransaction}, unless transactional storage is disabled.
637
+ * Deliberately does **not** commit on return - the pipeline hands failures to `handleRequestFailure`, so a
638
+ * normal return says nothing about success. `handleRequest` commits on success.
588
639
  */
589
- private initializeRequestManager;
640
+ private runInStorageTransaction;
590
641
  /**
591
- * Fetches the next request to process from the underlying request provider.
592
- */
593
- protected _fetchNextRequest(): Promise<Request<Dictionary> | null>;
594
- /**
595
- * Delays processing of the request based on the `sameDomainDelaySecs` option,
596
- * adding it back to the queue after the timeout passes. Returns `true` if the request
597
- * should be ignored and will be reclaimed to the queue once ready.
598
- */
599
- protected delayRequest(request: Request, source: IRequestList | RequestProvider | IRequestManager): boolean;
600
- /**
601
- * Wrapper around requestHandler that fetches requests from RequestList/RequestQueue
602
- * then retries them in a case of an error, etc.
642
+ * Handles blocked request
603
643
  */
604
- protected _runTaskFunction(): Promise<void>;
644
+ protected throwOnBlockedRequest(statusCode: number): void;
605
645
  /**
606
- * Wrapper around the crawling context's `enqueueLinks` method:
607
- * - Injects `crawlDepth` to each request being added based on the crawling context request.
608
- * - Provides defaults for the `enqueueLinks` options based on the crawler configuration.
609
- * - These options can be overridden by the user.
610
- * @internal
646
+ * Records an HTTP 429 against the URL's domain so the request manager can hold the retry back.
647
+ *
648
+ * @param retryAfterHeader The raw `Retry-After` response header, if the server sent one.
649
+ * @returns `true` if the manager took responsibility for the delay, in which case the caller should throw
650
+ * {@link RequestThrottledError} rather than treating the response as a blocked session.
611
651
  */
612
- protected enqueueLinksWithCrawlDepth(options: SetRequired<EnqueueLinksOptions, 'urls'>, request: Request<Dictionary>, requestQueue: RequestProvider): Promise<BatchAddRequestsResult>;
652
+ protected recordDomainRateLimit(url: string, retryAfterHeader?: string | null): boolean;
653
+ /** @internal */
654
+ protected getRobotsTxtFileForUrl(url: string): Promise<RobotsTxtFile | undefined>;
655
+ /** Runs the request handler and marks the request as handled. Failures are left to `handleRequestFailure`. */
656
+ private handleRequest;
613
657
  /**
614
658
  * Generator function that yields requests injected with the given crawl depth.
615
659
  * @internal
616
660
  */
617
661
  protected addCrawlDepthRequestGenerator(requests: RequestsLike, newRequestDepth: number): AsyncGenerator<Source, void, undefined>;
618
- /**
619
- * Run async callback with given timeout and retry. Returns the result of the callback.
620
- * @ignore
621
- */
622
- protected _timeoutAndRetry<T>(handler: () => Promise<T>, timeout: number, error: Error | string, maxRetries?: number, retried?: number): Promise<T>;
623
- /**
624
- * Returns true if either RequestList or RequestQueue have a request ready for processing.
625
- */
626
- protected _isTaskReadyFunction(): Promise<boolean>;
627
- /**
628
- * Returns true if both RequestList and RequestQueue have all requests finished.
629
- */
630
- protected _defaultIsFinishedFunction(): Promise<boolean>;
631
- private _rotateSession;
632
- /**
633
- * Unwraps errors thrown by the context pipeline to get the actual user error.
634
- * RequestHandlerError and ContextPipelineInitializationError wrap the actual error.
635
- */
636
- private unwrapError;
637
662
  /**
638
663
  * Handles errors thrown by user provided requestHandler()
639
664
  */
640
- protected _requestFunctionErrorHandler(error: Error, crawlingContext: CrawlingContext, source: IRequestList | IRequestManager): Promise<void>;
641
- protected _tagUserHandlerError<T>(cb: () => unknown): Promise<T>;
642
- protected _handleFailedRequestHandler(crawlingContext: CrawlingContext, error: Error): Promise<void>;
665
+ private requestFunctionErrorHandler;
666
+ private handleFailedRequestHandler;
643
667
  /**
644
668
  * Resolves the most verbose error message from a thrown error
645
669
  * @param error The error received
646
670
  * @returns The message to be logged
647
671
  */
648
- protected _getMessageFromError(error: Error, forceStack?: boolean): string | TimeoutError | undefined;
649
- protected _canRequestBeRetried(request: Request, error: Error): boolean;
672
+ protected getMessageFromError(error: Error, forceStack?: boolean): string;
650
673
  /**
651
- * Updates handledRequestsCount from possibly stored counts, usually after worker migration.
652
- */
653
- protected _loadHandledRequestCount(): Promise<void>;
654
- protected _executeHooks<HookLike extends (...args: any[]) => Awaitable<void>>(hooks: HookLike[], ...args: Parameters<HookLike>): Promise<void>;
655
- /**
656
- * Stops the crawler immediately.
674
+ * Ends the run in progress immediately, without waiting for the requests already in flight.
657
675
  *
658
- * This method doesn't wait for currently active requests to finish.
676
+ * This runs at the end of every {@link BasicCrawler.run|`run()`}, so what it releases is per-run only, and
677
+ * a repeated `run()` has to find everything it needs either intact or re-establishable. What outlives a run is
678
+ * released by {@link BasicCrawler.destroy|`destroy()`} instead.
659
679
  *
660
680
  * To stop the crawler gracefully (waiting for all running requests to finish), use {@link BasicCrawler.stop|`crawler.stop()`} instead.
661
681
  */
662
682
  teardown(): Promise<void>;
663
- protected _getCookieHeaderFromRequest(request: Request): string;
664
- private _getRequestQueue;
665
- private requestMatchesEnqueueStrategy;
666
- }
667
- export interface CreateContextOptions {
668
- request: Request;
669
- session?: Session;
670
- proxyInfo?: ProxyInfo;
683
+ /**
684
+ * Releases what the crawler owns beyond a single run — in the browser crawlers, the browser pool. A finished
685
+ * `run()` has already released everything a run owns, so this is only needed before dropping a crawler you
686
+ * could otherwise have run again.
687
+ */
688
+ destroy(): Promise<void>;
689
+ [Symbol.asyncDispose](): Promise<void>;
690
+ protected getCookieHeaderFromRequest(request: Request): string;
671
691
  }
672
- export interface CrawlerAddRequestsOptions extends AddRequestsBatchedOptions {
692
+ export interface CrawlerAddRequestsOptions extends AddRequestsBatchedOptions, EnqueueUrlsOptions {
673
693
  }
674
694
  export interface CrawlerAddRequestsResult extends AddRequestsBatchedResult {
675
695
  }
676
696
  export interface CrawlerRunOptions extends CrawlerAddRequestsOptions {
677
- /**
678
- * Whether to purge the RequestQueue before running the crawler again. Defaults to true, so it is possible to reprocess failed requests.
679
- * When disabled, only new requests will be considered. Note that even a failed request is considered as handled.
680
- * @default true
681
- */
682
- purgeRequestQueue?: boolean;
683
697
  }
684
698
  /**
685
699
  * Creates new {@link Router} instance that works based on request labels.
@@ -705,5 +719,5 @@ export interface CrawlerRunOptions extends CrawlerAddRequestsOptions {
705
719
  * await crawler.run();
706
720
  * ```
707
721
  */
708
- export declare function createBasicRouter<Context extends BasicCrawlingContext = BasicCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): RouterHandler<Context>;
709
- //# sourceMappingURL=basic-crawler.d.ts.map
722
+ export declare function createBasicRouter<Context extends BasicCrawlingContext = BasicCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
723
+ export declare function createBasicRouter<Context extends BasicCrawlingContext = BasicCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;