@crawlee/basic 4.0.0-beta.28 → 4.0.0-beta.281

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/README.md +15 -15
  2. package/index.d.ts +11 -2
  3. package/index.js +11 -1
  4. package/internals/autoscaling/autoscaled-pool.d.ts +178 -0
  5. package/internals/autoscaling/autoscaled-pool.js +389 -0
  6. package/internals/autoscaling/concurrency-system.d.ts +235 -0
  7. package/internals/autoscaling/concurrency-system.js +357 -0
  8. package/internals/autoscaling/cpu-load-signal.d.ts +43 -0
  9. package/internals/autoscaling/cpu-load-signal.js +46 -0
  10. package/internals/autoscaling/event-loop-load-signal.d.ts +51 -0
  11. package/internals/autoscaling/event-loop-load-signal.js +60 -0
  12. package/internals/autoscaling/index.d.ts +9 -0
  13. package/internals/autoscaling/index.js +9 -0
  14. package/internals/autoscaling/load-signal.d.ts +98 -0
  15. package/internals/autoscaling/load-signal.js +103 -0
  16. package/internals/autoscaling/memory-load-signal.d.ts +45 -0
  17. package/internals/autoscaling/memory-load-signal.js +104 -0
  18. package/internals/autoscaling/snapshotter.d.ts +84 -0
  19. package/internals/autoscaling/snapshotter.js +67 -0
  20. package/internals/autoscaling/storage-backend-load-signal.d.ts +56 -0
  21. package/internals/autoscaling/storage-backend-load-signal.js +73 -0
  22. package/internals/autoscaling/system-status.d.ts +112 -0
  23. package/internals/autoscaling/system-status.js +139 -0
  24. package/internals/autoscaling/weighted-avg.d.ts +5 -0
  25. package/internals/autoscaling/weighted-avg.js +14 -0
  26. package/internals/basic-crawler.d.ts +370 -374
  27. package/internals/basic-crawler.js +1416 -857
  28. package/internals/cookie-utils.d.ts +40 -0
  29. package/internals/cookie-utils.js +122 -0
  30. package/internals/crawler-run.d.ts +76 -0
  31. package/internals/crawler-run.js +144 -0
  32. package/internals/crawlers/context-pipeline.d.ts +75 -0
  33. package/internals/crawlers/context-pipeline.js +109 -0
  34. package/internals/crawlers/crawler-commons.d.ts +216 -0
  35. package/internals/crawlers/crawler-commons.js +1 -0
  36. package/internals/crawlers/error-snapshotter.d.ts +29 -0
  37. package/internals/crawlers/error-snapshotter.js +116 -0
  38. package/internals/crawlers/error-tracker.d.ts +51 -0
  39. package/internals/crawlers/error-tracker.js +316 -0
  40. package/internals/crawlers/index.d.ts +4 -0
  41. package/internals/crawlers/index.js +3 -0
  42. package/internals/crawlers/internals/types.d.ts +7 -0
  43. package/internals/crawlers/internals/types.js +1 -0
  44. package/internals/crawlers/statistics.d.ts +326 -0
  45. package/internals/crawlers/statistics.js +559 -0
  46. package/internals/crawling-request.d.ts +86 -0
  47. package/internals/crawling-request.js +127 -0
  48. package/internals/enqueue-links/enqueue-links.d.ts +147 -0
  49. package/internals/enqueue-links/enqueue-links.js +35 -0
  50. package/internals/enqueue-links/index.d.ts +3 -0
  51. package/internals/enqueue-links/index.js +3 -0
  52. package/internals/enqueue-links/shared.d.ts +107 -0
  53. package/internals/enqueue-links/shared.js +228 -0
  54. package/internals/errors.d.ts +73 -0
  55. package/internals/errors.js +108 -0
  56. package/internals/http.d.ts +9 -0
  57. package/internals/http.js +28 -0
  58. package/internals/request-timeout.d.ts +54 -0
  59. package/internals/request-timeout.js +98 -0
  60. package/internals/router.d.ts +299 -0
  61. package/internals/router.js +312 -0
  62. package/internals/send-request.d.ts +4 -4
  63. package/internals/send-request.js +18 -8
  64. package/internals/session-pool/consts.d.ts +1 -0
  65. package/internals/session-pool/consts.js +1 -0
  66. package/internals/session-pool/errors.d.ts +7 -0
  67. package/internals/session-pool/errors.js +11 -0
  68. package/internals/session-pool/fingerprint.d.ts +9 -0
  69. package/internals/session-pool/fingerprint.js +30 -0
  70. package/internals/session-pool/index.d.ts +4 -0
  71. package/internals/session-pool/index.js +4 -0
  72. package/internals/session-pool/session-pool.d.ts +193 -0
  73. package/internals/session-pool/session-pool.js +453 -0
  74. package/internals/session-pool/session.d.ts +151 -0
  75. package/internals/session-pool/session.js +225 -0
  76. package/internals/sitemap-request-loader.d.ts +165 -0
  77. package/internals/sitemap-request-loader.js +492 -0
  78. package/internals/throttling-request-manager.d.ts +229 -0
  79. package/internals/throttling-request-manager.js +860 -0
  80. package/package.json +23 -17
  81. package/index.d.ts.map +0 -1
  82. package/index.js.map +0 -1
  83. package/internals/basic-crawler.d.ts.map +0 -1
  84. package/internals/basic-crawler.js.map +0 -1
  85. package/internals/send-request.d.ts.map +0 -1
  86. package/internals/send-request.js.map +0 -1
@@ -1,25 +1,44 @@
1
- import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, AutoscaledPoolOptions, Configuration, CrawlingContext, DatasetExportOptions, EnqueueLinksOptions, EventManager, FinalStatistics, GetUserDataFromRequest, IRequestList, IRequestManager, ProxyConfiguration, Request, RequestsLike, RouterHandler, RouterRoutes, Session, SessionPoolOptions, SkippedRequestCallback, Source, StatisticsOptions, StatisticState } from '@crawlee/core';
2
- import { AutoscaledPool, ContextPipeline, Dataset, RequestProvider, SessionPool, Statistics } from '@crawlee/core';
3
- import type { Awaitable, BaseHttpClient, BatchAddRequestsResult, Dictionary, ProxyInfo, SetStatusMessageOptions, StorageClient } from '@crawlee/types';
1
+ import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, CrawleeLogger, DatasetExportOptions, IProxyConfiguration, IRequestLoader, IRequestManager, RequestsLike, Source, StorageIdentifier, StorageWritePolicy } from '@crawlee/core';
2
+ import { Configuration, Request, Dataset, EventManager, RequestQueue } from '@crawlee/core';
3
+ import { BaseHttpClient } from '@crawlee/http-client';
4
+ import type { Awaitable, Dictionary, ISessionPool, SetStatusMessageOptions, StorageBackend } from '@crawlee/types';
4
5
  import { RobotsTxtFile } from '@crawlee/utils';
5
- import type { ReadonlyDeep, SetRequired } from 'type-fest';
6
- import type { Log } from '@apify/log';
7
- import { TimeoutError } from '@apify/timeout';
6
+ import type { ReadonlyDeep } from 'type-fest';
7
+ import { z } from 'zod';
8
+ import type { TaskLoopPredicates } from './autoscaling/autoscaled-pool.js';
9
+ import type { ConcurrencySystemOptions, IConcurrencySystem } from './autoscaling/concurrency-system.js';
10
+ import { ConcurrencySystem } from './autoscaling/concurrency-system.js';
11
+ import type { FinalStatistics } from './autoscaling/system-status.js';
12
+ import type { ContextMiddleware } from './crawlers/context-pipeline.js';
13
+ import { ContextPipeline } from './crawlers/context-pipeline.js';
14
+ import type { CrawlingContext, TypedRequestsLike } from './crawlers/crawler-commons.js';
15
+ import type { IStatistics, StatisticState } from './crawlers/statistics.js';
16
+ import type { EnqueueUrlsOptions, SkippedRequestCallback } from './enqueue-links/index.js';
17
+ import type { GetUserDataFromRequest, RouterHandler, RouterRoutes } from './router.js';
8
18
  export interface BasicCrawlingContext<UserData extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
9
19
  }
20
+ export { navigationDeadlineKey, remainingNavigationWindowMillis } from './request-timeout.js';
10
21
  export type RequestHandler<Context extends CrawlingContext = CrawlingContext> = (inputs: Context) => Awaitable<void>;
11
- export type ErrorHandler<Context extends CrawlingContext = CrawlingContext, ExtendedContext extends Context = Context> = (inputs: Context & Partial<ExtendedContext>, error: Error) => Awaitable<void>;
12
- export interface StatusMessageCallbackParams<Context extends CrawlingContext = BasicCrawlingContext, Crawler extends BasicCrawler<any> = BasicCrawler<Context>> {
22
+ /**
23
+ * An error handler receives the crawling context and the error that was thrown while processing the request.
24
+ *
25
+ * Unlike the {@link RequestHandler}, an error handler may run before the context pipeline has finished
26
+ * building the full context (e.g. when navigation or session setup fails). Therefore only `BaseContext` is
27
+ * guaranteed to be present, while the extra properties added by the pipeline and `extendContext` (the
28
+ * difference between `BaseContext` and `ExtendedContext`) are only available as a `Partial`.
29
+ */
30
+ export type ErrorHandler<BaseContext extends CrawlingContext = CrawlingContext, ExtendedContext extends BaseContext = BaseContext> = (inputs: BaseContext & Partial<ExtendedContext>, error: Error) => Awaitable<void>;
31
+ export interface StatusMessageCallbackParams<Context extends CrawlingContext = BasicCrawlingContext, Crawler extends BasicCrawler<any, any, any, any> = BasicCrawler<Context>> {
13
32
  state: StatisticState;
14
33
  crawler: Crawler;
15
34
  previousState: StatisticState;
16
35
  message: string;
17
36
  }
18
- export type StatusMessageCallback<Context extends CrawlingContext = BasicCrawlingContext, Crawler extends BasicCrawler<any> = BasicCrawler<Context>> = (params: StatusMessageCallbackParams<Context, Crawler>) => Awaitable<void>;
37
+ export type StatusMessageCallback<Context extends CrawlingContext = BasicCrawlingContext, Crawler extends BasicCrawler<any, any, any, any> = BasicCrawler<Context>> = (params: StatusMessageCallbackParams<Context, Crawler>) => Awaitable<void>;
19
38
  export type RequireContextPipeline<DefaultContextType extends CrawlingContext, FinalContextType extends DefaultContextType> = DefaultContextType extends FinalContextType ? {} : {
20
39
  contextPipelineBuilder: () => ContextPipeline<CrawlingContext, FinalContextType>;
21
40
  };
22
- export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension> {
41
+ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> {
23
42
  /**
24
43
  * User-provided function that performs the logic of the crawler. It is called for each URL to crawl.
25
44
  *
@@ -35,11 +54,21 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
35
54
  * To make this work, we should **always**
36
55
  * let our function throw exceptions rather than catch them.
37
56
  * The exceptions are logged to the request using the
38
- * {@link Request.pushErrorMessage|`Request.pushErrorMessage()`} function.
57
+ * {@link CrawlingRequest.pushErrorMessage|`request.pushErrorMessage()`} function.
39
58
  */
40
- requestHandler?: RequestHandler<ExtendedContext>;
59
+ requestHandler?: RouterHandler<ExtendedContext, Routes> | RequestHandler<ExtendedContext>;
41
60
  /**
42
- * Allows the user to extend the crawling context passed to the request handler with custom functionality.
61
+ * Allows the user to extend the crawling context with custom functionality (helpers, references, etc.).
62
+ *
63
+ * `extendContext` runs *before* navigation, so the returned members are visible to the
64
+ * `preNavigationHooks`, `postNavigationHooks`, and the `requestHandler` alike. As a consequence,
65
+ * the `context` passed to `extendContext` is the pre-navigation {@link CrawlingContext} and does
66
+ * **not** include navigation-dependent members (e.g. `page`, `response`, `$`, `body`). If you need
67
+ * those, use a `postNavigationHook` or the `requestHandler` instead.
68
+ *
69
+ * The second argument is the same {@link CleanupRegistrar|`onCleanup`} registrar used by
70
+ * {@link ContextPipeline.compose}. Use it to register teardown callbacks that run after the
71
+ * request handler or a pipeline failure, in reverse registration order.
43
72
  *
44
73
  * **Example usage:**
45
74
  *
@@ -59,7 +88,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
59
88
  * });
60
89
  * ```
61
90
  */
62
- extendContext?: (context: Context) => Awaitable<ContextExtension>;
91
+ extendContext?: ContextMiddleware<CrawlingContext, ContextExtension>;
63
92
  /**
64
93
  * *Intended for BasicCrawler subclasses*. Prepares a context pipeline that transforms the initial crawling context into the shape given by the `Context` type parameter.
65
94
  *
@@ -68,23 +97,25 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
68
97
  contextPipelineBuilder?: () => ContextPipeline<CrawlingContext, Context>;
69
98
  /**
70
99
  * Static list of URLs to be processed.
71
- * If not provided, the crawler will open the default request queue when the {@link BasicCrawler.addRequests|`crawler.addRequests()`} function is called.
72
- * > Alternatively, `requests` parameter of {@link BasicCrawler.run|`crawler.run()`} could be used to enqueue the initial requests -
73
- * it is a shortcut for running `crawler.addRequests()` before the `crawler.run()`.
100
+ *
101
+ * @deprecated Use the `requestManager` option instead. To combine a read-only loader (such as a `RequestList`)
102
+ * with a writable queue, build a tandem with {@link IRequestLoader.toTandem|`requestList.toTandem(requestQueue)`}
103
+ * and pass the result as `requestManager`. When both `requestList` and `requestQueue` are provided, they are
104
+ * combined into a tandem automatically.
74
105
  */
75
- requestList?: IRequestList;
106
+ requestList?: IRequestLoader;
76
107
  /**
77
108
  * Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
78
- * If not provided, the crawler will open the default request queue when the {@link BasicCrawler.addRequests|`crawler.addRequests()`} function is called.
79
- * > Alternatively, `requests` parameter of {@link BasicCrawler.run|`crawler.run()`} could be used to enqueue the initial requests -
80
- * it is a shortcut for running `crawler.addRequests()` before the `crawler.run()`.
109
+ *
110
+ * @deprecated Use the `requestManager` option instead. A `RequestQueue` is itself a request manager, so you can
111
+ * pass it directly as `requestManager`.
81
112
  */
82
- requestQueue?: RequestProvider;
113
+ requestQueue?: RequestQueue;
83
114
  /**
84
- * Allows explicitly configuring a request manager. Mutually exclusive with the `requestQueue` and `requestList` options.
115
+ * Manager of requests that should be processed by the crawler. Mutually exclusive with the deprecated
116
+ * `requestQueue` and `requestList` options.
85
117
  *
86
- * This enables explicitly configuring the crawler to use `RequestManagerTandem`, for instance.
87
- * If using this, the type of `BasicCrawler.requestQueue` may not be fully compatible with the `RequestProvider` class.
118
+ * If not provided, the crawler will open the default {@link RequestQueue} when it is first needed.
88
119
  */
89
120
  requestManager?: IRequestManager;
90
121
  /**
@@ -113,27 +144,21 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
113
144
  failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
114
145
  /**
115
146
  * Specifies the maximum number of retries allowed for a request if its processing fails.
116
- * This includes retries due to navigation errors or errors thrown from user-supplied functions
117
- * (`requestHandler`, `preNavigationHooks`, `postNavigationHooks`).
118
- *
119
- * This limit does not apply to retries triggered by session rotation
120
- * (see {@link BasicCrawlerOptions.maxSessionRotations|`maxSessionRotations`}).
147
+ * This includes retries due to navigation errors, session/proxy errors, or errors thrown from user-supplied
148
+ * functions (`requestHandler`, `preNavigationHooks`, `postNavigationHooks`).
121
149
  * @default 3
122
150
  */
123
151
  maxRequestRetries?: number;
124
152
  /**
125
- * Indicates how much time (in seconds) to wait before crawling another same domain request.
153
+ * Indicates how much time (in seconds) to wait before crawling another same domain request. Subdomains are
154
+ * paced together with the site they belong to.
155
+ *
156
+ * Offered to the crawler's request manager as a `minIntervalEverywhere` {@link PacingSignal}; a manager that
157
+ * already paces every domain it dispatches to takes it, so no domain ends up with two clocks. Otherwise the
158
+ * crawler wraps its request manager in a {@link ThrottlingRequestManager} of its own.
126
159
  * @default 0
127
160
  */
128
161
  sameDomainDelaySecs?: number;
129
- /**
130
- * Maximum number of session rotations per request.
131
- * The crawler will automatically rotate the session in case of a proxy error or if it gets blocked by the website.
132
- *
133
- * The session rotations are not counted towards the {@link BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} limit.
134
- * @default 10
135
- */
136
- maxSessionRotations?: number;
137
162
  /**
138
163
  * Maximum number of pages that the crawler will open. The crawl will stop when this limit is reached.
139
164
  * This value should always be set in order to prevent infinite loops in misconfigured crawlers.
@@ -147,46 +172,82 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
147
172
  */
148
173
  maxCrawlDepth?: number;
149
174
  /**
150
- * Custom options passed to the underlying {@link AutoscaledPool} constructor.
151
- * > *NOTE:* The {@link AutoscaledPoolOptions.runTaskFunction|`runTaskFunction`}
152
- * option is provided by the crawler and cannot be overridden.
153
- * However, we can provide custom implementations of {@link AutoscaledPoolOptions.isFinishedFunction|`isFinishedFunction`}
154
- * and {@link AutoscaledPoolOptions.isTaskReadyFunction|`isTaskReadyFunction`}.
175
+ * Lets you override the predicates that steer the crawler's task loop: `isTaskReadyFunction` (may another request
176
+ * start?) and `isFinishedFunction` (is the crawl over?). The task itself — fetching a request and running it
177
+ * through the pipeline — is owned by the crawler and cannot be overridden.
178
+ *
179
+ * Concurrency is configured elsewhere — through the `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute`
180
+ * shortcuts, or a {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} for finer control.
181
+ */
182
+ taskLoopOptions?: TaskLoopPredicates;
183
+ /**
184
+ * A pre-configured concurrency governor — the component that decides whether there is free compute for one more
185
+ * task. Typically a {@link ConcurrencySystem}, though any {@link IConcurrencySystem} is accepted. All
186
+ * scaling configuration (min/max/desired concurrency, scaling ratios, `maxTasksPerMinute`, snapshotter tuning)
187
+ * lives on the instance itself.
188
+ *
189
+ * Inject the *same* instance into several concurrent crawlers to cap their **combined** concurrency against a
190
+ * single budget. Each crawler still builds and drives its own {@link AutoscaledPool}; only the load/scaling
191
+ * accounting is shared.
192
+ *
193
+ * Mutually exclusive with the `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute`
194
+ * shortcuts, which configure the default system this one replaces — combining the two throws.
195
+ *
196
+ * You own a supplied system's lifecycle: `start()` it before `run()` (which throws otherwise) and `stop()` it once
197
+ * every crawler borrowing it has finished. The crawler does neither on your behalf.
155
198
  */
156
- autoscaledPoolOptions?: AutoscaledPoolOptions;
199
+ concurrencySystem?: IConcurrencySystem;
157
200
  /**
158
201
  * Sets the minimum concurrency (parallelism) for the crawl. Shortcut for the
159
- * AutoscaledPool {@link AutoscaledPoolOptions.minConcurrency|`minConcurrency`} option.
202
+ * {@link ConcurrencySystemOptions.minConcurrency|`minConcurrency`} option of the crawler's default
203
+ * {@link ConcurrencySystem}.
160
204
  * > *WARNING:* If we set this value too high with respect to the available system memory and CPU, our crawler will run extremely slow or crash.
161
205
  * If not sure, it's better to keep the default value and the concurrency will scale up automatically.
162
206
  */
163
207
  minConcurrency?: number;
164
208
  /**
165
209
  * Sets the maximum concurrency (parallelism) for the crawl. Shortcut for the
166
- * AutoscaledPool {@link AutoscaledPoolOptions.maxConcurrency|`maxConcurrency`} option.
210
+ * {@link ConcurrencySystemOptions.maxConcurrency|`maxConcurrency`} option of the crawler's default
211
+ * {@link ConcurrencySystem}.
167
212
  */
168
213
  maxConcurrency?: number;
214
+ /**
215
+ * Sets the concurrency (parallelism) the crawl starts with, before any scaling happens. Shortcut for the
216
+ * {@link ConcurrencySystemOptions.desiredConcurrency|`desiredConcurrency`} option of the crawler's default
217
+ * {@link ConcurrencySystem}. Defaults to `minConcurrency`.
218
+ */
219
+ initialConcurrency?: number;
169
220
  /**
170
221
  * The maximum number of requests per minute the crawler should run.
171
222
  * By default, this is set to `Infinity`, but we can pass any positive, non-zero integer.
172
- * Shortcut for the AutoscaledPool {@link AutoscaledPoolOptions.maxTasksPerMinute|`maxTasksPerMinute`} option.
223
+ * Shortcut for the {@link ConcurrencySystemOptions.maxTasksPerMinute|`maxTasksPerMinute`} option of the
224
+ * crawler's default {@link ConcurrencySystem}.
173
225
  */
174
226
  maxRequestsPerMinute?: number;
175
227
  /**
176
228
  * Allows to keep the crawler alive even if the {@link RequestQueue} gets empty.
177
229
  * By default, the `crawler.run()` will resolve once the queue is empty. With `keepAlive: true` it will keep running,
178
- * waiting for more requests to come. Use `crawler.stop()` to exit the crawler gracefully, or `crawler.teardown()` to stop it immediately.
230
+ * waiting for more requests to come.
231
+ *
232
+ * `crawler.stop()` only stops the crawler from dispatching new requests — it does not wait for or drain
233
+ * requests already sitting in the queue. If this crawler is fed by another crawler or an external producer,
234
+ * calling `stop()` as soon as the producer finishes can abandon requests the producer already enqueued but
235
+ * this crawler had not yet picked up.
236
+ *
237
+ * To shut down only once the queue is genuinely empty and the producer has stopped, override
238
+ * {@link BasicCrawlerOptions.taskLoopOptions|`taskLoopOptions.isFinishedFunction`} to check both
239
+ * `(await requestManager.checkReadiness()).status === 'finished'` and the producer's own running/finished flag.
179
240
  */
180
241
  keepAlive?: boolean;
181
242
  /**
182
- * Basic crawler will initialize the {@link SessionPool} with the corresponding {@link SessionPoolOptions|`sessionPoolOptions`}.
183
- * The session instance will be than available in the {@link BasicCrawlerOptions.requestHandler|`requestHandler`}.
184
- */
185
- useSessionPool?: boolean;
186
- /**
187
- * The configuration options for {@link SessionPool} to use.
243
+ * An existing session pool instance to use. When provided, the crawler will use this pool directly instead of
244
+ * creating a new one, enabling session sharing across multiple crawlers. The crawler will not tear down a shared
245
+ * pool — the caller is responsible for its lifecycle.
246
+ *
247
+ * Accepts the built-in {@link SessionPool} or any object implementing the {@link ISessionPool} interface,
248
+ * so custom session-management strategies can be plugged in.
188
249
  */
189
- sessionPoolOptions?: SessionPoolOptions;
250
+ sessionPool?: ISessionPool;
190
251
  /**
191
252
  * Defines the length of the interval for calling the `setStatusMessage` in seconds.
192
253
  */
@@ -208,12 +269,23 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
208
269
  * ```
209
270
  */
210
271
  statusMessageCallback?: StatusMessageCallback;
272
+ /**
273
+ * HTTP status codes that indicate the session should be retired.
274
+ *
275
+ * A 429 from a domain covered by a {@link ThrottlingRequestManager} is handled as a rate limit before
276
+ * this is consulted, so removing 429 here only affects domains that manager does not cover.
277
+ *
278
+ * @default [401, 403, 429]
279
+ */
280
+ blockedStatusCodes?: number[];
211
281
  /**
212
282
  * If set to `true`, the crawler will automatically try to bypass any detected bot protection.
213
283
  *
214
284
  * Currently supports:
215
285
  * - [**Cloudflare** Bot Management](https://www.cloudflare.com/products/bot-management/)
286
+ * - [**AWS WAF** Challenge and CAPTCHA](https://docs.aws.amazon.com/waf/latest/developerguide/waf-captcha-and-challenge.html)
216
287
  * - [**Google Search** Rate Limiting](https://www.google.com/sorry/)
288
+ * - **Imperva Incapsula**
217
289
  */
218
290
  retryOnBlocked?: boolean;
219
291
  /**
@@ -232,46 +304,64 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
232
304
  * 1. based on robots.txt file,
233
305
  * 2. because they don't match enqueueLinks filters,
234
306
  * 3. because they are redirected to a URL that doesn't match the enqueueLinks strategy,
235
- * 4. or because the {@link BasicCrawlerOptions.maxRequestsPerCrawl|`maxRequestsPerCrawl`} limit has been reached
307
+ * 4. because the {@link BasicCrawlerOptions.maxRequestsPerCrawl|`maxRequestsPerCrawl`} limit has been reached,
308
+ * 5. or by {@link RestrictedCrawlingContext.skipRequest|`context.skipRequest()`}.
236
309
  */
237
310
  onSkippedRequest?: SkippedRequestCallback;
238
- /** @internal */
239
- log?: Log;
240
- /**
241
- * Enables experimental features of Crawlee, which can alter the behavior of the crawler.
242
- * WARNING: these options are not guaranteed to be stable and may change or be removed at any time.
243
- */
244
- experiments?: CrawlerExperiments;
245
311
  /**
246
- * Customize the way statistics collecting works, such as logging interval or
247
- * whether to output them to the Key-Value store.
312
+ * A preconfigured statistics instance. When provided, the crawler records into it instead of building its own and
313
+ * will not `reset()` it between `run()` calls. Accepts the built-in {@link Statistics} or any object
314
+ * implementing {@link IStatistics}.
315
+ *
316
+ * Custom fields declared via {@link StatisticsOptions.stateExtension|`stateExtension`} are carried over to
317
+ * {@link BasicCrawler.statistics|`crawler.statistics.state`}:
318
+ *
319
+ * ```ts
320
+ * const statistics = new Statistics({ stateExtension: { defaultState: { productsFound: 0 } } });
321
+ *
322
+ * const crawler = new BasicCrawler({
323
+ * statistics,
324
+ * requestHandler: async () => {
325
+ * statistics.state.productsFound++;
326
+ * },
327
+ * });
328
+ *
329
+ * await crawler.run();
330
+ * // the custom fields are typed on `crawler.statistics` too
331
+ * console.log(crawler.statistics.state.productsFound);
332
+ * ```
248
333
  */
249
- statisticsOptions?: StatisticsOptions;
334
+ statistics?: IStatistics<StatisticStateExtension>;
250
335
  /**
251
336
  * HTTP client implementation for the `sendRequest` context helper and for plain HTTP crawling.
252
- * Defaults to a new instance of {@link GotScrapingHttpClient}
337
+ * Defaults to {@link ImpitHttpClient} when `@crawlee/impit-client` is installed, otherwise {@link FetchHttpClient}.
253
338
  */
254
339
  httpClient?: BaseHttpClient;
255
340
  /**
256
341
  * If set, the crawler will be configured for all connections to use
257
342
  * the Proxy URLs provided and rotated according to the configuration.
258
343
  */
259
- proxyConfiguration?: ProxyConfiguration;
344
+ proxyConfiguration?: IProxyConfiguration;
260
345
  /**
261
346
  * Custom configuration to use for this crawler.
262
347
  * If provided, the crawler will use its own ServiceLocator instance instead of the global one.
263
348
  */
264
349
  configuration?: Configuration;
265
350
  /**
266
- * Custom storage client to use for this crawler.
351
+ * Custom storage backend to use for this crawler.
267
352
  * If provided, the crawler will use its own ServiceLocator instance instead of the global one.
268
353
  */
269
- storageClient?: StorageClient;
354
+ storageBackend?: StorageBackend;
270
355
  /**
271
356
  * Custom event manager to use for this crawler.
272
357
  * If provided, the crawler will use its own ServiceLocator instance instead of the global one.
273
358
  */
274
359
  eventManager?: EventManager;
360
+ /**
361
+ * Custom logger to use for this crawler.
362
+ * If provided, the crawler will use its own ServiceLocator instance instead of the global one.
363
+ */
364
+ logger?: CrawleeLogger;
275
365
  /**
276
366
  * A unique identifier for the crawler instance. This ID is used to isolate the state returned by
277
367
  * {@link BasicCrawler.useState|`crawler.useState()`} from other crawler instances.
@@ -284,246 +374,153 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
284
374
  *
285
375
  */
286
376
  id?: string;
287
- }
288
- /**
289
- * A set of options that you can toggle to enable experimental features in Crawlee.
290
- *
291
- * NOTE: These options will not respect semantic versioning and may be removed or changed at any time. Use at your own risk.
292
- * If you do use these and encounter issues, please report them to us.
293
- */
294
- export interface CrawlerExperiments {
295
377
  /**
296
- * @deprecated This experiment is now enabled by default, and this flag will be removed in a future release.
297
- * If you encounter issues due to this change, please:
298
- * - report it to us: https://github.com/apify/crawlee
299
- * - set `requestLocking` to `false` in the `experiments` option of the crawler
300
- */
301
- requestLocking?: boolean;
302
- }
303
- /**
304
- * Provides a simple framework for parallel crawling of web pages.
305
- * The URLs to crawl are fed either from a static list of URLs
306
- * or from a dynamic queue of URLs enabling recursive crawling of websites.
307
- *
308
- * `BasicCrawler` is a low-level tool that requires the user to implement the page
309
- * download and data extraction functionality themselves.
310
- * If we want a crawler that already facilitates this functionality,
311
- * we should consider using {@link CheerioCrawler}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler}.
312
- *
313
- * `BasicCrawler` invokes the user-provided {@link BasicCrawlerOptions.requestHandler|`requestHandler`}
314
- * for each {@link Request} object, which represents a single URL to crawl.
315
- * The {@link Request} objects are fed from the {@link RequestList} or {@link RequestQueue}
316
- * instances provided by the {@link BasicCrawlerOptions.requestList|`requestList`} or {@link BasicCrawlerOptions.requestQueue|`requestQueue`}
317
- * constructor options, respectively. If neither `requestList` nor `requestQueue` options are provided,
318
- * the crawler will open the default request queue either when the {@link BasicCrawler.addRequests|`crawler.addRequests()`} function is called,
319
- * or if `requests` parameter (representing the initial requests) of the {@link BasicCrawler.run|`crawler.run()`} function is provided.
320
- *
321
- * If both {@link BasicCrawlerOptions.requestList|`requestList`} and {@link BasicCrawlerOptions.requestQueue|`requestQueue`} options are used,
322
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
323
- * to the {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
324
- *
325
- * The crawler finishes if there are no more {@link Request} objects to crawl.
326
- *
327
- * New requests are only dispatched when there is enough free CPU and memory available,
328
- * using the functionality provided by the {@link AutoscaledPool} class.
329
- * All {@link AutoscaledPool} configuration options can be passed to the {@link BasicCrawlerOptions.autoscaledPoolOptions|`autoscaledPoolOptions`}
330
- * parameter of the `BasicCrawler` constructor.
331
- * For user convenience, the {@link AutoscaledPoolOptions.minConcurrency|`minConcurrency`} and
332
- * {@link AutoscaledPoolOptions.maxConcurrency|`maxConcurrency`} options of the
333
- * underlying {@link AutoscaledPool} constructor are available directly in the `BasicCrawler` constructor.
334
- *
335
- * **Example usage:**
336
- *
337
- * ```javascript
338
- * import { BasicCrawler, Dataset } from 'crawlee';
339
- *
340
- * // Create a crawler instance
341
- * const crawler = new BasicCrawler({
342
- * async requestHandler({ request, sendRequest }) {
343
- * // 'request' contains an instance of the Request class
344
- * // Here we simply fetch the HTML of the page and store it to a dataset
345
- * const { body } = await sendRequest({
346
- * url: request.url,
347
- * method: request.method,
348
- * body: request.payload,
349
- * headers: request.headers,
350
- * });
351
- *
352
- * await Dataset.pushData({
353
- * url: request.url,
354
- * html: body,
355
- * })
356
- * },
357
- * });
358
- *
359
- * // Enqueue the initial requests and run the crawler
360
- * await crawler.run([
361
- * 'http://www.example.com/page-1',
362
- * 'http://www.example.com/page-2',
363
- * ]);
364
- * ```
365
- * @category Crawlers
366
- */
367
- export declare class BasicCrawler<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension> {
368
- protected static readonly CRAWLEE_STATE_KEY = "CRAWLEE_STATE";
369
- /**
370
- * Tracks crawler instances that accessed shared state without having an explicit id.
371
- * Used to detect and warn about multiple crawlers sharing the same state.
378
+ * Makes the storage writes performed while handling a request atomic with respect to the request
379
+ * succeeding: they are recorded in a {@link StorageTransaction} spanning the whole request
380
+ * lifecycle and only applied when the request handler succeeds, so a thrown handler leaves no partial
381
+ * writes behind and a retry does not double-write. Reads within the handler see its own writes.
382
+ *
383
+ * `false` disables the mechanism entirely; an object overrides the per-storage-type
384
+ * {@link StorageWritePolicy} (e.g. `{ requestQueue: 'deferred' }` for all-or-nothing enqueues).
385
+ * {@link withDirectStorageAccess} is the per-call-site escape hatch; `useState()` is deliberately
386
+ * *not* transactional.
387
+ *
388
+ * @default true
372
389
  */
373
- private static useStateCrawlerIds;
390
+ transactionalStorage?: boolean | Partial<StorageWritePolicy>;
374
391
  /**
375
- * A reference to the underlying {@link Statistics} class that collects and logs run statistics for requests.
392
+ * An array of HTTP response [Status Codes](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status) to be excluded from error consideration.
393
+ * By default, status codes >= 500 trigger errors.
376
394
  */
377
- readonly stats: Statistics;
395
+ ignoreHttpErrorStatusCodes?: number[];
378
396
  /**
379
- * A reference to the underlying {@link RequestList} class that manages the crawler's {@link Request|requests}.
380
- * Only available if used by the crawler.
397
+ * An array of additional HTTP response [Status Codes](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status) to be treated as errors.
398
+ * By default, status codes >= 500 trigger errors.
381
399
  */
382
- requestList?: IRequestList;
400
+ additionalHttpErrorStatusCodes?: number[];
401
+ }
402
+ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> {
403
+ #private;
383
404
  /**
384
- * Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
385
- * A reference to the underlying {@link RequestQueue} class that manages the crawler's {@link Request|requests}.
386
- * Only available if used by the crawler.
405
+ * The statistics instance collecting the crawler's run statistics - either the injected `statistics` option or a
406
+ * crawler-built default. Typed as {@link IStatistics} so custom implementations can be plugged in.
387
407
  */
388
- requestQueue?: RequestProvider;
408
+ get statistics(): IStatistics<StatisticStateExtension>;
389
409
  /**
390
- * The main request-handling component of the crawler. It's initialized during the crawler startup.
410
+ * The main request-handling component of the crawler. It manages the requests that the crawler processes,
411
+ * combining any provided request loader and/or queue. It's initialized during the crawler startup or lazily
412
+ * via {@link BasicCrawler.getRequestManager|`getRequestManager()`}.
391
413
  */
392
- protected requestManager?: IRequestManager;
414
+ protected get requestManager(): IRequestManager | undefined;
393
415
  /**
394
- * A reference to the underlying {@link SessionPool} class that manages the crawler's {@link Session|sessions}.
395
- * Only available if used by the crawler.
416
+ * A reference to the underlying session pool that manages the crawler's {@link Session|sessions}. Typed as
417
+ * {@link ISessionPool} so custom implementations can be plugged in via the `sessionPool` constructor option.
396
418
  */
397
- sessionPool?: SessionPool;
419
+ get sessionPool(): ISessionPool;
420
+ /** Whether a {@link BasicCrawler.run|`run()`} is in progress on this instance. */
421
+ get running(): boolean;
398
422
  /**
399
- * A reference to the underlying {@link AutoscaledPool} class that manages the concurrency of the crawler.
400
- * > *NOTE:* This property is only initialized after calling the {@link BasicCrawler.run|`crawler.run()`} function.
401
- * We can use it to change the concurrency settings on the fly,
402
- * to pause the crawler by calling {@link AutoscaledPool.pause|`autoscaledPool.pause()`}
403
- * or to abort it by calling {@link AutoscaledPool.abort|`autoscaledPool.abort()`}.
423
+ * The concurrency governor the run in progress is booking its requests against — either the
424
+ * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} that was injected, or the default the
425
+ * crawler built for itself. Read it for telemetry: `desiredConcurrency`, `currentConcurrency`, `isRunning`.
426
+ *
427
+ * > *NOTE:* `undefined` outside a run, and a crawler-owned default is rebuilt for every run — so read it
428
+ * during a run rather than caching it across runs.
429
+ *
430
+ * {@link IConcurrencySystem} is deliberately read-only. Tuning concurrency *while a crawl is running* means
431
+ * owning the instance: build a {@link ConcurrencySystem} yourself and inject it, then set
432
+ * `minConcurrency`/`maxConcurrency`/`desiredConcurrency` on your own reference.
404
433
  */
405
- autoscaledPool?: AutoscaledPool;
434
+ get concurrencySystem(): IConcurrencySystem | undefined;
406
435
  /**
407
- * A reference to the underlying {@link ProxyConfiguration} class that manages the crawler's proxies.
436
+ * A reference to the underlying {@link IProxyConfiguration} instance that manages the crawler's proxies.
408
437
  * Only available if used by the crawler.
409
438
  */
410
- proxyConfiguration?: ProxyConfiguration;
439
+ readonly proxyConfiguration?: IProxyConfiguration;
411
440
  /**
412
441
  * Default {@link Router} instance that will be used if we don't specify any {@link BasicCrawlerOptions.requestHandler|`requestHandler`}.
413
442
  * See {@link Router.addHandler|`router.addHandler()`} and {@link Router.addDefaultHandler|`router.addDefaultHandler()`}.
414
443
  */
415
- readonly router: RouterHandler<Context>;
416
- private contextPipelineBuilder;
417
- private _contextPipeline?;
444
+ readonly router: RouterHandler<Context, Routes>;
445
+ /** @internal */
418
446
  get contextPipeline(): ContextPipeline<CrawlingContext, ExtendedContext>;
419
- running: boolean;
420
- hasFinishedBefore: boolean;
421
- readonly log: Log;
422
- protected requestHandler: RequestHandler<ExtendedContext>;
423
- protected errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
424
- protected failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
425
- protected requestHandlerTimeoutMillis: number;
426
- protected internalTimeoutMillis: number;
427
- protected maxRequestRetries: number;
428
- protected maxCrawlDepth?: number;
429
- protected sameDomainDelayMillis: number;
430
- protected domainAccessedTime: Map<string, number>;
431
- protected maxSessionRotations: number;
432
- protected maxRequestsPerCrawl?: number;
433
- protected handledRequestsCount: number;
434
- protected statusMessageLoggingInterval: number;
435
- protected statusMessageCallback?: StatusMessageCallback;
436
- protected sessionPoolOptions: SessionPoolOptions;
437
- protected useSessionPool: boolean;
438
- protected autoscaledPoolOptions: AutoscaledPoolOptions;
439
- protected httpClient: BaseHttpClient;
440
- protected retryOnBlocked: boolean;
441
- protected respectRobotsTxtFile: boolean | {
442
- userAgent?: string;
443
- };
444
- protected onSkippedRequest?: SkippedRequestCallback;
445
- private _closeEvents?;
446
- private shouldLogMaxProcessedRequestsExceeded;
447
- private shouldLogMaxEnqueuedRequestsExceeded;
448
- private experiments;
449
- private readonly robotsTxtFileCache;
450
- private _experimentWarnings;
451
- private readonly crawlerId;
452
- private readonly hasExplicitId;
447
+ /** Whether a `run()` on this instance has already finished - a repeated one continues where it left off. */
448
+ get hasFinishedBefore(): boolean;
449
+ get log(): CrawleeLogger;
450
+ protected readonly requestHandler: RequestHandler<ExtendedContext>;
451
+ protected readonly internalTimeoutMillis: number;
452
+ protected blockedStatusCodes: Set<number>;
453
+ protected readonly additionalHttpErrorStatusCodes: Set<number>;
454
+ protected readonly httpClient: BaseHttpClient;
455
+ protected readonly retryOnBlocked: boolean;
456
+ /**
457
+ * @internal
458
+ */
453
459
  protected static optionsShape: {
454
- // @ts-ignore optional peer dependency or compatibility with es2022
455
- contextPipelineBuilder: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
456
- // @ts-ignore optional peer dependency or compatibility with es2022
457
- extendContext: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
458
- // @ts-ignore optional peer dependency or compatibility with es2022
459
- requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
460
- // @ts-ignore optional peer dependency or compatibility with es2022
461
- requestQueue: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
462
- // @ts-ignore optional peer dependency or compatibility with es2022
463
- requestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
464
- // @ts-ignore optional peer dependency or compatibility with es2022
465
- requestHandlerTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
466
- // @ts-ignore optional peer dependency or compatibility with es2022
467
- errorHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
468
- // @ts-ignore optional peer dependency or compatibility with es2022
469
- failedRequestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
470
- // @ts-ignore optional peer dependency or compatibility with es2022
471
- maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
472
- // @ts-ignore optional peer dependency or compatibility with es2022
473
- sameDomainDelaySecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
474
- // @ts-ignore optional peer dependency or compatibility with es2022
475
- maxSessionRotations: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
476
- // @ts-ignore optional peer dependency or compatibility with es2022
477
- maxRequestsPerCrawl: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
478
- // @ts-ignore optional peer dependency or compatibility with es2022
479
- maxCrawlDepth: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
480
- // @ts-ignore optional peer dependency or compatibility with es2022
481
- autoscaledPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
482
- // @ts-ignore optional peer dependency or compatibility with es2022
483
- sessionPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
484
- // @ts-ignore optional peer dependency or compatibility with es2022
485
- useSessionPool: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
486
- // @ts-ignore optional peer dependency or compatibility with es2022
487
- proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
488
- // @ts-ignore optional peer dependency or compatibility with es2022
489
- statusMessageLoggingInterval: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
490
- // @ts-ignore optional peer dependency or compatibility with es2022
491
- statusMessageCallback: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
492
- // @ts-ignore optional peer dependency or compatibility with es2022
493
- retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
494
- // @ts-ignore optional peer dependency or compatibility with es2022
495
- respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
496
- // @ts-ignore optional peer dependency or compatibility with es2022
497
- onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
498
- // @ts-ignore optional peer dependency or compatibility with es2022
499
- httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
500
- // @ts-ignore optional peer dependency or compatibility with es2022
501
- configuration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
502
- // @ts-ignore optional peer dependency or compatibility with es2022
503
- storageClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
504
- // @ts-ignore optional peer dependency or compatibility with es2022
505
- eventManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
506
- // @ts-ignore optional peer dependency or compatibility with es2022
507
- minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
508
- // @ts-ignore optional peer dependency or compatibility with es2022
509
- maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
510
- // @ts-ignore optional peer dependency or compatibility with es2022
511
- maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
512
- // @ts-ignore optional peer dependency or compatibility with es2022
513
- keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
514
- // @ts-ignore optional peer dependency or compatibility with es2022
515
- log: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
516
- // @ts-ignore optional peer dependency or compatibility with es2022
517
- experiments: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
518
- // @ts-ignore optional peer dependency or compatibility with es2022
519
- statisticsOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
520
- // @ts-ignore optional peer dependency or compatibility with es2022
521
- id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
460
+ contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
461
+ extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
462
+ requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
463
+ requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
464
+ requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
465
+ requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
466
+ requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
467
+ errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
468
+ failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
469
+ maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
470
+ sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
471
+ maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
472
+ maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
473
+ taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
474
+ concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
475
+ sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
476
+ proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
477
+ statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
478
+ statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
479
+ additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
480
+ ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
481
+ blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
482
+ retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
483
+ respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
484
+ transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
485
+ requestQueue: z.ZodOptional<z.ZodEnum<{
486
+ deferred: "deferred";
487
+ writeThrough: "writeThrough";
488
+ }>>;
489
+ }, z.core.$strict>]>>;
490
+ onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
491
+ httpClient: z.ZodOptional<z.ZodInstanceOf<BaseHttpClient>>;
492
+ configuration: z.ZodOptional<z.ZodInstanceOf<Configuration>>;
493
+ storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
494
+ eventManager: z.ZodOptional<z.ZodInstanceOf<EventManager>>;
495
+ logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
496
+ minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
497
+ maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
498
+ initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
499
+ maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
500
+ keepAlive: z.ZodOptional<z.ZodBoolean>;
501
+ statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
502
+ id: z.ZodOptional<z.ZodString>;
522
503
  };
523
504
  /**
524
505
  * All `BasicCrawler` parameters are passed via an options object.
525
506
  */
526
- constructor(options?: BasicCrawlerOptions<Context, ContextExtension, ExtendedContext> & RequireContextPipeline<CrawlingContext, Context>);
507
+ constructor(options?: BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> & RequireContextPipeline<CrawlingContext, Context>);
508
+ /**
509
+ * Builds the crawler-owned default {@link ConcurrencySystem} from the resolved
510
+ * `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute` shortcuts. Not called when a
511
+ * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} was injected.
512
+ *
513
+ * Subclasses may override this to tune the default system (e.g. {@link HttpCrawler} raises the starting
514
+ * concurrency and relaxes the event loop signal) while still honouring the user's shortcuts.
515
+ */
516
+ protected createDefaultConcurrencySystem(options: ConcurrencySystemOptions): ConcurrencySystem;
517
+ /**
518
+ * Determines if the given HTTP status code is an error status code given
519
+ * the default behaviour and user-set preferences.
520
+ * @param status
521
+ * @returns `true` if the status code is considered an error, `false` otherwise
522
+ */
523
+ protected isErrorStatusCode(status: number): boolean;
527
524
  /**
528
525
  * Checks if the given error is a proxy error by comparing its message to a list of known proxy error messages.
529
526
  * Used for retrying requests that failed due to proxy errors.
@@ -532,21 +529,31 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
532
529
  */
533
530
  protected isProxyError(error: Error): boolean;
534
531
  /**
532
+ * Sets the status message for the current crawler run.
533
+ *
535
534
  * This method is periodically called by the crawler, every `statusMessageLoggingInterval` seconds.
535
+ *
536
+ * The message is logged and broadcast via the {@link EventType.STATUS_MESSAGE|`statusMessage`}
537
+ * event. Integrations such as the Apify SDK subscribe to that event and forward the message to
538
+ * their status-reporting backend (e.g. the Apify platform).
536
539
  */
537
- setStatusMessage(message: string, options?: SetStatusMessageOptions): Promise<void>;
538
- private getPeriodicLogger;
540
+ setStatusMessage(message: string, options?: SetStatusMessageOptions): void;
539
541
  /**
540
- * Runs the crawler. Returns a promise that resolves once all the requests are processed
541
- * and `autoscaledPool.isFinished` returns `true`.
542
+ * Runs the crawler. Returns a promise that resolves once every request has been processed and the crawler's
543
+ * finished-check ({@link BasicCrawlerOptions.taskLoopOptions|`taskLoopOptions.isFinishedFunction`}, or the
544
+ * default "the request manager is empty") reports that the crawl is over.
542
545
  *
543
546
  * We can use the `requests` parameter to enqueue the initial requests — it is a shortcut for
544
547
  * running {@link BasicCrawler.addRequests|`crawler.addRequests()`} before {@link BasicCrawler.run|`crawler.run()`}.
545
548
  *
549
+ * Calling `run()` again on the same instance keeps crawling the same request manager - requests the previous
550
+ * run handled (a failed one counts as handled) are not processed again. Purge the queue or open a fresh one
551
+ * if that is what you want.
552
+ *
546
553
  * @param [requests] The requests to add.
547
- * @param [options] Options for the request queue.
554
+ * @param [options] Options for adding the initial requests.
548
555
  */
549
- run(requests?: RequestsLike, options?: CrawlerRunOptions): Promise<FinalStatistics>;
556
+ run(requests?: TypedRequestsLike<Routes>, options?: CrawlerRunOptions): Promise<FinalStatistics>;
550
557
  /**
551
558
  * Gracefully stops the current run of the crawler.
552
559
  *
@@ -554,32 +561,57 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
554
561
  *
555
562
  * To stop the crawler immediately, use {@link BasicCrawler.teardown|`crawler.teardown()`} instead.
556
563
  */
557
- stop(message?: string): void;
558
- getRequestQueue(): Promise<RequestProvider>;
564
+ stop(reason?: string): void;
565
+ /**
566
+ * Stops dispatching new requests, letting the in-progress ones finish. Resolves once they have settled, or rejects
567
+ * after `timeoutSecs` if they take too long. Unlike {@link BasicCrawler.stop|`stop()`}, this does not end the
568
+ * run — {@link BasicCrawler.run|`run()`} stays pending until {@link BasicCrawler.resume|`resume()`}.
569
+ *
570
+ * > *NOTE:* The {@link BasicCrawler.concurrencySystem|concurrency system} keeps monitoring and autoscaling
571
+ * throughout, since a shared one may still be serving other crawlers.
572
+ */
573
+ pause(timeoutSecs?: number): Promise<void>;
574
+ /**
575
+ * Resumes a run suspended with {@link BasicCrawler.pause|`pause()`}, letting the crawler dispatch requests
576
+ * again. A no-op on a crawler that is not paused.
577
+ */
578
+ resume(): void;
579
+ /**
580
+ * Returns the crawler's {@link IRequestManager|request manager}, opening the default {@link RequestQueue}
581
+ * if none has been configured or opened yet.
582
+ */
583
+ getRequestManager(): Promise<IRequestManager>;
584
+ /**
585
+ * @deprecated Use {@link BasicCrawler.getRequestManager|`getRequestManager()`} instead. This returns the
586
+ * crawler's request manager, which is no longer guaranteed to be a {@link RequestQueue}.
587
+ */
588
+ getRequestQueue(): Promise<IRequestManager>;
559
589
  useState<State extends Dictionary = Dictionary>(defaultValue?: State): Promise<State>;
560
- protected get pendingRequestCountApproximation(): number;
561
- protected calculateEnqueuedRequestLimit(explicitLimit?: number): number | undefined;
562
- protected handleSkippedRequest(options: Parameters<SkippedRequestCallback>[0]): Promise<void>;
563
590
  /**
564
591
  * Adds requests to the queue in batches. By default, it will resolve after the initial batch is added, and continue
565
592
  * adding the rest in background. You can configure the batch size via `batchSize` option and the sleep time in between
566
593
  * the batches via `waitBetweenBatchesMillis`. If you want to wait for all batches to be added to the queue, you can use
567
594
  * the `waitForAllRequestsToBeAdded` promise you get in the response object.
568
595
  *
596
+ * Optionally, the requests can be filtered using `include`/`exclude` glob or regexp patterns and an
597
+ * enqueue `strategy` (both AND-ed together, same as {@link CrawlingContext.enqueueLinks|`enqueueLinks`}),
598
+ * relative to `baseUrl`. Unlike `enqueueLinks`, there is no implicit "current page" to anchor the strategy
599
+ * to, so `strategy` defaults to {@link EnqueueStrategy.All|`all`} here.
600
+ *
569
601
  * This is an alias for calling `addRequestsBatched()` on the implicit `RequestQueue` for this crawler instance.
570
602
  *
571
603
  * @param requests The requests to add
572
604
  * @param options Options for the request queue
573
605
  */
574
- addRequests(requests: ReadonlyDeep<RequestsLike>, options?: CrawlerAddRequestsOptions): Promise<CrawlerAddRequestsResult>;
606
+ addRequests(requests: ReadonlyDeep<TypedRequestsLike<Routes>>, options?: CrawlerAddRequestsOptions): Promise<CrawlerAddRequestsResult>;
575
607
  /**
576
608
  * Pushes data to the specified {@link Dataset}, or the default crawler {@link Dataset} by calling {@link Dataset.pushData}.
577
609
  */
578
- pushData(data: Parameters<Dataset['pushData']>[0], datasetIdOrName?: string): Promise<void>;
610
+ pushData(data: Parameters<Dataset['pushData']>[0], datasetIdentifier?: string | StorageIdentifier): Promise<void>;
579
611
  /**
580
612
  * Retrieves the specified {@link Dataset}, or the default crawler {@link Dataset}.
581
613
  */
582
- getDataset(idOrName?: string): Promise<Dataset>;
614
+ getDataset(identifier?: string | StorageIdentifier): Promise<Dataset>;
583
615
  /**
584
616
  * Retrieves data from the default crawler {@link Dataset} by calling {@link Dataset.getData}.
585
617
  */
@@ -592,112 +624,76 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
592
624
  /**
593
625
  * Initializes the crawler.
594
626
  */
595
- protected _init(): Promise<void>;
596
- protected runRequestHandler(crawlingContext: CrawlingContext): Promise<void>;
597
- /**
598
- * Handles blocked request
599
- */
600
- protected _throwOnBlockedRequest(session: Session, statusCode: number): void;
601
- private isAllowedBasedOnRobotsTxtFile;
602
- protected getRobotsTxtFileForUrl(url: string): Promise<RobotsTxtFile | undefined>;
603
- protected _pauseOnMigration(): Promise<void>;
604
- /**
605
- * Initializes the RequestManager based on the configured requestList and requestQueue.
606
- */
607
- private initializeRequestManager;
627
+ protected init(): Promise<void>;
608
628
  /**
609
- * Fetches the next request to process from the underlying request provider.
629
+ * The navigation timeout (pre-navigation hooks, navigation, and post-navigation hooks) in milliseconds, used
630
+ * to size the internal request timeout. `BasicCrawler` has no navigation phase, so this is 0; the HTTP and
631
+ * browser crawlers override it with their `navigationTimeoutSecs`.
610
632
  */
611
- protected _fetchNextRequest(): Promise<Request<Dictionary> | null>;
633
+ protected getNavigationTimeoutMillis(): number;
634
+ protected runRequestHandler(crawlingContext: ExtendedContext): Promise<void>;
612
635
  /**
613
- * Delays processing of the request based on the `sameDomainDelaySecs` option,
614
- * adding it back to the queue after the timeout passes. Returns `true` if the request
615
- * should be ignored and will be reclaimed to the queue once ready.
636
+ * Runs `callback` inside a {@link StorageTransaction}, unless transactional storage is disabled.
637
+ * Deliberately does **not** commit on return - the pipeline hands failures to `handleRequestFailure`, so a
638
+ * normal return says nothing about success. `handleRequest` commits on success.
616
639
  */
617
- protected delayRequest(request: Request, source: IRequestList | RequestProvider | IRequestManager): boolean;
640
+ private runInStorageTransaction;
618
641
  /**
619
- * Wrapper around requestHandler that fetches requests from RequestList/RequestQueue
620
- * then retries them in a case of an error, etc.
642
+ * Handles blocked request
621
643
  */
622
- protected _runTaskFunction(): Promise<void>;
644
+ protected throwOnBlockedRequest(statusCode: number): void;
623
645
  /**
624
- * Wrapper around the crawling context's `enqueueLinks` method:
625
- * - Injects `crawlDepth` to each request being added based on the crawling context request.
626
- * - Provides defaults for the `enqueueLinks` options based on the crawler configuration.
627
- * - These options can be overridden by the user.
628
- * @internal
646
+ * Records an HTTP 429 against the URL's domain so the request manager can hold the retry back.
647
+ *
648
+ * @param retryAfterHeader The raw `Retry-After` response header, if the server sent one.
649
+ * @returns `true` if the manager took responsibility for the delay, in which case the caller should throw
650
+ * {@link RequestThrottledError} rather than treating the response as a blocked session.
629
651
  */
630
- protected enqueueLinksWithCrawlDepth(options: SetRequired<EnqueueLinksOptions, 'urls'>, request: Request<Dictionary>, requestQueue: RequestProvider): Promise<BatchAddRequestsResult>;
652
+ protected recordDomainRateLimit(url: string, retryAfterHeader?: string | null): boolean;
653
+ /** @internal */
654
+ protected getRobotsTxtFileForUrl(url: string): Promise<RobotsTxtFile | undefined>;
655
+ /** Runs the request handler and marks the request as handled. Failures are left to `handleRequestFailure`. */
656
+ private handleRequest;
631
657
  /**
632
658
  * Generator function that yields requests injected with the given crawl depth.
633
659
  * @internal
634
660
  */
635
661
  protected addCrawlDepthRequestGenerator(requests: RequestsLike, newRequestDepth: number): AsyncGenerator<Source, void, undefined>;
636
- /**
637
- * Run async callback with given timeout and retry. Returns the result of the callback.
638
- * @ignore
639
- */
640
- protected _timeoutAndRetry<T>(handler: () => Promise<T>, timeout: number, error: Error | string, maxRetries?: number, retried?: number): Promise<T>;
641
- /**
642
- * Returns true if either RequestList or RequestQueue have a request ready for processing.
643
- */
644
- protected _isTaskReadyFunction(): Promise<boolean>;
645
- /**
646
- * Returns true if both RequestList and RequestQueue have all requests finished.
647
- */
648
- protected _defaultIsFinishedFunction(): Promise<boolean>;
649
- private _rotateSession;
650
- /**
651
- * Unwraps errors thrown by the context pipeline to get the actual user error.
652
- * RequestHandlerError and ContextPipelineInitializationError wrap the actual error.
653
- */
654
- private unwrapError;
655
662
  /**
656
663
  * Handles errors thrown by user provided requestHandler()
657
664
  */
658
- protected _requestFunctionErrorHandler(error: Error, crawlingContext: CrawlingContext, source: IRequestList | IRequestManager): Promise<void>;
659
- protected _tagUserHandlerError<T>(cb: () => unknown): Promise<T>;
660
- protected _handleFailedRequestHandler(crawlingContext: CrawlingContext, error: Error): Promise<void>;
665
+ private requestFunctionErrorHandler;
666
+ private handleFailedRequestHandler;
661
667
  /**
662
668
  * Resolves the most verbose error message from a thrown error
663
669
  * @param error The error received
664
670
  * @returns The message to be logged
665
671
  */
666
- protected _getMessageFromError(error: Error, forceStack?: boolean): string | TimeoutError | undefined;
667
- protected _canRequestBeRetried(request: Request, error: Error): boolean;
672
+ protected getMessageFromError(error: Error, forceStack?: boolean): string;
668
673
  /**
669
- * Updates handledRequestsCount from possibly stored counts, usually after worker migration.
670
- */
671
- protected _loadHandledRequestCount(): Promise<void>;
672
- protected _executeHooks<HookLike extends (...args: any[]) => Awaitable<void>>(hooks: HookLike[], ...args: Parameters<HookLike>): Promise<void>;
673
- /**
674
- * Stops the crawler immediately.
674
+ * Ends the run in progress immediately, without waiting for the requests already in flight.
675
675
  *
676
- * This method doesn't wait for currently active requests to finish.
676
+ * This runs at the end of every {@link BasicCrawler.run|`run()`}, so what it releases is per-run only, and
677
+ * a repeated `run()` has to find everything it needs either intact or re-establishable. What outlives a run is
678
+ * released by {@link BasicCrawler.destroy|`destroy()`} instead.
677
679
  *
678
680
  * To stop the crawler gracefully (waiting for all running requests to finish), use {@link BasicCrawler.stop|`crawler.stop()`} instead.
679
681
  */
680
682
  teardown(): Promise<void>;
681
- protected _getCookieHeaderFromRequest(request: Request): string;
682
- private _getRequestQueue;
683
- private requestMatchesEnqueueStrategy;
684
- }
685
- export interface CreateContextOptions {
686
- request: Request;
687
- session?: Session;
688
- proxyInfo?: ProxyInfo;
683
+ /**
684
+ * Releases what the crawler owns beyond a single run — in the browser crawlers, the browser pool. A finished
685
+ * `run()` has already released everything a run owns, so this is only needed before dropping a crawler you
686
+ * could otherwise have run again.
687
+ */
688
+ destroy(): Promise<void>;
689
+ [Symbol.asyncDispose](): Promise<void>;
690
+ protected getCookieHeaderFromRequest(request: Request): string;
689
691
  }
690
- export interface CrawlerAddRequestsOptions extends AddRequestsBatchedOptions {
692
+ export interface CrawlerAddRequestsOptions extends AddRequestsBatchedOptions, EnqueueUrlsOptions {
691
693
  }
692
694
  export interface CrawlerAddRequestsResult extends AddRequestsBatchedResult {
693
695
  }
694
696
  export interface CrawlerRunOptions extends CrawlerAddRequestsOptions {
695
- /**
696
- * Whether to purge the RequestQueue before running the crawler again. Defaults to true, so it is possible to reprocess failed requests.
697
- * When disabled, only new requests will be considered. Note that even a failed request is considered as handled.
698
- * @default true
699
- */
700
- purgeRequestQueue?: boolean;
701
697
  }
702
698
  /**
703
699
  * Creates new {@link Router} instance that works based on request labels.
@@ -723,5 +719,5 @@ export interface CrawlerRunOptions extends CrawlerAddRequestsOptions {
723
719
  * await crawler.run();
724
720
  * ```
725
721
  */
726
- export declare function createBasicRouter<Context extends BasicCrawlingContext = BasicCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): RouterHandler<Context>;
727
- //# sourceMappingURL=basic-crawler.d.ts.map
722
+ export declare function createBasicRouter<Context extends BasicCrawlingContext = BasicCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
723
+ export declare function createBasicRouter<Context extends BasicCrawlingContext = BasicCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;