@crawlee/basic 4.0.0-beta.28 → 4.0.0-beta.280

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/README.md +15 -15
  2. package/index.d.ts +11 -2
  3. package/index.js +11 -1
  4. package/internals/autoscaling/autoscaled-pool.d.ts +178 -0
  5. package/internals/autoscaling/autoscaled-pool.js +389 -0
  6. package/internals/autoscaling/concurrency-system.d.ts +235 -0
  7. package/internals/autoscaling/concurrency-system.js +357 -0
  8. package/internals/autoscaling/cpu-load-signal.d.ts +43 -0
  9. package/internals/autoscaling/cpu-load-signal.js +46 -0
  10. package/internals/autoscaling/event-loop-load-signal.d.ts +51 -0
  11. package/internals/autoscaling/event-loop-load-signal.js +60 -0
  12. package/internals/autoscaling/index.d.ts +9 -0
  13. package/internals/autoscaling/index.js +9 -0
  14. package/internals/autoscaling/load-signal.d.ts +98 -0
  15. package/internals/autoscaling/load-signal.js +103 -0
  16. package/internals/autoscaling/memory-load-signal.d.ts +45 -0
  17. package/internals/autoscaling/memory-load-signal.js +104 -0
  18. package/internals/autoscaling/snapshotter.d.ts +84 -0
  19. package/internals/autoscaling/snapshotter.js +67 -0
  20. package/internals/autoscaling/storage-backend-load-signal.d.ts +56 -0
  21. package/internals/autoscaling/storage-backend-load-signal.js +73 -0
  22. package/internals/autoscaling/system-status.d.ts +112 -0
  23. package/internals/autoscaling/system-status.js +139 -0
  24. package/internals/autoscaling/weighted-avg.d.ts +5 -0
  25. package/internals/autoscaling/weighted-avg.js +14 -0
  26. package/internals/basic-crawler.d.ts +370 -374
  27. package/internals/basic-crawler.js +1416 -857
  28. package/internals/cookie-utils.d.ts +40 -0
  29. package/internals/cookie-utils.js +122 -0
  30. package/internals/crawler-run.d.ts +76 -0
  31. package/internals/crawler-run.js +144 -0
  32. package/internals/crawlers/context-pipeline.d.ts +75 -0
  33. package/internals/crawlers/context-pipeline.js +109 -0
  34. package/internals/crawlers/crawler-commons.d.ts +216 -0
  35. package/internals/crawlers/crawler-commons.js +1 -0
  36. package/internals/crawlers/error-snapshotter.d.ts +29 -0
  37. package/internals/crawlers/error-snapshotter.js +116 -0
  38. package/internals/crawlers/error-tracker.d.ts +51 -0
  39. package/internals/crawlers/error-tracker.js +316 -0
  40. package/internals/crawlers/index.d.ts +4 -0
  41. package/internals/crawlers/index.js +3 -0
  42. package/internals/crawlers/internals/types.d.ts +7 -0
  43. package/internals/crawlers/internals/types.js +1 -0
  44. package/internals/crawlers/statistics.d.ts +326 -0
  45. package/internals/crawlers/statistics.js +559 -0
  46. package/internals/crawling-request.d.ts +86 -0
  47. package/internals/crawling-request.js +127 -0
  48. package/internals/enqueue-links/enqueue-links.d.ts +147 -0
  49. package/internals/enqueue-links/enqueue-links.js +35 -0
  50. package/internals/enqueue-links/index.d.ts +3 -0
  51. package/internals/enqueue-links/index.js +3 -0
  52. package/internals/enqueue-links/shared.d.ts +107 -0
  53. package/internals/enqueue-links/shared.js +228 -0
  54. package/internals/errors.d.ts +73 -0
  55. package/internals/errors.js +108 -0
  56. package/internals/http.d.ts +9 -0
  57. package/internals/http.js +28 -0
  58. package/internals/request-timeout.d.ts +54 -0
  59. package/internals/request-timeout.js +98 -0
  60. package/internals/router.d.ts +299 -0
  61. package/internals/router.js +312 -0
  62. package/internals/send-request.d.ts +4 -4
  63. package/internals/send-request.js +18 -8
  64. package/internals/session-pool/consts.d.ts +1 -0
  65. package/internals/session-pool/consts.js +1 -0
  66. package/internals/session-pool/errors.d.ts +7 -0
  67. package/internals/session-pool/errors.js +11 -0
  68. package/internals/session-pool/fingerprint.d.ts +9 -0
  69. package/internals/session-pool/fingerprint.js +30 -0
  70. package/internals/session-pool/index.d.ts +4 -0
  71. package/internals/session-pool/index.js +4 -0
  72. package/internals/session-pool/session-pool.d.ts +193 -0
  73. package/internals/session-pool/session-pool.js +453 -0
  74. package/internals/session-pool/session.d.ts +151 -0
  75. package/internals/session-pool/session.js +225 -0
  76. package/internals/sitemap-request-loader.d.ts +165 -0
  77. package/internals/sitemap-request-loader.js +492 -0
  78. package/internals/throttling-request-manager.d.ts +229 -0
  79. package/internals/throttling-request-manager.js +860 -0
  80. package/package.json +23 -17
  81. package/index.d.ts.map +0 -1
  82. package/index.js.map +0 -1
  83. package/internals/basic-crawler.d.ts.map +0 -1
  84. package/internals/basic-crawler.js.map +0 -1
  85. package/internals/send-request.d.ts.map +0 -1
  86. package/internals/send-request.js.map +0 -1
@@ -1,17 +1,48 @@
1
- import { writeFile } from 'node:fs/promises';
1
+ import { mkdir, writeFile } from 'node:fs/promises';
2
2
  import { dirname } from 'node:path';
3
- import { AutoscaledPool, bindMethodsToServiceLocator, ContextPipeline, ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, CriticalError, Dataset, enqueueLinks, EnqueueStrategy, KeyValueStore, mergeCookies, NonRetryableError, purgeDefaultStorages, RequestHandlerError, RequestListAdapter, RequestManagerTandem, RequestProvider, RequestQueue, RequestQueueV1, RequestState, RetryRequestError, Router, ServiceLocator, serviceLocator, SessionError, SessionPool, Statistics, validators, } from '@crawlee/core';
4
- import { GotScrapingHttpClient } from '@crawlee/got-scraping-client';
5
- import { getObjectType, isAsyncIterable, isIterable, RobotsTxtFile, ROTATE_PROXY_ERRORS } from '@crawlee/utils';
6
- import { stringify } from 'csv-stringify/sync';
7
- import { ensureDir, writeJSON } from 'fs-extra/esm';
8
- import ow from 'ow';
9
- import { getDomain } from 'tldts';
3
+ import { bindMethodsToServiceLocator, Configuration, createStorageTransaction, Request, CriticalError, currentStorageTransaction, Dataset, EnqueueStrategy, EventManager, EventType, getObjectType, KeyValueStore, log, NonRetryableError, OwnedOrInjected, purgeDefaultStorages, RequestManagerTandem, RequestQueue, ServiceLocator, serviceLocator, SessionError, validators, withDirectStorageAccess, } from '@crawlee/core';
4
+ import { BaseHttpClient, FetchHttpClient } from '@crawlee/http-client';
5
+ import { filterUrl, isAsyncIterable, isIterable, matchesEnqueueStrategy, parseArgument, ROTATE_PROXY_ERRORS, schemas, } from '@crawlee/utils/internal';
6
+ import { RobotsTxtFile } from '@crawlee/utils';
7
+ import { z } from 'zod';
10
8
  import { LruCache } from '@apify/datastructures';
11
- import defaultLog, { LogLevel } from '@apify/log';
12
- import { addTimeoutToPromise, TimeoutError, tryCancel } from '@apify/timeout';
9
+ import { addTimeoutToPromise, extendTimeout, storage as timeoutStorage, TimeoutError, tryCancel } from '@apify/timeout';
13
10
  import { cryptoRandomObjectId } from '@apify/utilities';
11
+ import { ConcurrencySystem } from './autoscaling/concurrency-system.js';
12
+ import { mergeCookies } from './cookie-utils.js';
13
+ import { ContextPipeline } from './crawlers/context-pipeline.js';
14
+ import { CrawlingRequest, RequestState } from './crawling-request.js';
15
+ import { ContextPipelineCleanupError, ContextPipelineInterruptedError, MissingSessionError, NavigationSkippedError, PersistentRateLimitError, ContextPipelineInitializationError, RequestHandlerError, RequestThrottledError, RetryRequestError, } from './errors.js';
16
+ import { Statistics } from './crawlers/statistics.js';
17
+ import { applyRequestTransform, constructUrlPatternObjects, createRequestOptions, createSkippedRequestArgs, filterRequestOptionsByPatterns, } from './enqueue-links/index.js';
18
+ import { parseRetryAfterHeader } from './http.js';
19
+ import { CrawlerRun } from './crawler-run.js';
20
+ import { extendTimeoutKey, navigationDeadlineKey, raceWithTimeout, suspendTimeoutKey, timeoutExpiredKey, } from './request-timeout.js';
14
21
  import { createSendRequest } from './send-request.js';
22
+ import { Router, validateUserData } from './router.js';
23
+ import { BLOCKED_STATUS_CODES } from './session-pool/consts.js';
24
+ import { Session } from './session-pool/session.js';
25
+ import { SessionPool } from './session-pool/session-pool.js';
26
+ import { ThrottlingRequestManager } from './throttling-request-manager.js';
27
+ class LazyDefaultHttpClient extends BaseHttpClient {
28
+ #delegatePromise;
29
+ constructor(options) {
30
+ super(options);
31
+ this.#delegatePromise = import('@crawlee/impit-client')
32
+ .then(({ ImpitHttpClient }) => new ImpitHttpClient(options))
33
+ .catch(() => {
34
+ (options?.logger ?? log).warning('Optional dependency @crawlee/impit-client is not installed. ' +
35
+ 'Falling back to native fetch — proxy support and browser fingerprinting are unavailable.');
36
+ return new FetchHttpClient(options);
37
+ });
38
+ }
39
+ fetch() {
40
+ throw new Error('LazyDefaultHttpClient delegates `sendRequest` entirely; `fetch` is never called.');
41
+ }
42
+ async sendRequest(...args) {
43
+ return (await this.#delegatePromise).sendRequest(...args);
44
+ }
45
+ }
15
46
  /**
16
47
  * Since there's no set number of seconds before the container is terminated after
17
48
  * a migration event, we need some reasonable number to use for RequestList persistence.
@@ -22,111 +53,119 @@ import { createSendRequest } from './send-request.js';
22
53
  * @ignore
23
54
  */
24
55
  const SAFE_MIGRATION_WAIT_MILLIS = 20000;
25
- /**
26
- * Provides a simple framework for parallel crawling of web pages.
27
- * The URLs to crawl are fed either from a static list of URLs
28
- * or from a dynamic queue of URLs enabling recursive crawling of websites.
29
- *
30
- * `BasicCrawler` is a low-level tool that requires the user to implement the page
31
- * download and data extraction functionality themselves.
32
- * If we want a crawler that already facilitates this functionality,
33
- * we should consider using {@link CheerioCrawler}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler}.
34
- *
35
- * `BasicCrawler` invokes the user-provided {@link BasicCrawlerOptions.requestHandler|`requestHandler`}
36
- * for each {@link Request} object, which represents a single URL to crawl.
37
- * The {@link Request} objects are fed from the {@link RequestList} or {@link RequestQueue}
38
- * instances provided by the {@link BasicCrawlerOptions.requestList|`requestList`} or {@link BasicCrawlerOptions.requestQueue|`requestQueue`}
39
- * constructor options, respectively. If neither `requestList` nor `requestQueue` options are provided,
40
- * the crawler will open the default request queue either when the {@link BasicCrawler.addRequests|`crawler.addRequests()`} function is called,
41
- * or if `requests` parameter (representing the initial requests) of the {@link BasicCrawler.run|`crawler.run()`} function is provided.
42
- *
43
- * If both {@link BasicCrawlerOptions.requestList|`requestList`} and {@link BasicCrawlerOptions.requestQueue|`requestQueue`} options are used,
44
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
45
- * to the {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
46
- *
47
- * The crawler finishes if there are no more {@link Request} objects to crawl.
48
- *
49
- * New requests are only dispatched when there is enough free CPU and memory available,
50
- * using the functionality provided by the {@link AutoscaledPool} class.
51
- * All {@link AutoscaledPool} configuration options can be passed to the {@link BasicCrawlerOptions.autoscaledPoolOptions|`autoscaledPoolOptions`}
52
- * parameter of the `BasicCrawler` constructor.
53
- * For user convenience, the {@link AutoscaledPoolOptions.minConcurrency|`minConcurrency`} and
54
- * {@link AutoscaledPoolOptions.maxConcurrency|`maxConcurrency`} options of the
55
- * underlying {@link AutoscaledPool} constructor are available directly in the `BasicCrawler` constructor.
56
- *
57
- * **Example usage:**
58
- *
59
- * ```javascript
60
- * import { BasicCrawler, Dataset } from 'crawlee';
61
- *
62
- * // Create a crawler instance
63
- * const crawler = new BasicCrawler({
64
- * async requestHandler({ request, sendRequest }) {
65
- * // 'request' contains an instance of the Request class
66
- * // Here we simply fetch the HTML of the page and store it to a dataset
67
- * const { body } = await sendRequest({
68
- * url: request.url,
69
- * method: request.method,
70
- * body: request.payload,
71
- * headers: request.headers,
72
- * });
73
- *
74
- * await Dataset.pushData({
75
- * url: request.url,
76
- * html: body,
77
- * })
78
- * },
79
- * });
80
- *
81
- * // Enqueue the initial requests and run the crawler
82
- * await crawler.run([
83
- * 'http://www.example.com/page-1',
84
- * 'http://www.example.com/page-2',
85
- * ]);
86
- * ```
87
- * @category Crawlers
88
- */
56
+ // The request timeout plumbing (the window helper, the context symbols, and the race) lives in its own module.
57
+ export { navigationDeadlineKey, remainingNavigationWindowMillis } from './request-timeout.js';
58
+ const urlPatternSchema = z.union([
59
+ z.string(),
60
+ z.instanceof(RegExp),
61
+ schemas.objectWithKeys(['glob']),
62
+ schemas.objectWithKeys(['regexp']),
63
+ ]);
64
+ // `looseObject` (rather than `strictObject`) lets subclasses forward their own extraction-only options
65
+ // (e.g. `selector`) straight through without having to strip them out first.
66
+ const addRequestsOptionsSchema = z.looseObject({
67
+ forefront: z.boolean().optional(),
68
+ cache: z.boolean().optional(),
69
+ waitForAllRequestsToBeAdded: z.boolean().optional(),
70
+ batchSize: schemas.anyNumber.optional(),
71
+ waitBetweenBatchesMillis: schemas.anyNumber.optional(),
72
+ maxNewRequests: schemas.anyNumber.optional(),
73
+ limit: schemas.anyNumber.optional(),
74
+ baseUrl: z.string().optional(),
75
+ userData: schemas.anyObject.optional(),
76
+ label: z.string().optional(),
77
+ sessionId: z.string().optional(),
78
+ skipNavigation: z.boolean().optional(),
79
+ include: schemas.arrayOf(urlPatternSchema, 'URL patterns').min(1).optional(),
80
+ exclude: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
81
+ transformRequestFunction: schemas.anyFunction.optional(),
82
+ strategy: z.enum(EnqueueStrategy).optional(),
83
+ onSkippedRequest: schemas.anyFunction.optional(),
84
+ });
89
85
  export class BasicCrawler {
90
- static CRAWLEE_STATE_KEY = 'CRAWLEE_STATE';
86
+ static #CRAWLEE_STATE_KEY = 'CRAWLEE_STATE';
87
+ /**
88
+ * Tracks the number of crawler instances created. The first crawler uses the default
89
+ * request queue; subsequent ones get their own queue via a unique alias so they don't
90
+ * collide.
91
+ */
92
+ static #instanceCount = 0;
91
93
  /**
92
94
  * Tracks crawler instances that accessed shared state without having an explicit id.
93
95
  * Used to detect and warn about multiple crawlers sharing the same state.
94
96
  */
95
- static useStateCrawlerIds = new Set();
97
+ static #useStateAnonymousIndices = new Set();
98
+ /** Backs the {@link BasicCrawler.statistics|`statistics`} getter. */
99
+ #statisticsDep;
96
100
  /**
97
- * A reference to the underlying {@link Statistics} class that collects and logs run statistics for requests.
101
+ * The statistics instance collecting the crawler's run statistics - either the injected `statistics` option or a
102
+ * crawler-built default. Typed as {@link IStatistics} so custom implementations can be plugged in.
98
103
  */
99
- stats;
104
+ get statistics() {
105
+ return this.#statisticsDep.value;
106
+ }
107
+ #requestManager;
100
108
  /**
101
- * A reference to the underlying {@link RequestList} class that manages the crawler's {@link Request|requests}.
102
- * Only available if used by the crawler.
109
+ * The main request-handling component of the crawler. It manages the requests that the crawler processes,
110
+ * combining any provided request loader and/or queue. It's initialized during the crawler startup or lazily
111
+ * via {@link BasicCrawler.getRequestManager|`getRequestManager()`}.
103
112
  */
104
- requestList;
113
+ get requestManager() {
114
+ return this.#requestManager;
115
+ }
116
+ /** Backs the {@link BasicCrawler.sessionPool|`sessionPool`} getter. */
117
+ #sessionPoolDep;
105
118
  /**
106
- * Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
107
- * A reference to the underlying {@link RequestQueue} class that manages the crawler's {@link Request|requests}.
108
- * Only available if used by the crawler.
119
+ * A reference to the underlying session pool that manages the crawler's {@link Session|sessions}. Typed as
120
+ * {@link ISessionPool} so custom implementations can be plugged in via the `sessionPool` constructor option.
109
121
  */
110
- requestQueue;
122
+ get sessionPool() {
123
+ return this.#sessionPoolDep.value;
124
+ }
111
125
  /**
112
- * The main request-handling component of the crawler. It's initialized during the crawler startup.
126
+ * Whether the request-processing-time hint has already been forwarded to the request manager. The hint
127
+ * derives only from `requestHandlerTimeoutMillis` (constant for the crawler's lifetime) and is raise-only,
128
+ * so it only needs to be applied once, at the first async access of the manager.
113
129
  */
114
- requestManager;
130
+ #requestManagerTimeoutsApplied = false;
115
131
  /**
116
- * A reference to the underlying {@link SessionPool} class that manages the crawler's {@link Session|sessions}.
117
- * Only available if used by the crawler.
132
+ * Resolves the governor for one run: either the injected
133
+ * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} (borrowed) or a freshly built default with
134
+ * the concurrency shortcuts folded in (owned, so the crawler starts and stops it).
118
135
  */
119
- sessionPool;
136
+ #resolveConcurrencySystem;
137
+ /** The last run started on this instance. It outlives the run, so anything that drives one goes through `#liveRun`. */
138
+ #run;
139
+ #concurrencySystem;
140
+ /** The run in progress, if any — a finished run is history and cannot be driven. */
141
+ get #liveRun() {
142
+ return this.#run?.isLive === true ? this.#run : undefined;
143
+ }
144
+ /** The run whose task loop is dispatching, if any — only that one can be paused and resumed. */
145
+ get #dispatchingRun() {
146
+ return this.#liveRun?.isDispatching === true ? this.#liveRun : undefined;
147
+ }
148
+ /** Whether a {@link BasicCrawler.run|`run()`} is in progress on this instance. */
149
+ get running() {
150
+ return this.#liveRun !== undefined;
151
+ }
120
152
  /**
121
- * A reference to the underlying {@link AutoscaledPool} class that manages the concurrency of the crawler.
122
- * > *NOTE:* This property is only initialized after calling the {@link BasicCrawler.run|`crawler.run()`} function.
123
- * We can use it to change the concurrency settings on the fly,
124
- * to pause the crawler by calling {@link AutoscaledPool.pause|`autoscaledPool.pause()`}
125
- * or to abort it by calling {@link AutoscaledPool.abort|`autoscaledPool.abort()`}.
153
+ * The concurrency governor the run in progress is booking its requests against — either the
154
+ * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} that was injected, or the default the
155
+ * crawler built for itself. Read it for telemetry: `desiredConcurrency`, `currentConcurrency`, `isRunning`.
156
+ *
157
+ * > *NOTE:* `undefined` outside a run, and a crawler-owned default is rebuilt for every run — so read it
158
+ * during a run rather than caching it across runs.
159
+ *
160
+ * {@link IConcurrencySystem} is deliberately read-only. Tuning concurrency *while a crawl is running* means
161
+ * owning the instance: build a {@link ConcurrencySystem} yourself and inject it, then set
162
+ * `minConcurrency`/`maxConcurrency`/`desiredConcurrency` on your own reference.
126
163
  */
127
- autoscaledPool;
164
+ get concurrencySystem() {
165
+ return this.#concurrencySystem?.value;
166
+ }
128
167
  /**
129
- * A reference to the underlying {@link ProxyConfiguration} class that manages the crawler's proxies.
168
+ * A reference to the underlying {@link IProxyConfiguration} instance that manages the crawler's proxies.
130
169
  * Only available if used by the crawler.
131
170
  */
132
171
  proxyConfiguration;
@@ -135,265 +174,535 @@ export class BasicCrawler {
135
174
  * See {@link Router.addHandler|`router.addHandler()`} and {@link Router.addDefaultHandler|`router.addDefaultHandler()`}.
136
175
  */
137
176
  router = Router.create();
138
- contextPipelineBuilder;
139
- _contextPipeline;
177
+ #basicContextPipeline;
178
+ /**
179
+ * The basic part of the context pipeline. Unlike the subclass pipeline, this
180
+ * part has no major side effects (e.g. launching a browser). It also makes typing more explicit, as subclass
181
+ * pipelines expect the basic crawler fields to already be present in the context at runtime.
182
+ *
183
+ * Context built with this pipeline can be passed into multiple crawler pipelines at once.
184
+ */
185
+ get #basicPipeline() {
186
+ this.#basicContextPipeline ??= this.#buildBasicContextPipeline();
187
+ return this.#basicContextPipeline;
188
+ }
189
+ #contextPipeline;
190
+ /** @internal */
140
191
  get contextPipeline() {
141
- if (this._contextPipeline === undefined) {
142
- this._contextPipeline = this.contextPipelineBuilder();
192
+ if (this.#contextPipeline === undefined) {
193
+ this.#contextPipeline = this.#buildFinalContextPipeline();
143
194
  }
144
- return this._contextPipeline;
195
+ return this.#contextPipeline;
196
+ }
197
+ #hasFinishedBefore = false;
198
+ /** Whether a `run()` on this instance has already finished - a repeated one continues where it left off. */
199
+ get hasFinishedBefore() {
200
+ return this.#hasFinishedBefore;
201
+ }
202
+ #log;
203
+ get log() {
204
+ return this.#log;
145
205
  }
146
- running = false;
147
- hasFinishedBefore = false;
148
- log;
149
206
  requestHandler;
150
- errorHandler;
151
- failedRequestHandler;
152
- requestHandlerTimeoutMillis;
207
+ #errorHandler;
208
+ #failedRequestHandler;
209
+ #requestHandlerTimeoutMillis;
153
210
  internalTimeoutMillis;
154
- maxRequestRetries;
155
- maxCrawlDepth;
156
- sameDomainDelayMillis;
157
- domainAccessedTime;
158
- maxSessionRotations;
159
- maxRequestsPerCrawl;
160
- handledRequestsCount = 0;
161
- statusMessageLoggingInterval;
162
- statusMessageCallback;
163
- sessionPoolOptions;
164
- useSessionPool;
165
- autoscaledPoolOptions;
211
+ #maxRequestRetries;
212
+ #maxCrawlDepth;
213
+ #maxRequestsPerCrawl;
214
+ get #handledRequestsCount() {
215
+ return this.statistics.state.requestsSucceeded + this.statistics.state.requestsFailed;
216
+ }
217
+ #statusMessageLoggingInterval;
218
+ #statusMessageCallback;
219
+ blockedStatusCodes = new Set();
220
+ additionalHttpErrorStatusCodes;
221
+ #ignoreHttpErrorStatusCodes;
222
+ /**
223
+ * The resolved options for the crawler's own task loop — the crawler-owned `runTaskFunction`, the (possibly
224
+ * user-overridden) ready/finished predicates and cadence/logging. Concurrency configuration lives on the
225
+ * {@link ConcurrencySystem} instead, and the loop's `consumer` identity is the crawler's own, so neither is
226
+ * settable here.
227
+ */
228
+ #taskLoopOptions;
166
229
  httpClient;
167
230
  retryOnBlocked;
168
- respectRobotsTxtFile;
169
- onSkippedRequest;
170
- _closeEvents;
171
- shouldLogMaxProcessedRequestsExceeded = true;
172
- shouldLogMaxEnqueuedRequestsExceeded = true;
173
- experiments;
174
- robotsTxtFileCache;
175
- _experimentWarnings = {};
176
- crawlerId;
177
- hasExplicitId;
231
+ #respectRobotsTxtFile;
232
+ /** Whether `runInStorageTransaction()` opens a transaction at all. */
233
+ #transactionalStorageEnabled;
234
+ /** The resolved per-storage-type write policy overrides forwarded to each request's transaction. */
235
+ #storageWritePolicy;
236
+ #onSkippedRequest;
237
+ #robotsTxtFileCache;
238
+ #identity;
239
+ #contextPipelineOptions;
240
+ /**
241
+ * @internal
242
+ */
178
243
  static optionsShape = {
179
- contextPipelineBuilder: ow.optional.object,
180
- extendContext: ow.optional.function,
181
- requestList: ow.optional.object.validate(validators.requestList),
182
- requestQueue: ow.optional.object.validate(validators.requestQueue),
244
+ contextPipelineBuilder: schemas.anyObject.optional(),
245
+ extendContext: schemas.anyFunction.optional(),
246
+ requestList: validators.requestList.optional(),
247
+ requestQueue: validators.requestQueue.optional(),
248
+ requestManager: validators.requestManager.optional(),
183
249
  // Subclasses override this function instead of passing it
184
250
  // in constructor, so this validation needs to apply only
185
251
  // if the user creates an instance of BasicCrawler directly.
186
- requestHandler: ow.optional.function,
187
- requestHandlerTimeoutSecs: ow.optional.number,
188
- errorHandler: ow.optional.function,
189
- failedRequestHandler: ow.optional.function,
190
- maxRequestRetries: ow.optional.number,
191
- sameDomainDelaySecs: ow.optional.number,
192
- maxSessionRotations: ow.optional.number,
193
- maxRequestsPerCrawl: ow.optional.number,
194
- maxCrawlDepth: ow.optional.number,
195
- autoscaledPoolOptions: ow.optional.object,
196
- sessionPoolOptions: ow.optional.object,
197
- useSessionPool: ow.optional.boolean,
198
- proxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration),
199
- statusMessageLoggingInterval: ow.optional.number,
200
- statusMessageCallback: ow.optional.function,
201
- retryOnBlocked: ow.optional.boolean,
202
- respectRobotsTxtFile: ow.optional.any(ow.boolean, ow.object),
203
- onSkippedRequest: ow.optional.function,
204
- httpClient: ow.optional.object,
205
- configuration: ow.optional.object,
206
- storageClient: ow.optional.object,
207
- eventManager: ow.optional.object,
252
+ requestHandler: schemas.anyFunction.optional(),
253
+ requestHandlerTimeoutSecs: schemas.anyNumber.optional(),
254
+ errorHandler: schemas.anyFunction.optional(),
255
+ failedRequestHandler: schemas.anyFunction.optional(),
256
+ maxRequestRetries: schemas.anyNumber.default(3),
257
+ sameDomainDelaySecs: schemas.anyNumber.default(0),
258
+ maxRequestsPerCrawl: schemas.anyNumber.optional(),
259
+ maxCrawlDepth: schemas.anyNumber.optional(),
260
+ // No zod default — subclasses provide their own fallback (e.g. HTTP-optimized pool options).
261
+ taskLoopOptions: schemas.anyObject.optional(),
262
+ concurrencySystem: schemas.anyObject.optional(),
263
+ sessionPool: validators.sessionPool.optional(),
264
+ proxyConfiguration: validators.proxyConfiguration.optional(),
265
+ statusMessageLoggingInterval: schemas.anyNumber.default(10),
266
+ statusMessageCallback: schemas.anyFunction.optional(),
267
+ additionalHttpErrorStatusCodes: schemas.arrayOf(schemas.anyNumber, 'numbers').default(() => []),
268
+ ignoreHttpErrorStatusCodes: schemas.arrayOf(schemas.anyNumber, 'numbers').default(() => []),
269
+ blockedStatusCodes: schemas.arrayOf(schemas.anyNumber, 'numbers').optional(),
270
+ retryOnBlocked: z.boolean().default(false),
271
+ respectRobotsTxtFile: z.union([z.boolean(), schemas.anyObject]).default(false),
272
+ transactionalStorage: z
273
+ .union([z.boolean(), z.strictObject({ requestQueue: z.enum(['deferred', 'writeThrough']).optional() })])
274
+ .optional(),
275
+ onSkippedRequest: schemas.anyFunction.optional(),
276
+ httpClient: schemas.httpClient.optional(),
277
+ configuration: z.instanceof(Configuration).optional(),
278
+ storageBackend: validators.storageBackend.optional(),
279
+ eventManager: z.instanceof(EventManager).optional(),
280
+ logger: validators.logger.optional(),
208
281
  // AutoscaledPool shorthands
209
- minConcurrency: ow.optional.number,
210
- maxConcurrency: ow.optional.number,
211
- maxRequestsPerMinute: ow.optional.number.integerOrInfinite.positive.greaterThanOrEqual(1),
212
- keepAlive: ow.optional.boolean,
213
- // internal
214
- log: ow.optional.object,
215
- experiments: ow.optional.object,
216
- statisticsOptions: ow.optional.object,
217
- id: ow.optional.string,
282
+ minConcurrency: schemas.anyNumber.optional(),
283
+ maxConcurrency: schemas.anyNumber.optional(),
284
+ initialConcurrency: schemas.anyNumber.optional(),
285
+ maxRequestsPerMinute: schemas.anyNumber
286
+ .refine((value) => Number.isInteger(value) || value === Infinity, 'Expected an integer or infinite number')
287
+ .refine((value) => value >= 1, 'Expected a number greater than or equal to 1')
288
+ .optional(),
289
+ keepAlive: z.boolean().optional(),
290
+ statistics: schemas.anyObject.optional(),
291
+ id: z.string().optional(),
218
292
  };
293
+ static #optionsSchema = z.strictObject(BasicCrawler.optionsShape);
219
294
  /**
220
295
  * All `BasicCrawler` parameters are passed via an options object.
221
296
  */
222
297
  constructor(options = {}) {
223
- ow(options, 'BasicCrawlerOptions', ow.object.exactShape(BasicCrawler.optionsShape));
224
- const { requestList, requestQueue, requestManager, maxRequestRetries = 3, sameDomainDelaySecs = 0, maxSessionRotations = 10, maxRequestsPerCrawl, maxCrawlDepth, autoscaledPoolOptions = {}, keepAlive, sessionPoolOptions = {}, useSessionPool = true, proxyConfiguration,
298
+ const parsedOptions = parseArgument(options, BasicCrawler.#optionsSchema, 'BasicCrawlerOptions');
299
+ const {
300
+ // oxlint-disable-next-line typescript/no-deprecated -- still accepted and folded into `requestManager` for back-compat
301
+ requestList,
302
+ // oxlint-disable-next-line typescript/no-deprecated -- still accepted and folded into `requestManager` for back-compat
303
+ requestQueue, requestManager, maxRequestRetries, sameDomainDelaySecs, maxRequestsPerCrawl, maxCrawlDepth, taskLoopOptions = {}, concurrencySystem, keepAlive, sessionPool, proxyConfiguration, additionalHttpErrorStatusCodes, ignoreHttpErrorStatusCodes,
225
304
  // Service locator options
226
- configuration, storageClient, eventManager,
305
+ configuration, storageBackend, eventManager, logger,
227
306
  // AutoscaledPool shorthands
228
- minConcurrency, maxConcurrency, maxRequestsPerMinute, retryOnBlocked = false, respectRobotsTxtFile = false, onSkippedRequest, requestHandler, requestHandlerTimeoutSecs, errorHandler, failedRequestHandler, statusMessageLoggingInterval = 10, statusMessageCallback, statisticsOptions, httpClient,
229
- // internal
230
- log = defaultLog.child({ prefix: this.constructor.name }), experiments = {}, id, } = options;
307
+ minConcurrency, maxConcurrency, initialConcurrency, maxRequestsPerMinute, blockedStatusCodes: blockedStatusCodesInput, retryOnBlocked, respectRobotsTxtFile, transactionalStorage, onSkippedRequest, requestHandler, requestHandlerTimeoutSecs, errorHandler, failedRequestHandler, statusMessageLoggingInterval, statusMessageCallback, statistics, httpClient, id, } = parsedOptions;
308
+ // All concurrency configuration lives on the `ConcurrencySystem`, so the shortcuts have nowhere to go once
309
+ // one is supplied - and silently dropping a `maxConcurrency` the user asked for is how crawls end up
310
+ // hammering a site.
311
+ if (concurrencySystem !== undefined &&
312
+ (minConcurrency !== undefined ||
313
+ maxConcurrency !== undefined ||
314
+ initialConcurrency !== undefined ||
315
+ maxRequestsPerMinute !== undefined)) {
316
+ throw new Error('The `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute` shortcuts ' +
317
+ 'cannot be combined with `concurrencySystem` - they configure the default `ConcurrencySystem` ' +
318
+ 'that a supplied one replaces. Pass them to the `ConcurrencySystem` constructor instead.');
319
+ }
231
320
  // Create per-crawler service locator if custom services were provided.
232
321
  // This wraps every method on the crawler instance so that calls to the global `serviceLocator`
233
322
  // (via AsyncLocalStorage) resolve to this scoped instance instead.
234
323
  // We also enter the scope for the rest of the constructor body, so that any code below
235
324
  // that accesses `serviceLocator` will see the correct (scoped) instance.
236
325
  let serviceLocatorScope = { enterScope: () => { }, exitScope: () => { } };
237
- if (storageClient ||
326
+ if (storageBackend ||
238
327
  eventManager ||
328
+ logger ||
239
329
  (configuration !== undefined && configuration !== serviceLocator.getConfiguration())) {
240
- const scopedServiceLocator = new ServiceLocator(configuration, eventManager, storageClient);
330
+ // Inherit the ambient locator's already-set services for anything not explicitly
331
+ // provided - e.g. only passing a `logger` must not detach the crawler from a globally
332
+ // configured storage backend.
333
+ const ambientServices = serviceLocator.getServicesIfSet();
334
+ const scopedServiceLocator = new ServiceLocator(configuration ?? ambientServices.configuration, eventManager ?? ambientServices.eventManager, storageBackend ?? ambientServices.storageBackend, logger ?? ambientServices.logger);
241
335
  serviceLocatorScope = bindMethodsToServiceLocator(scopedServiceLocator, this);
242
336
  }
243
337
  try {
244
338
  serviceLocatorScope.enterScope();
245
- // Store whether the user explicitly provided an ID
246
- this.hasExplicitId = id !== undefined;
247
- // Store the user-provided ID, or generate a unique one for tracking purposes (not for state key)
248
- this.crawlerId = id ?? cryptoRandomObjectId();
249
- // Store the builder so that it can be run when the contextPipeline is needed.
250
- // Invoking it immediately would cause problems with parent constructor call order.
251
- this.contextPipelineBuilder = () => {
252
- let contextPipeline = (options.contextPipelineBuilder?.() ??
253
- ContextPipeline.create()); // Thanks to the RequireContextPipeline, contextPipeline will only be undefined if InitialContextType is CrawlingContext
254
- if (options.extendContext !== undefined) {
255
- contextPipeline = contextPipeline.compose({
256
- action: async (context) => await options.extendContext(context),
257
- });
258
- }
259
- contextPipeline = contextPipeline.compose({
260
- action: async (context) => {
261
- const { request } = context;
262
- if (!this.requestMatchesEnqueueStrategy(request)) {
263
- // eslint-disable-next-line dot-notation
264
- const message = `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`;
265
- this.log.debug(message);
266
- request.noRetry = true;
267
- request.state = RequestState.SKIPPED;
268
- await this.handleSkippedRequest({ url: request.url, reason: 'redirect' });
269
- throw new ContextPipelineInterruptedError(message);
270
- }
271
- return context;
272
- },
273
- });
274
- return contextPipeline;
339
+ this.#contextPipelineOptions = {
340
+ contextPipelineBuilder: parsedOptions.contextPipelineBuilder,
341
+ extendContext: parsedOptions.extendContext,
275
342
  };
276
- if (requestManager !== undefined) {
277
- if (requestList !== undefined || requestQueue !== undefined) {
278
- throw new Error('The `requestManager` option cannot be used in conjunction with `requestList` and/or `requestQueue`');
279
- }
280
- this.requestManager = requestManager;
281
- this.requestQueue = requestManager; // TODO(v4) - the cast is not fully legitimate here, but it's fine for internal usage by the BasicCrawler
343
+ this.#log = serviceLocator.getLogger().child({ prefix: this.constructor.name });
344
+ // Initialize the Configuration instance to avoid lazy loading in the components
345
+ serviceLocator.getConfiguration();
346
+ const instanceIndex = BasicCrawler.#instanceCount++;
347
+ this.#identity = { instanceIndex, hasExplicitId: id !== undefined, id: id ?? String(instanceIndex) };
348
+ if (requestManager !== undefined && (requestList !== undefined || requestQueue !== undefined)) {
349
+ throw new Error('The `requestManager` option cannot be used in conjunction with `requestList` and/or `requestQueue`');
282
350
  }
283
- else {
284
- this.requestList = requestList;
285
- this.requestQueue = requestQueue;
351
+ const suppliedManager = requestManager ?? requestQueue;
352
+ // Offered before building a pacer of our own: anything that paces takes the floor - through any
353
+ // number of wrappers, since they all forward - so no domain ends up with two clocks.
354
+ const floorTaken = sameDomainDelaySecs > 0 &&
355
+ (suppliedManager?.recordPacingSignal({
356
+ reason: 'minIntervalEverywhere',
357
+ intervalMs: sameDomainDelaySecs * 1000,
358
+ // What `sameDomainDelaySecs` has always meant: one clock per site, subdomains included.
359
+ scope: 'registrableDomain',
360
+ }) ??
361
+ false);
362
+ const pacerNeeded = sameDomainDelaySecs > 0 && !floorTaken;
363
+ // Built here rather than at first use so it can sit *inside* the tandem below, which is where a
364
+ // loader's transferred requests pass through it.
365
+ const writableManager = pacerNeeded
366
+ ? new ThrottlingRequestManager({
367
+ domains: 'all',
368
+ minCrawlDelaySecs: sameDomainDelaySecs,
369
+ throttleBy: 'registrableDomain',
370
+ persistStateKey: `CRAWLEE_THROTTLED_DOMAINS_${this.#identity.id}`,
371
+ // A factory, because the default queue is only opened on first use.
372
+ inner: suppliedManager ?? (() => this.#openOwnedRequestQueue()),
373
+ })
374
+ : suppliedManager;
375
+ if (requestList !== undefined) {
376
+ // The list is read first, while new requests still have somewhere writable to go.
377
+ this.#requestManager = new RequestManagerTandem(requestList, writableManager ?? (() => this.#openOwnedRequestQueue()));
286
378
  }
287
- this.httpClient = httpClient ?? new GotScrapingHttpClient();
379
+ else if (writableManager !== undefined) {
380
+ // A RequestQueue is itself a request manager.
381
+ this.#requestManager = writableManager;
382
+ }
383
+ this.httpClient = httpClient ?? new LazyDefaultHttpClient({ logger: this.log });
288
384
  this.proxyConfiguration = proxyConfiguration;
289
- this.log = log;
290
- this.statusMessageLoggingInterval = statusMessageLoggingInterval;
291
- this.statusMessageCallback = statusMessageCallback;
292
- this.domainAccessedTime = new Map();
293
- this.experiments = experiments;
294
- this.robotsTxtFileCache = new LruCache({ maxLength: 1000 });
295
- this.handleSkippedRequest = this.handleSkippedRequest.bind(this);
385
+ this.#statusMessageLoggingInterval = statusMessageLoggingInterval;
386
+ this.#statusMessageCallback = statusMessageCallback;
387
+ this.#robotsTxtFileCache = new LruCache({ maxLength: 1000 });
388
+ this.additionalHttpErrorStatusCodes = new Set([...additionalHttpErrorStatusCodes]);
389
+ this.#ignoreHttpErrorStatusCodes = new Set([...ignoreHttpErrorStatusCodes]);
296
390
  this.requestHandler = requestHandler ?? this.router;
297
- this.failedRequestHandler = failedRequestHandler;
298
- this.errorHandler = errorHandler;
391
+ this.#failedRequestHandler = failedRequestHandler;
392
+ this.#errorHandler = errorHandler;
299
393
  if (requestHandlerTimeoutSecs) {
300
- this.requestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
394
+ this.#requestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
301
395
  }
302
396
  else {
303
- this.requestHandlerTimeoutMillis = 60_000;
397
+ this.#requestHandlerTimeoutMillis = 60_000;
304
398
  }
305
399
  this.retryOnBlocked = retryOnBlocked;
306
- this.respectRobotsTxtFile = respectRobotsTxtFile;
307
- this.onSkippedRequest = onSkippedRequest;
308
- const tryEnv = (val) => (val == null ? null : +val);
400
+ this.#respectRobotsTxtFile = respectRobotsTxtFile;
401
+ // The cast undoes ow's assertion signature, which mangles `boolean | object` unions.
402
+ const transactionalStorageOption = transactionalStorage;
403
+ this.#transactionalStorageEnabled = transactionalStorageOption !== false;
404
+ this.#storageWritePolicy = typeof transactionalStorageOption === 'object' ? transactionalStorageOption : {};
405
+ this.#onSkippedRequest = onSkippedRequest;
309
406
  // allow at least 5min for internal timeouts
310
407
  this.internalTimeoutMillis =
311
- tryEnv(process.env.CRAWLEE_INTERNAL_TIMEOUT) ?? Math.max(this.requestHandlerTimeoutMillis * 2, 300e3);
312
- // override the default internal timeout of request queue to respect `requestHandlerTimeoutMillis`
313
- if (this.requestQueue) {
314
- this.requestQueue.internalTimeoutMillis = this.internalTimeoutMillis;
315
- // for request queue v2, we want to lock requests for slightly longer than the request handler timeout so that there is some padding for locking-related overhead,
316
- // but never for less than a minute
317
- this.requestQueue.requestLockSecs = Math.max(this.requestHandlerTimeoutMillis / 1000 + 5, 60);
318
- }
319
- this.maxRequestRetries = maxRequestRetries;
320
- this.maxCrawlDepth = maxCrawlDepth;
321
- this.sameDomainDelayMillis = sameDomainDelaySecs * 1000;
322
- this.maxSessionRotations = maxSessionRotations;
323
- this.stats = new Statistics({
324
- logMessage: `${log.getOptions().prefix} request statistics:`,
325
- log,
326
- ...(this.hasExplicitId ? { id: this.crawlerId } : {}),
327
- ...statisticsOptions,
328
- });
329
- this.sessionPoolOptions = {
330
- ...sessionPoolOptions,
331
- log,
332
- };
333
- if (this.retryOnBlocked) {
334
- this.sessionPoolOptions.blockedStatusCodes = sessionPoolOptions.blockedStatusCodes ?? [];
335
- if (this.sessionPoolOptions.blockedStatusCodes.length !== 0) {
336
- log.warning(`Both 'blockedStatusCodes' and 'retryOnBlocked' are set. Please note that the 'retryOnBlocked' feature might not work as expected.`);
337
- }
408
+ serviceLocator.getConfiguration().internalTimeoutMillis ??
409
+ Math.max(this.#requestHandlerTimeoutMillis * 2, 300e3);
410
+ this.#maxRequestRetries = maxRequestRetries;
411
+ this.#maxCrawlDepth = maxCrawlDepth;
412
+ this.#statisticsDep = OwnedOrInjected.resolve(statistics,
413
+ // A crawler-built default tracks the built-in fields only. A non-empty `StatisticStateExtension` can
414
+ // only be satisfied by an injected instance carrying the matching `state`, so this branch does
415
+ // not run in that case - hence the cast.
416
+ () => new Statistics({
417
+ logMessage: `${this.constructor.name} request statistics:`,
418
+ log: this.log,
419
+ id: this.#identity.id,
420
+ }));
421
+ if (sessionPool && proxyConfiguration) {
422
+ this.log.warning('Both `sessionPool` and `proxyConfiguration` were provided to the crawler. ' +
423
+ 'The `proxyConfiguration` is ignored - sessions from the supplied pool keep whatever ' +
424
+ '`proxyInfo` they were created with. Configure proxies on the pool instead, ' +
425
+ 'e.g. via `addSession({ proxyInfo })` or a custom `createSessionFunction`.');
338
426
  }
339
- this.useSessionPool = useSessionPool;
427
+ this.#sessionPoolDep = OwnedOrInjected.resolve(sessionPool, () => new SessionPool({
428
+ createSessionFunction: async (opts) => new Session({
429
+ ...opts?.sessionOptions,
430
+ proxyInfo: this.proxyConfiguration
431
+ ? await this.proxyConfiguration.newProxyInfo(opts?.sessionOptions?.proxyInfo)
432
+ : opts?.sessionOptions?.proxyInfo,
433
+ }),
434
+ }));
435
+ this.blockedStatusCodes = new Set(blockedStatusCodesInput ?? BLOCKED_STATUS_CODES);
340
436
  const maxSignedInteger = 2 ** 31 - 1;
341
- if (this.requestHandlerTimeoutMillis > maxSignedInteger) {
342
- log.warning(`requestHandlerTimeoutMillis ${this.requestHandlerTimeoutMillis}` +
437
+ if (this.#requestHandlerTimeoutMillis > maxSignedInteger) {
438
+ this.log.warning(`requestHandlerTimeoutMillis ${this.#requestHandlerTimeoutMillis}` +
343
439
  ` does not fit a signed 32-bit integer. Limiting the value to ${maxSignedInteger}`);
344
- this.requestHandlerTimeoutMillis = maxSignedInteger;
440
+ this.#requestHandlerTimeoutMillis = maxSignedInteger;
345
441
  }
346
442
  this.internalTimeoutMillis = Math.min(this.internalTimeoutMillis, maxSignedInteger);
347
- this.maxRequestsPerCrawl = maxRequestsPerCrawl;
348
- const isMaxPagesExceeded = () => this.maxRequestsPerCrawl && this.maxRequestsPerCrawl <= this.handledRequestsCount;
443
+ this.#maxRequestsPerCrawl = maxRequestsPerCrawl;
444
+ const isMaxPagesExceeded = () => this.#maxRequestsPerCrawl && this.#maxRequestsPerCrawl <= this.#handledRequestsCount;
349
445
  // eslint-disable-next-line prefer-const
350
- let { isFinishedFunction, isTaskReadyFunction } = autoscaledPoolOptions;
446
+ let { isFinishedFunction, isTaskReadyFunction } = taskLoopOptions;
351
447
  // override even if `isFinishedFunction` provided by user - `keepAlive` has higher priority
352
448
  if (keepAlive) {
353
449
  isFinishedFunction = async () => false;
354
450
  }
355
- const basicCrawlerAutoscaledPoolConfiguration = {
356
- minConcurrency: minConcurrency ?? autoscaledPoolOptions?.minConcurrency,
357
- maxConcurrency: maxConcurrency ?? autoscaledPoolOptions?.maxConcurrency,
358
- maxTasksPerMinute: maxRequestsPerMinute ?? autoscaledPoolOptions?.maxTasksPerMinute,
359
- runTaskFunction: this._runTaskFunction.bind(this),
451
+ const crawlerOwnedTaskLoopConfiguration = {
452
+ runTaskFunction: async () => {
453
+ const source = this.requestManager;
454
+ if (!source)
455
+ throw new Error('Request provider is not initialized!');
456
+ const request = await this.#resolveRequest();
457
+ if (!request) {
458
+ return;
459
+ }
460
+ // Started here, rather than in `handleRequest`, so that a failure during context pipeline
461
+ // initialization (e.g. a browser page timing out before the request handler ever runs) is
462
+ // still accounted for by `recordRequestFailure` below - which is a no-op without a
463
+ // matching `recordRequestStart`.
464
+ this.statistics.recordRequestStart(request.id || request.uniqueKey);
465
+ const crawlingContext = { request };
466
+ try {
467
+ // The transaction spans the whole pipeline call, covering the navigation hooks
468
+ // and `extendContext` too; `handleRequest` commits it and `handleRequestFailure` rolls it back.
469
+ await this.runInStorageTransaction(async () =>
470
+ // Navigation, the navigation hooks and the request handler are timed individually, but the
471
+ // phases between them are not, so a request could still get stuck indefinitely. This is the
472
+ // catch-all for that - see `raceWithTimeout` for why it is a bare timer, not a timeout frame.
473
+ await this.#withRequestTimeout(crawlingContext, this.#basicPipeline.chain(this.contextPipeline).call(crawlingContext, (ctx) => this.handleRequest(ctx, source), (error) => this.#handleRequestFailure(error, crawlingContext, source))));
474
+ }
475
+ catch (error) {
476
+ // The internal timeout above fired, or `handleRequestFailure` threw a `SessionError` - both
477
+ // take the normal error flow. Anything else is fatal.
478
+ if (error instanceof SessionError || crawlingContext[timeoutExpiredKey]?.()) {
479
+ const unwrappedError = this.#unwrapError(error);
480
+ await this.requestFunctionErrorHandler(unwrappedError, crawlingContext, this.requestManager);
481
+ if (!this.#errorAbsolvesSession(unwrappedError)) {
482
+ crawlingContext.session?.markBad();
483
+ }
484
+ return;
485
+ }
486
+ throw this.#unwrapError(error);
487
+ }
488
+ },
360
489
  isTaskReadyFunction: async () => {
361
490
  if (isMaxPagesExceeded()) {
362
- if (this.shouldLogMaxProcessedRequestsExceeded) {
363
- log.info('Crawler reached the maxRequestsPerCrawl limit of ' +
364
- `${this.maxRequestsPerCrawl} requests and will shut down soon. Requests that are in progress will be allowed to finish.`);
365
- this.shouldLogMaxProcessedRequestsExceeded = false;
366
- }
491
+ this.#liveRun?.logOnce('shuttingDown', 'Crawler reached the maxRequestsPerCrawl limit of ' +
492
+ `${this.#maxRequestsPerCrawl} requests and will shut down soon. Requests that are in progress will be allowed to finish.`);
367
493
  return false;
368
494
  }
369
- return isTaskReadyFunction ? await isTaskReadyFunction() : await this._isTaskReadyFunction();
495
+ if (this.#run?.stopRequested) {
496
+ this.#liveRun?.logOnce('shuttingDown', 'No new requests are allowed because the `stop()` method has been called. ' +
497
+ 'Ongoing requests will be allowed to complete.');
498
+ return false;
499
+ }
500
+ return isTaskReadyFunction ? await isTaskReadyFunction() : await this.#isTaskReadyFunction();
370
501
  },
371
502
  isFinishedFunction: async () => {
372
503
  if (isMaxPagesExceeded()) {
373
- log.info(`Earlier, the crawler reached the maxRequestsPerCrawl limit of ${this.maxRequestsPerCrawl} requests ` +
504
+ this.log.info(`Earlier, the crawler reached the maxRequestsPerCrawl limit of ${this.#maxRequestsPerCrawl} requests ` +
374
505
  'and all requests that were in progress at that time have now finished. ' +
375
- `In total, the crawler processed ${this.handledRequestsCount} requests and will shut down.`);
506
+ `In total, the crawler processed ${this.#handledRequestsCount} requests and will shut down.`);
376
507
  return true;
377
508
  }
509
+ if (this.#run?.stopRequested) {
510
+ this.log.info('The crawler has finished all the remaining ongoing requests and will shut down now.');
511
+ return true;
512
+ }
513
+ // `maybeFinish()` calls this only once nothing is in flight (`autoscaled-pool.ts`) - the point
514
+ // where a crawl that cannot progress becomes distinguishable from one that is merely waiting,
515
+ // and the only place where throwing does not abandon requests mid-processing.
516
+ const state = await this.requestManager?.checkReadiness();
517
+ // Under `keepAlive`, outliving a domain that will not let us through is the whole point.
518
+ if (state?.status === 'stalled' && !keepAlive) {
519
+ throw new PersistentRateLimitError(`Giving up: ${state.reason}`);
520
+ }
378
521
  const isFinished = isFinishedFunction
379
522
  ? await isFinishedFunction()
380
- : await this._defaultIsFinishedFunction();
523
+ : state === undefined || state.status === 'finished';
381
524
  if (isFinished) {
382
525
  const reason = isFinishedFunction
383
526
  ? "Crawler's custom isFinishedFunction() returned true, the crawler will shut down."
384
527
  : 'All requests from the queue have been processed, the crawler will shut down.';
385
- log.info(reason);
528
+ this.log.info(reason);
386
529
  }
387
530
  return isFinished;
388
531
  },
389
- log,
532
+ log: this.log,
390
533
  };
391
- this.autoscaledPoolOptions = { ...autoscaledPoolOptions, ...basicCrawlerAutoscaledPoolConfiguration };
534
+ this.#taskLoopOptions = { ...taskLoopOptions, ...crawlerOwnedTaskLoopConfiguration };
535
+ this.#resolveConcurrencySystem = () => OwnedOrInjected.resolve(concurrencySystem, () => this.createDefaultConcurrencySystem({
536
+ minConcurrency,
537
+ maxConcurrency,
538
+ maxTasksPerMinute: maxRequestsPerMinute,
539
+ // Spread conditionally - an explicit `undefined` would clobber a subclass default, see
540
+ // `HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS`.
541
+ ...(initialConcurrency !== undefined && { desiredConcurrency: initialConcurrency }),
542
+ log: this.log,
543
+ }));
392
544
  }
393
545
  finally {
394
546
  serviceLocatorScope.exitScope();
395
547
  }
396
548
  }
549
+ /**
550
+ * Builds the crawler-owned default {@link ConcurrencySystem} from the resolved
551
+ * `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute` shortcuts. Not called when a
552
+ * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} was injected.
553
+ *
554
+ * Subclasses may override this to tune the default system (e.g. {@link HttpCrawler} raises the starting
555
+ * concurrency and relaxes the event loop signal) while still honouring the user's shortcuts.
556
+ */
557
+ createDefaultConcurrencySystem(options) {
558
+ return new ConcurrencySystem(options);
559
+ }
560
+ /**
561
+ * Determines if the given HTTP status code is an error status code given
562
+ * the default behaviour and user-set preferences.
563
+ * @param status
564
+ * @returns `true` if the status code is considered an error, `false` otherwise
565
+ */
566
+ isErrorStatusCode(status) {
567
+ const excludeError = this.#ignoreHttpErrorStatusCodes.has(status);
568
+ const includeError = this.additionalHttpErrorStatusCodes.has(status);
569
+ return (status >= 500 && !excludeError) || includeError;
570
+ }
571
+ /**
572
+ * Builds the basic context pipeline that transforms `{ request }` into a full `CrawlingContext`.
573
+ * This handles base context creation, session resolution, and context helpers.
574
+ */
575
+ #buildBasicContextPipeline() {
576
+ return ContextPipeline.create()
577
+ .compose(this.#checkRobotsTxt.bind(this))
578
+ .compose(this.#createBaseContext.bind(this))
579
+ .compose(this.#resolveSession.bind(this))
580
+ .compose(this.#createContextHelpers.bind(this));
581
+ }
582
+ async #checkRobotsTxt({ request }) {
583
+ if (!(await this.#isAllowedBasedOnRobotsTxtFile(request.url))) {
584
+ this.log.warning(`Skipping request ${request.url} (${request.id}) because it is disallowed based on robots.txt`);
585
+ throw new ContextPipelineInterruptedError('robotsTxt');
586
+ }
587
+ return {};
588
+ }
589
+ #createBaseContext(context, onCleanup) {
590
+ return {
591
+ id: cryptoRandomObjectId(10),
592
+ log: this.log,
593
+ pushData: this.pushData.bind(this),
594
+ useState: this.useState.bind(this),
595
+ getKeyValueStore: async (identifier) => KeyValueStore.open(identifier),
596
+ registerDeferredCleanup: (cleanup) => {
597
+ onCleanup(async () => {
598
+ try {
599
+ await cleanup();
600
+ }
601
+ catch (error) {
602
+ this.log.debug('Error in deferred cleanup', { error });
603
+ }
604
+ });
605
+ },
606
+ afterStorageCommit: (callback) => {
607
+ const transaction = currentStorageTransaction();
608
+ if (transaction === undefined) {
609
+ throw new Error('afterStorageCommit() needs an active storage transaction, and there is none. ' +
610
+ "With `transactionalStorage: false` the request's writes are applied as they are " +
611
+ 'made, so they throw at the call site instead of at commit time.');
612
+ }
613
+ transaction.afterCommit(callback);
614
+ },
615
+ extendTimeout: (secs) => {
616
+ const extraMillis = secs * 1000;
617
+ // the current `addTimeoutToPromise` window (the request handler, or a navigation hook)...
618
+ extendTimeout(extraMillis);
619
+ // ...the internal timeout around the whole request, which is not an `addTimeoutToPromise` frame...
620
+ context[extendTimeoutKey]?.(extraMillis);
621
+ // ...and, when called from within the navigation phase, its shared window, so extending a hook
622
+ // extends the whole navigation budget rather than just that hook's step.
623
+ if (context[navigationDeadlineKey] !== undefined) {
624
+ context[navigationDeadlineKey] += extraMillis;
625
+ }
626
+ // Extension failure must not fail the request that asked for more time.
627
+ this.requestManager?.extendRequestProcessingTime?.(context.request, secs)?.catch((error) => {
628
+ this.log.debug('Extending the request processing time failed', { url: context.request.url, error });
629
+ });
630
+ },
631
+ skipRequest: (message) => {
632
+ this.log.debug(`Skipping request ${context.request.url} (${context.request.id})${message ? `: ${message}` : ''}`);
633
+ throw new ContextPipelineInterruptedError('manual', message);
634
+ },
635
+ };
636
+ }
637
+ /**
638
+ * Fetches the next request and rebuilds it as a {@link CrawlingRequest}. A copy rather than the manager's
639
+ * instance: managers key on `uniqueKey`/`id`, and it is this copy that flows back into `reclaimRequest` /
640
+ * `markRequestAsHandled`, so nothing observes the difference.
641
+ */
642
+ async #resolveRequest() {
643
+ const fetched = await this.#timeoutAndRetry(this.#fetchNextRequest.bind(this), this.internalTimeoutMillis, `Fetching next request timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
644
+ if (!fetched) {
645
+ return null;
646
+ }
647
+ const request = CrawlingRequest.fromSchema(fetched);
648
+ // Reset loadedUrl so an old one is not carried over to retries.
649
+ request.loadedUrl = undefined;
650
+ return request;
651
+ }
652
+ async #resolveSession({ request }) {
653
+ const session = await this.#timeoutAndRetry(async () => {
654
+ const existingSession = await this.sessionPool.getSession(request.sessionId);
655
+ if (!existingSession) {
656
+ throw new MissingSessionError(request.sessionId);
657
+ }
658
+ return existingSession;
659
+ }, this.internalTimeoutMillis, `Fetching session timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
660
+ return { session, proxyInfo: session?.proxyInfo };
661
+ }
662
+ async #createContextHelpers({ request, session }) {
663
+ const addRequests = async (requests, options = {}) => {
664
+ const newCrawlDepth = request.crawlDepth + 1;
665
+ const requestsGenerator = this.addCrawlDepthRequestGenerator(requests, newCrawlDepth);
666
+ return await this.addRequests(requestsGenerator, options);
667
+ };
668
+ const sendRequest = createSendRequest(this.httpClient, request, session);
669
+ return { addRequests, sendRequest };
670
+ }
671
+ #buildFinalContextPipeline() {
672
+ const subclassPipeline = (this.#contextPipelineOptions.contextPipelineBuilder?.() ??
673
+ ContextPipeline.create());
674
+ // `extendContext` runs *before* the subclass navigation pipeline (which includes the
675
+ // pre/post-navigation hooks). This makes the extension visible to those hooks and to the
676
+ // request handler alike. The trade-off is that `extendContext` cannot access
677
+ // navigation-dependent context members (e.g. `page`, `response`, `$`, `body`), as those
678
+ // don't exist yet at this point in the pipeline.
679
+ // The `extendContext` output (`ContextExtension`) is carried through the subclass pipeline at
680
+ // runtime (the pipeline copies each middleware's returned members onto the shared context), but
681
+ // TypeScript cannot express that `Context` transitively includes `ContextExtension` here. The
682
+ // casts below are sound because `buildFinalContextPipeline` is declared to return the fully
683
+ // resolved `ExtendedContext` (= `Context & ContextExtension`).
684
+ const { extendContext } = this.#contextPipelineOptions;
685
+ let contextPipeline;
686
+ if (extendContext !== undefined) {
687
+ contextPipeline = ContextPipeline.create()
688
+ .compose(extendContext)
689
+ .chain(subclassPipeline);
690
+ }
691
+ else {
692
+ contextPipeline = subclassPipeline;
693
+ }
694
+ contextPipeline = contextPipeline.compose(async (context) => {
695
+ const { request } = context;
696
+ if (request && !this.#requestMatchesEnqueueStrategy(request)) {
697
+ // eslint-disable-next-line dot-notation
698
+ const message = `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`;
699
+ this.log.debug(message);
700
+ throw new ContextPipelineInterruptedError('redirect');
701
+ }
702
+ return context;
703
+ });
704
+ return contextPipeline;
705
+ }
397
706
  /**
398
707
  * Checks if the given error is a proxy error by comparing its message to a list of known proxy error messages.
399
708
  * Used for retrying requests that failed due to proxy errors.
@@ -401,158 +710,212 @@ export class BasicCrawler {
401
710
  * @param error The error to check.
402
711
  */
403
712
  isProxyError(error) {
404
- return ROTATE_PROXY_ERRORS.some((x) => this._getMessageFromError(error)?.includes(x));
713
+ return ROTATE_PROXY_ERRORS.some((x) => this.getMessageFromError(error).includes(x));
405
714
  }
406
715
  /**
716
+ * Sets the status message for the current crawler run.
717
+ *
407
718
  * This method is periodically called by the crawler, every `statusMessageLoggingInterval` seconds.
719
+ *
720
+ * The message is logged and broadcast via the {@link EventType.STATUS_MESSAGE|`statusMessage`}
721
+ * event. Integrations such as the Apify SDK subscribe to that event and forward the message to
722
+ * their status-reporting backend (e.g. the Apify platform).
408
723
  */
409
- async setStatusMessage(message, options = {}) {
724
+ setStatusMessage(message, options = {}) {
410
725
  const data = options.isStatusMessageTerminal != null ? { terminal: options.isStatusMessageTerminal } : undefined;
411
- this.log.internal(LogLevel[options.level ?? 'DEBUG'], message, data);
412
- const client = serviceLocator.getStorageClient();
413
- if (!client.setStatusMessage) {
414
- return;
415
- }
416
- // just to be sure, this should be fast
417
- await addTimeoutToPromise(async () => client.setStatusMessage(message, options), 1000, 'Setting status message timed out after 1s').catch((e) => this.log.debug(e.message));
726
+ // Each allowed level has its own method on the logger, so this goes through them rather than through
727
+ // `logWithLevel`, which is abstract and therefore cannot be instrumented.
728
+ this.log[{ DEBUG: 'debug', INFO: 'info', WARNING: 'warning', ERROR: 'error' }[options.level ?? 'DEBUG']](message, data);
729
+ // Broadcast the status message through the event system. Consumers (e.g. the Apify SDK) can
730
+ // subscribe to `EventType.STATUS_MESSAGE` and propagate it to their status-reporting backend.
731
+ // Setting the status message is not a storage concern, so we intentionally don't route it
732
+ // through the storage client anymore.
733
+ serviceLocator.getEventManager().emit(EventType.STATUS_MESSAGE, {
734
+ crawlerId: this.#identity.id,
735
+ message,
736
+ isStatusMessageTerminal: options.isStatusMessageTerminal,
737
+ level: options.level,
738
+ });
418
739
  }
419
- getPeriodicLogger() {
420
- let previousState = { ...this.stats.state };
740
+ #getPeriodicLogger() {
741
+ let previousState = { ...this.statistics.state };
421
742
  const getOperationMode = () => {
422
- const { requestsFailed } = this.stats.state;
743
+ const { requestsFailed } = this.statistics.state;
423
744
  const { requestsFailed: previousRequestsFailed } = previousState;
424
- previousState = { ...this.stats.state };
425
- if (requestsFailed - previousRequestsFailed > 0) {
426
- return 'ERROR';
745
+ previousState = { ...this.statistics.state };
746
+ const failedDelta = requestsFailed - previousRequestsFailed;
747
+ if (failedDelta > 0) {
748
+ return { mode: 'ERROR', failedDelta };
427
749
  }
428
- return 'REGULAR';
750
+ return { mode: 'REGULAR', failedDelta: 0 };
429
751
  };
430
752
  const log = async () => {
431
- const operationMode = getOperationMode();
753
+ const { mode: operationMode, failedDelta } = getOperationMode();
432
754
  let message;
433
755
  if (operationMode === 'ERROR') {
434
- message = `Experiencing problems, ${this.stats.state.requestsFailed - previousState.requestsFailed || this.stats.state.requestsFailed} failed requests in the past ${this.statusMessageLoggingInterval} seconds.`;
756
+ message = `Experiencing problems, ${failedDelta} failed requests in the past ${this.#statusMessageLoggingInterval} seconds.`;
435
757
  }
436
758
  else {
437
- const total = this.requestManager?.getTotalCount();
438
- message = `Crawled ${this.stats.state.requestsFinished}${total ? `/${total}` : ''} pages, ${this.stats.state.requestsFailed} failed requests, desired concurrency ${this.autoscaledPool?.desiredConcurrency ?? 0}.`;
759
+ const total = await this.requestManager?.getTotalCount();
760
+ message = `Crawled ${this.statistics.state.requestsSucceeded}${total ? `/${total}` : ''} pages, ${this.statistics.state.requestsFailed} failed requests, desired concurrency ${this.concurrencySystem?.desiredConcurrency ?? 0}.`;
439
761
  }
440
- if (this.statusMessageCallback) {
441
- await this.statusMessageCallback({
762
+ if (this.#statusMessageCallback) {
763
+ await this.#statusMessageCallback({
442
764
  crawler: this,
443
- state: this.stats.state,
765
+ state: this.statistics.state,
444
766
  previousState,
445
767
  message,
446
768
  });
447
769
  return;
448
770
  }
449
- await this.setStatusMessage(message);
771
+ this.setStatusMessage(message);
450
772
  };
451
- const interval = setInterval(log, this.statusMessageLoggingInterval * 1e3);
773
+ const interval = setInterval(log, this.#statusMessageLoggingInterval * 1e3);
452
774
  return { log, stop: () => clearInterval(interval) };
453
775
  }
454
776
  /**
455
- * Runs the crawler. Returns a promise that resolves once all the requests are processed
456
- * and `autoscaledPool.isFinished` returns `true`.
777
+ * Runs the crawler. Returns a promise that resolves once every request has been processed and the crawler's
778
+ * finished-check ({@link BasicCrawlerOptions.taskLoopOptions|`taskLoopOptions.isFinishedFunction`}, or the
779
+ * default "the request manager is empty") reports that the crawl is over.
457
780
  *
458
781
  * We can use the `requests` parameter to enqueue the initial requests — it is a shortcut for
459
782
  * running {@link BasicCrawler.addRequests|`crawler.addRequests()`} before {@link BasicCrawler.run|`crawler.run()`}.
460
783
  *
784
+ * Calling `run()` again on the same instance keeps crawling the same request manager - requests the previous
785
+ * run handled (a failed one counts as handled) are not processed again. Purge the queue or open a fresh one
786
+ * if that is what you want.
787
+ *
461
788
  * @param [requests] The requests to add.
462
- * @param [options] Options for the request queue.
789
+ * @param [options] Options for adding the initial requests.
463
790
  */
464
791
  async run(requests, options) {
465
- if (this.running) {
466
- throw new Error('This crawler instance is already running, you can add more requests to it via `crawler.addRequests()`.');
467
- }
468
- const { purgeRequestQueue = true, ...addRequestsOptions } = options ?? {};
469
- if (this.hasFinishedBefore) {
470
- // When executing the run method for the second time explicitly,
471
- // we need to purge the default RQ to allow processing the same requests again - this is important so users can
472
- // pass in failed requests back to the `crawler.run()`, otherwise they would be considered as handled and
473
- // ignored - as a failed requests is still handled.
474
- if (this.requestQueue?.name === 'default' && purgeRequestQueue) {
475
- await this.requestQueue.drop();
476
- this.requestQueue = await this._getRequestQueue();
477
- this.requestManager = undefined;
478
- await this.initializeRequestManager();
479
- this.handledRequestsCount = 0; // This would've been reset by this._init() further down below, but at that point `handledRequestsCount` could prevent `addRequests` from adding the initial requests
792
+ // A crawl is the top level of its own transaction and timeout scope, not a participant in the caller's.
793
+ return withDirectStorageAccess(async () => timeoutStorage.exit(async () => {
794
+ if (this.running) {
795
+ throw new Error('This crawler instance is already running, you can add more requests to it via `crawler.addRequests()`.');
480
796
  }
481
- this.stats.reset();
482
- await this.stats.resetStore();
483
- await this.sessionPool?.resetStore();
484
- }
485
- this.running = true;
486
- this.shouldLogMaxProcessedRequestsExceeded = true;
487
- this.shouldLogMaxEnqueuedRequestsExceeded = true;
488
- await purgeDefaultStorages({
489
- onlyPurgeOnce: true,
490
- client: serviceLocator.getStorageClient(),
491
- config: serviceLocator.getConfiguration(),
492
- });
493
- if (requests) {
494
- await this.addRequests(requests, addRequestsOptions);
495
- }
496
- await this._init();
497
- await this.stats.startCapturing();
498
- const periodicLogger = this.getPeriodicLogger();
499
- // Don't await, we don't want to block the execution
500
- void this.setStatusMessage('Starting the crawler.', { level: 'INFO' });
501
- const sigintHandler = async () => {
502
- this.log.warning('Pausing... Press CTRL+C again to force exit. To resume, do: CRAWLEE_PURGE_ON_START=0 npm start');
503
- await this._pauseOnMigration();
504
- await this.autoscaledPool.abort();
505
- };
506
- // Attach a listener to handle migration and aborting events gracefully.
507
- const boundPauseOnMigration = this._pauseOnMigration.bind(this);
508
- process.once('SIGINT', sigintHandler);
509
- const eventManager = serviceLocator.getEventManager();
510
- eventManager.on("migrating" /* EventType.MIGRATING */, boundPauseOnMigration);
511
- eventManager.on("aborting" /* EventType.ABORTING */, boundPauseOnMigration);
512
- let stats = {};
513
- try {
514
- await this.autoscaledPool.run();
515
- }
516
- finally {
517
- await this.teardown();
518
- await this.stats.stopCapturing();
519
- process.off('SIGINT', sigintHandler);
520
- eventManager.off("migrating" /* EventType.MIGRATING */, boundPauseOnMigration);
521
- eventManager.off("aborting" /* EventType.ABORTING */, boundPauseOnMigration);
522
- const finalStats = this.stats.calculate();
523
- stats = {
524
- requestsFinished: this.stats.state.requestsFinished,
525
- requestsFailed: this.stats.state.requestsFailed,
526
- retryHistogram: this.stats.requestRetryHistogram,
527
- ...finalStats,
528
- };
529
- this.log.info('Final request statistics:', stats);
530
- if (this.stats.errorTracker.total !== 0) {
531
- const prettify = ([count, info]) => `${count}x: ${info.at(-1).trim()} (${info[0]})`;
532
- this.log.info(`Error analysis:`, {
533
- totalErrors: this.stats.errorTracker.total,
534
- uniqueErrors: this.stats.errorTracker.getUniqueErrorCount(),
535
- mostCommonErrors: this.stats.errorTracker.getMostPopularErrors(3).map(prettify),
797
+ // An owned governor is rebuilt for every run, so it always starts from a clean slate — stale
798
+ // resource snapshots or a previous run's scaled desired concurrency would otherwise distort this
799
+ // run's scaling. An injected one is long-lived and its lifecycle belongs to the caller.
800
+ this.#concurrencySystem = this.#resolveConcurrencySystem();
801
+ const run = new CrawlerRun({
802
+ log: this.log,
803
+ concurrencySystem: this.#concurrencySystem.value,
804
+ taskLoopOptions: this.#taskLoopOptions,
805
+ consumer: this.#identity,
806
+ });
807
+ this.#run = run;
808
+ try {
809
+ if (this.#hasFinishedBefore) {
810
+ // A supplied statistics instance keeps whatever state it was handed - only wipe a default we built.
811
+ await this.#statisticsDep.ifOwned(async (stats) => {
812
+ stats.reset();
813
+ await stats.resetStore();
814
+ });
815
+ // A session pool the crawler built follows the same blank-slate rule as its statistics.
816
+ await this.#sessionPoolDep.ifOwned(async (pool) => {
817
+ pool.reset();
818
+ await pool.resetStore();
819
+ });
820
+ }
821
+ await purgeDefaultStorages({
822
+ onlyPurgeOnce: true,
823
+ storageBackend: serviceLocator.getStorageBackend(),
824
+ configuration: serviceLocator.getConfiguration(),
825
+ });
826
+ if (requests) {
827
+ await this.addRequests(requests, options);
828
+ }
829
+ // An injected governor is the caller's to run; an owned one is started here and stopped again
830
+ // below if the startup fails. Either way the task loop rejects one that is not running.
831
+ await this.#concurrencySystem.ifOwned((system) => system.start());
832
+ await this.init();
833
+ await this.statistics.startCapturing();
834
+ }
835
+ catch (error) {
836
+ // Clean up here before propagating, otherwise a failed startup would leave the process hanging.
837
+ await this.teardown().catch((teardownError) => {
838
+ this.log.exception(teardownError, 'Cleaning up after a failed crawler startup failed.');
536
839
  });
840
+ // The run never began, so let the instance be run again instead of leaving it wedged as `running`.
841
+ await run.finish();
842
+ await this.#concurrencySystem.ifOwned((system) => system.stop());
843
+ throw error;
844
+ }
845
+ const periodicLogger = this.#getPeriodicLogger();
846
+ this.setStatusMessage('Starting the crawler.', { level: 'INFO' });
847
+ const sigintHandler = async () => {
848
+ this.log.warning('Pausing... Press CTRL+C again to force exit. To resume, do: CRAWLEE_PURGE_ON_START=0 npm start');
849
+ await this.#pauseOnMigration();
850
+ await run.abort();
851
+ };
852
+ // Attach a listener to handle migration and aborting events gracefully.
853
+ const boundPauseOnMigration = this.#pauseOnMigration.bind(this);
854
+ process.once('SIGINT', sigintHandler);
855
+ const eventManager = serviceLocator.getEventManager();
856
+ eventManager.on(EventType.MIGRATING, boundPauseOnMigration);
857
+ eventManager.on(EventType.ABORTING, boundPauseOnMigration);
858
+ let stats = {};
859
+ try {
860
+ await run.dispatchRequests();
537
861
  }
538
- const client = serviceLocator.getStorageClient();
539
- if (client.teardown) {
540
- let finished = false;
541
- setTimeout(() => {
542
- if (!finished) {
543
- this.log.info('Waiting for the storage to write its state to file system.');
862
+ finally {
863
+ await this.statistics.stopCapturing();
864
+ await this.teardown();
865
+ process.off('SIGINT', sigintHandler);
866
+ eventManager.off(EventType.MIGRATING, boundPauseOnMigration);
867
+ eventManager.off(EventType.ABORTING, boundPauseOnMigration);
868
+ const finalStats = this.statistics.calculate();
869
+ stats = {
870
+ requestsSucceeded: this.statistics.state.requestsSucceeded,
871
+ requestsFailed: this.statistics.state.requestsFailed,
872
+ retryHistogram: this.statistics.requestRetryHistogram,
873
+ ...finalStats,
874
+ };
875
+ this.log.info('Final request statistics:', stats);
876
+ // A crawl that did nothing while the manager holds only handled requests is a mistake whoever
877
+ // handled them - this run, another crawler on the same queue, or a previous process. Starting
878
+ // against handled requests is not: that is what resuming a crawl looks like.
879
+ if (stats.requestsSucceeded + stats.requestsFailed === 0) {
880
+ // Never let the diagnostic itself break the run.
881
+ const alreadyHandled = (await this.requestManager?.getHandledCount().catch(() => 0)) ?? 0;
882
+ if (alreadyHandled > 0) {
883
+ this.log.warningOnce('This crawl processed no requests - the request manager holds ' +
884
+ `${alreadyHandled} request${alreadyHandled === 1 ? '' : 's'}, all of them ` +
885
+ 'already handled, and a failed request counts as handled too. Nothing ' +
886
+ 'empties a queue between runs, so to crawl them again, purge it ' +
887
+ '(`await queue.purge()`) or use a fresh one (e.g. ' +
888
+ '`RequestQueue.open({ alias: "second-run" })`) with a freshly created ' +
889
+ 'crawler instance.');
544
890
  }
545
- }, 1000);
546
- await client.teardown();
547
- finished = true;
891
+ }
892
+ if (this.statistics.errorTracker.total !== 0) {
893
+ const prettify = ([count, info]) => `${count}x: ${info.at(-1).trim()} (${info[0]})`;
894
+ this.log.info(`Error analysis:`, {
895
+ totalErrors: this.statistics.errorTracker.total,
896
+ uniqueErrors: this.statistics.errorTracker.getUniqueErrorCount(),
897
+ mostCommonErrors: this.statistics.errorTracker.getMostPopularErrors(3).map(prettify),
898
+ });
899
+ }
900
+ const client = serviceLocator.getStorageBackend();
901
+ if (client.teardown) {
902
+ let finished = false;
903
+ setTimeout(() => {
904
+ if (!finished) {
905
+ this.log.info('Waiting for the storage to write its state to file system.');
906
+ }
907
+ }, 1000);
908
+ await client.teardown();
909
+ finished = true;
910
+ }
911
+ periodicLogger.stop();
912
+ this.setStatusMessage(`Finished! Total ${this.statistics.state.requestsSucceeded + this.statistics.state.requestsFailed} requests: ${this.statistics.state.requestsSucceeded} succeeded, ${this.statistics.state.requestsFailed} failed.`, { isStatusMessageTerminal: true, level: 'INFO' });
913
+ await run.finish();
914
+ await this.#concurrencySystem.ifOwned((system) => system.stop());
915
+ this.#hasFinishedBefore = true;
548
916
  }
549
- periodicLogger.stop();
550
- // Don't await, we don't want to block the execution
551
- void this.setStatusMessage(`Finished! Total ${this.stats.state.requestsFinished + this.stats.state.requestsFailed} requests: ${this.stats.state.requestsFinished} succeeded, ${this.stats.state.requestsFailed} failed.`, { isStatusMessageTerminal: true, level: 'INFO' });
552
- this.running = false;
553
- this.hasFinishedBefore = true;
554
- }
555
- return stats;
917
+ return stats;
918
+ }));
556
919
  }
557
920
  /**
558
921
  * Gracefully stops the current run of the crawler.
@@ -561,71 +924,160 @@ export class BasicCrawler {
561
924
  *
562
925
  * To stop the crawler immediately, use {@link BasicCrawler.teardown|`crawler.teardown()`} instead.
563
926
  */
564
- stop(message = 'The crawler has been gracefully stopped.') {
565
- // Gracefully starve the this.autoscaledPool, so it doesn't start new tasks. Resolves once the pool is cleared.
566
- this.autoscaledPool
567
- ?.pause()
568
- // Resolves the `autoscaledPool.run()` promise in the `BasicCrawler.run()` method. Since the pool is already paused, it resolves immediately and doesn't kill any tasks.
569
- .then(async () => this.autoscaledPool?.abort())
570
- .then(() => this.log.info(message))
571
- .catch((err) => {
572
- this.log.error('An error occurred when stopping the crawler:', err);
573
- });
927
+ stop(reason = 'The crawler has been gracefully stopped.') {
928
+ const run = this.#liveRun;
929
+ if (run === undefined) {
930
+ this.log.warning('Cannot stop a crawler that is not running.');
931
+ return;
932
+ }
933
+ run.stop(reason);
934
+ }
935
+ /**
936
+ * Stops dispatching new requests, letting the in-progress ones finish. Resolves once they have settled, or rejects
937
+ * after `timeoutSecs` if they take too long. Unlike {@link BasicCrawler.stop|`stop()`}, this does not end the
938
+ * run — {@link BasicCrawler.run|`run()`} stays pending until {@link BasicCrawler.resume|`resume()`}.
939
+ *
940
+ * > *NOTE:* The {@link BasicCrawler.concurrencySystem|concurrency system} keeps monitoring and autoscaling
941
+ * throughout, since a shared one may still be serving other crawlers.
942
+ */
943
+ async pause(timeoutSecs) {
944
+ const run = this.#dispatchingRun;
945
+ if (run === undefined) {
946
+ this.log.warning('Cannot pause a crawler that is not running.');
947
+ return;
948
+ }
949
+ await run.pause(timeoutSecs);
950
+ }
951
+ /**
952
+ * Resumes a run suspended with {@link BasicCrawler.pause|`pause()`}, letting the crawler dispatch requests
953
+ * again. A no-op on a crawler that is not paused.
954
+ */
955
+ resume() {
956
+ const run = this.#dispatchingRun;
957
+ if (run === undefined) {
958
+ this.log.warning('Cannot resume a crawler that is not running.');
959
+ return;
960
+ }
961
+ run.resume();
962
+ }
963
+ /**
964
+ * Returns the crawler's {@link IRequestManager|request manager}, opening the default {@link RequestQueue}
965
+ * if none has been configured or opened yet.
966
+ */
967
+ async getRequestManager() {
968
+ if (!this.#requestManager) {
969
+ this.#requestManager = await this.#openOwnedRequestQueue();
970
+ }
971
+ // Apply the processing-time hint here (an async lifecycle point) rather than in the constructor,
972
+ // now that `setExpectedRequestProcessingTime` is async. The hint is raise-only and idempotent,
973
+ // but guard so we do not re-issue it on every call.
974
+ if (!this.#requestManagerTimeoutsApplied) {
975
+ this.#requestManagerTimeoutsApplied = true;
976
+ await this.#applyRequestManagerTimeouts(this.#requestManager);
977
+ }
978
+ return this.#requestManager;
574
979
  }
980
+ /**
981
+ * @deprecated Use {@link BasicCrawler.getRequestManager|`getRequestManager()`} instead. This returns the
982
+ * crawler's request manager, which is no longer guaranteed to be a {@link RequestQueue}.
983
+ */
575
984
  async getRequestQueue() {
576
- if (!this.requestQueue && this.requestList) {
577
- this.log.warningOnce('When using RequestList and RequestQueue at the same time, you should instantiate both explicitly and provide them in the crawler options, to ensure correctly handled restarts of the crawler.');
985
+ return this.getRequestManager();
986
+ }
987
+ /**
988
+ * Opens the default {@link RequestQueue} — the crawler's own, read from when the caller supplied nothing.
989
+ */
990
+ async #openOwnedRequestQueue() {
991
+ // The first crawler instance uses the default queue (null identifier);
992
+ // subsequent instances get their own queue via a unique alias so they don't collide.
993
+ const identifier = this.#identity.instanceIndex === 0 ? null : { alias: `__default_${this.#identity.id}__` };
994
+ return RequestQueue.open(identifier, { configuration: serviceLocator.getConfiguration() });
995
+ }
996
+ /**
997
+ * Tells a request manager how long we expect to hold a fetched request, so that one backed by a
998
+ * locking storage backend keeps it reserved for slightly longer than the request handler timeout
999
+ * (with some padding for overhead), but never for less than a minute. This prevents a long-running
1000
+ * request from being handed out a second time while it is still being processed — and it works
1001
+ * regardless of whether the manager is a plain {@link RequestQueue} or a `RequestManagerTandem`.
1002
+ */
1003
+ async #applyRequestManagerTimeouts(requestManager) {
1004
+ // A router route may hold a request for longer than the crawler's own timeout, and we cannot know
1005
+ // which routes a run will hit, so reserve for the longest one any route asked for. The hint is
1006
+ // raise-only, so erring high here is safe.
1007
+ const maxRouteTimeoutSecs = this.requestHandler.getMaxTimeoutSecs?.() ?? 0;
1008
+ const handlerTimeoutSecs = Math.max(this.#requestHandlerTimeoutMillis / 1000, maxRouteTimeoutSecs);
1009
+ await requestManager.setExpectedRequestProcessingTime?.(Math.max(handlerTimeoutSecs + 5, 60));
1010
+ }
1011
+ /**
1012
+ * Validates a request source's `userData` against the {@link RouteSchemas|Standard Schema} registered
1013
+ * for its label on the crawler's schema-router (if any), throwing a {@link RequestValidationError} on
1014
+ * mismatch. A no-op when the user's request handler is not a schema-router, or no schema is registered for
1015
+ * the request's label. Applied by the crawler on the add paths it owns — `crawler.addRequests`,
1016
+ * `crawler.run`, `context.addRequests` and `context.enqueueLinks`.
1017
+ */
1018
+ async #validateRequestUserData(source) {
1019
+ if (typeof source === 'string') {
1020
+ return;
578
1021
  }
579
- if (!this.requestQueue) {
580
- this.requestQueue = await this._getRequestQueue();
581
- this.requestManager = undefined;
1022
+ const getSchema = this.requestHandler.getSchema;
1023
+ if (typeof getSchema !== 'function') {
1024
+ return;
582
1025
  }
583
- if (!this.requestManager) {
584
- this.requestManager =
585
- this.requestList === undefined
586
- ? this.requestQueue
587
- : new RequestManagerTandem(this.requestList, this.requestQueue);
1026
+ // Resolve the label via its public accessors only — the top-level `label` of a `RequestOptions` or the
1027
+ // `Request.label` getter — rather than reaching into `userData`, where the request happens to store it.
1028
+ const target = source;
1029
+ const schema = getSchema(target.label);
1030
+ if (!schema) {
1031
+ return;
588
1032
  }
589
- return this.requestQueue;
1033
+ // Store the parsed value rather than the raw input, so the queue holds the same coerced `userData` the
1034
+ // handler will see. Assigning through a `Request` instance's setter keeps its internal `__crawlee` meta.
1035
+ target.userData = await validateUserData(target.label, schema, target.userData ?? {});
590
1036
  }
591
1037
  async useState(defaultValue = {}) {
592
- const kvs = await KeyValueStore.open(null, { config: serviceLocator.getConfiguration() });
593
- if (this.hasExplicitId) {
594
- const stateKey = `${BasicCrawler.CRAWLEE_STATE_KEY}_${this.crawlerId}`;
1038
+ const kvs = await KeyValueStore.open(null, { configuration: serviceLocator.getConfiguration() });
1039
+ if (this.#identity.hasExplicitId) {
1040
+ const stateKey = `${BasicCrawler.#CRAWLEE_STATE_KEY}_${this.#identity.id}`;
595
1041
  return kvs.getAutoSavedValue(stateKey, defaultValue);
596
1042
  }
597
- BasicCrawler.useStateCrawlerIds.add(this.crawlerId);
598
- if (BasicCrawler.useStateCrawlerIds.size > 1) {
599
- defaultLog.warningOnce('Multiple crawler instances are calling useState() without an explicit `id` option. \n' +
1043
+ BasicCrawler.#useStateAnonymousIndices.add(this.#identity.instanceIndex);
1044
+ if (BasicCrawler.#useStateAnonymousIndices.size > 1) {
1045
+ serviceLocator
1046
+ .getLogger()
1047
+ .warningOnce('Multiple crawler instances are calling useState() without an explicit `id` option. \n' +
600
1048
  'This means they will share the same state object, which is likely unintended. \n' +
601
1049
  'To fix this, provide a unique `id` option to each crawler instance. \n' +
602
1050
  'Example: new BasicCrawler({ id: "my-crawler-1", ... })');
603
1051
  }
604
- return kvs.getAutoSavedValue(BasicCrawler.CRAWLEE_STATE_KEY, defaultValue);
1052
+ return kvs.getAutoSavedValue(BasicCrawler.#CRAWLEE_STATE_KEY, defaultValue);
605
1053
  }
606
- get pendingRequestCountApproximation() {
607
- return this.requestManager?.getPendingCount() ?? 0;
1054
+ async #getPendingRequestCountApproximation() {
1055
+ return (await this.requestManager?.getPendingCount()) ?? 0;
608
1056
  }
609
- calculateEnqueuedRequestLimit(explicitLimit) {
610
- if (this.maxRequestsPerCrawl === undefined) {
1057
+ async #calculateEnqueuedRequestLimit(explicitLimit) {
1058
+ if (this.#maxRequestsPerCrawl === undefined) {
611
1059
  return explicitLimit;
612
1060
  }
613
- const limit = Math.max(0, this.maxRequestsPerCrawl - this.handledRequestsCount - this.pendingRequestCountApproximation);
1061
+ const limit = Math.max(0, this.#maxRequestsPerCrawl -
1062
+ this.#handledRequestsCount -
1063
+ (await this.#getPendingRequestCountApproximation()));
614
1064
  return Math.min(limit, explicitLimit ?? Infinity);
615
1065
  }
616
- async handleSkippedRequest(options) {
617
- if (options.reason === 'limit' && this.shouldLogMaxEnqueuedRequestsExceeded) {
618
- this.log.info('The number of requests enqueued by the crawler reached the maxRequestsPerCrawl limit of ' +
619
- `${this.maxRequestsPerCrawl} requests and no further requests will be added.`);
620
- this.shouldLogMaxEnqueuedRequestsExceeded = false;
621
- }
622
- if (options.reason === 'enqueueLimit') {
623
- const enqueuedRequestLimit = this.calculateEnqueuedRequestLimit();
624
- if (enqueuedRequestLimit === undefined || enqueuedRequestLimit !== 0) {
625
- this.log.info('The number of requests enqueued by the crawler reached the enqueueLinks limit.');
1066
+ async #handleSkippedRequest(options) {
1067
+ // A skipped request is a *successful* outcome, but the interrupt still unwinds through the
1068
+ // transaction scope, which rolls back - so the skip bookkeeping must write directly.
1069
+ await withDirectStorageAccess(async () => {
1070
+ // Enqueueing happens outside a run too - `addRequests()` before the first `run()` - so these dedupe
1071
+ // against the logger for the crawler's lifetime rather than per run.
1072
+ if (options.reason === 'limit') {
1073
+ this.log.info('The number of requests enqueued by the crawler reached the maxRequestsPerCrawl limit of ' +
1074
+ `${this.#maxRequestsPerCrawl} requests and no further requests will be added.`, undefined, { once: true });
626
1075
  }
627
- }
628
- await this.onSkippedRequest?.(options);
1076
+ if (options.reason === 'depth') {
1077
+ this.log.info(`The crawler reached the maxCrawlDepth limit of ${this.#maxCrawlDepth} and no further requests will be enqueued.`, undefined, { once: true });
1078
+ }
1079
+ await this.#onSkippedRequest?.(options);
1080
+ });
629
1081
  }
630
1082
  /**
631
1083
  * Adds requests to the queue in batches. By default, it will resolve after the initial batch is added, and continue
@@ -633,76 +1085,148 @@ export class BasicCrawler {
633
1085
  * the batches via `waitBetweenBatchesMillis`. If you want to wait for all batches to be added to the queue, you can use
634
1086
  * the `waitForAllRequestsToBeAdded` promise you get in the response object.
635
1087
  *
1088
+ * Optionally, the requests can be filtered using `include`/`exclude` glob or regexp patterns and an
1089
+ * enqueue `strategy` (both AND-ed together, same as {@link CrawlingContext.enqueueLinks|`enqueueLinks`}),
1090
+ * relative to `baseUrl`. Unlike `enqueueLinks`, there is no implicit "current page" to anchor the strategy
1091
+ * to, so `strategy` defaults to {@link EnqueueStrategy.All|`all`} here.
1092
+ *
636
1093
  * This is an alias for calling `addRequestsBatched()` on the implicit `RequestQueue` for this crawler instance.
637
1094
  *
638
1095
  * @param requests The requests to add
639
1096
  * @param options Options for the request queue
640
1097
  */
641
1098
  async addRequests(requests, options = {}) {
642
- await this.getRequestQueue();
643
- const requestLimit = this.calculateEnqueuedRequestLimit();
644
- const skippedBecauseOfRobots = new Set();
645
- const skippedBecauseOfLimit = new Set();
646
- const skippedBecauseOfMaxCrawlDepth = new Set();
647
- const isAllowedBasedOnRobotsTxtFile = this.isAllowedBasedOnRobotsTxtFile.bind(this);
648
- const maxCrawlDepth = this.maxCrawlDepth;
649
- ow(requests, ow.object
650
- .is((value) => isIterable(value) || isAsyncIterable(value))
651
- .message((value) => `Expected an iterable or async iterable, got ${getObjectType(value)}`));
1099
+ await this.getRequestManager();
1100
+ if (!isIterable(requests) && !isAsyncIterable(requests)) {
1101
+ throw new Error(`Expected an iterable or async iterable, got ${getObjectType(requests)}`);
1102
+ }
1103
+ parseArgument(options, addRequestsOptionsSchema, 'EnqueueUrlsOptions');
1104
+ // `label`/`userData` apply to every request this call produces, so a single upfront validation
1105
+ // against the label's schema covers them all and fails the whole call fast, rather than failing
1106
+ // lazily once the generator below is drained. Skipped when neither is set - each item still gets
1107
+ // its own per-item validation below, and validating an absent label/userData here would spuriously
1108
+ // check them against a registered default-route schema.
1109
+ if (options.label !== undefined || options.userData !== undefined) {
1110
+ await this.#validateRequestUserData({ label: options.label, userData: options.userData });
1111
+ }
1112
+ const requestLimit = await this.#calculateEnqueuedRequestLimit(options.limit);
1113
+ const strategy = options.strategy ?? EnqueueStrategy.All;
1114
+ const urlExcludePatternObjects = options.exclude?.length ? constructUrlPatternObjects(options.exclude) : [];
1115
+ const urlPatternObjects = options.include?.length ? constructUrlPatternObjects(options.include) : [];
1116
+ // The strategy always applies, even when `include` patterns are provided - the two are AND-ed together
1117
+ // (a URL must match an `include` pattern *and* satisfy the strategy). This mirrors crawlee-python.
1118
+ const strategyBaseUrl = options.baseUrl ? new URL(options.baseUrl) : undefined;
1119
+ const isAllowedBasedOnRobotsTxtFile = this.#isAllowedBasedOnRobotsTxtFile.bind(this);
1120
+ const maxCrawlDepth = this.#maxCrawlDepth;
1121
+ const validateRequestUserData = this.#validateRequestUserData.bind(this);
1122
+ const allSkipped = [];
1123
+ // A skipped source (which can carry arbitrary userData) is only retained if something reads it -
1124
+ // otherwise the URL alone is enough to build the callback argument and to log with.
1125
+ const hasSkippedRequestCallback = this.#onSkippedRequest !== undefined || options.onSkippedRequest !== undefined;
1126
+ const keepSkippedSource = (source) => (hasSkippedRequestCallback ? source : source.url);
1127
+ const reportSkippedRequests = async () => {
1128
+ const skippedRequests = allSkipped.splice(0);
1129
+ if (skippedRequests.length === 0) {
1130
+ return;
1131
+ }
1132
+ const skippedRobotsUrls = skippedRequests
1133
+ .filter((s) => s.reason === 'robotsTxt')
1134
+ .map(({ source }) => (typeof source === 'string' ? source : source.url));
1135
+ if (skippedRobotsUrls.length > 0) {
1136
+ this.log.warning(`Some requests were skipped because they were disallowed based on the robots.txt file`, { skipped: skippedRobotsUrls });
1137
+ }
1138
+ // Only log the limit message when an explicit `limit` was passed (not the internal
1139
+ // `maxRequestsPerCrawl`-derived one), and only once per call.
1140
+ if (options.limit !== undefined && skippedRequests.some((s) => s.reason === 'limit')) {
1141
+ this.log.info(requestLimit === options.limit
1142
+ ? `Skipping requests in this call due to the enqueueLinks limit of ${options.limit}.`
1143
+ : `Skipping requests in this call due to the remaining maxRequestsPerCrawl budget of ${requestLimit}, which is lower than the enqueueLinks limit of ${options.limit}.`);
1144
+ }
1145
+ await Promise.all(skippedRequests.map(async ({ source, reason }) => {
1146
+ const args = createSkippedRequestArgs(source, reason);
1147
+ await this.#handleSkippedRequest(args);
1148
+ await options.onSkippedRequest?.(args);
1149
+ }));
1150
+ };
652
1151
  async function* filteredRequests() {
653
- let yieldedRequestCount = 0;
654
1152
  for await (const request of requests) {
655
- const url = typeof request === 'string' ? request : request.url;
656
- if (requestLimit !== undefined && yieldedRequestCount >= requestLimit) {
657
- skippedBecauseOfLimit.add(url);
1153
+ const [requestOptions] = createRequestOptions([typeof request === 'string' ? request : request], { ...options, strategy });
1154
+ if (!requestOptions) {
1155
+ continue; // invalid URL, silently dropped (matches `createRequestOptions`'s own filtering)
1156
+ }
1157
+ if (maxCrawlDepth !== undefined && requestOptions.crawlDepth > maxCrawlDepth) {
1158
+ allSkipped.push({ source: keepSkippedSource(requestOptions), reason: 'depth' });
658
1159
  continue;
659
1160
  }
660
- if (maxCrawlDepth !== undefined && request.crawlDepth > maxCrawlDepth) {
661
- skippedBecauseOfMaxCrawlDepth.add(url);
1161
+ if (!(await isAllowedBasedOnRobotsTxtFile(requestOptions.url))) {
1162
+ allSkipped.push({ source: keepSkippedSource(requestOptions), reason: 'robotsTxt' });
662
1163
  continue;
663
1164
  }
664
- if (await isAllowedBasedOnRobotsTxtFile(url)) {
665
- yield request;
666
- yieldedRequestCount += 1;
1165
+ const onSkippedByFilter = (opts) => allSkipped.push({ source: keepSkippedSource(opts), reason: 'filters' });
1166
+ // Filter by user patterns first (with exclude)...
1167
+ let filtered = filterRequestOptionsByPatterns([requestOptions], urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, strategy, onSkippedByFilter);
1168
+ // ...then filter by the enqueue strategy (making this an AND check)
1169
+ if (strategyBaseUrl) {
1170
+ filtered = filtered.filter((opts) => {
1171
+ const { allowed } = filterUrl(opts.url, strategyBaseUrl, strategy);
1172
+ if (!allowed)
1173
+ onSkippedByFilter(opts);
1174
+ return allowed;
1175
+ });
667
1176
  }
668
- else {
669
- skippedBecauseOfRobots.add(url);
1177
+ if (filtered.length === 0) {
1178
+ continue;
1179
+ }
1180
+ let [finalOptions] = filtered;
1181
+ if (options.transformRequestFunction) {
1182
+ const transformed = applyRequestTransform([finalOptions], options.transformRequestFunction, (r) => allSkipped.push({ source: keepSkippedSource(r), reason: 'transform' }));
1183
+ if (transformed.length === 0) {
1184
+ continue;
1185
+ }
1186
+ [finalOptions] = transformed;
670
1187
  }
1188
+ await validateRequestUserData(finalOptions);
1189
+ yield new Request(finalOptions);
671
1190
  }
672
1191
  }
673
- const result = await this.requestManager.addRequestsBatched(filteredRequests(), options);
674
- if (skippedBecauseOfRobots.size > 0) {
675
- this.log.warning(`Some requests were skipped because they were disallowed based on the robots.txt file`, {
676
- skipped: [...skippedBecauseOfRobots],
677
- });
678
- }
679
- if (skippedBecauseOfRobots.size > 0 ||
680
- skippedBecauseOfLimit.size > 0 ||
681
- skippedBecauseOfMaxCrawlDepth.size > 0) {
682
- await Promise.all([...skippedBecauseOfRobots]
683
- .map((url) => {
684
- return this.handleSkippedRequest({ url, reason: 'robotsTxt' });
685
- })
686
- .concat([...skippedBecauseOfLimit].map((url) => {
687
- return this.handleSkippedRequest({ url, reason: 'limit' });
688
- }), [...skippedBecauseOfMaxCrawlDepth].map((url) => {
689
- return this.handleSkippedRequest({ url, reason: 'depth' });
690
- })));
691
- }
692
- return result;
1192
+ const result = await this.requestManager.addRequestsBatched(filteredRequests(), {
1193
+ forefront: options.forefront,
1194
+ waitForAllRequestsToBeAdded: options.waitForAllRequestsToBeAdded || requestLimit !== undefined,
1195
+ batchSize: options.batchSize,
1196
+ waitBetweenBatchesMillis: options.waitBetweenBatchesMillis,
1197
+ maxNewRequests: requestLimit,
1198
+ });
1199
+ // Report requests skipped due to the maxNewRequests budget (i.e. maxRequestsPerCrawl limit, or an
1200
+ // explicit `limit` option)
1201
+ for (const request of result.requestsOverLimit ?? []) {
1202
+ allSkipped.push({ source: request, reason: 'limit' });
1203
+ }
1204
+ await reportSkippedRequests();
1205
+ const waitForAllRequestsToBeAdded = result.waitForAllRequestsToBeAdded.then(async (addedRequests) => {
1206
+ await reportSkippedRequests();
1207
+ return addedRequests;
1208
+ });
1209
+ // Keep callback failures observable to callers that await this promise without emitting an unhandled rejection
1210
+ // when callers intentionally leave background additions running, matching `drainRequestBatches` behavior.
1211
+ // Background failures are already logged by `drainRequestBatches`.
1212
+ void waitForAllRequestsToBeAdded.catch(() => { });
1213
+ return { ...result, waitForAllRequestsToBeAdded };
693
1214
  }
694
1215
  /**
695
1216
  * Pushes data to the specified {@link Dataset}, or the default crawler {@link Dataset} by calling {@link Dataset.pushData}.
696
1217
  */
697
- async pushData(data, datasetIdOrName) {
698
- const dataset = await this.getDataset(datasetIdOrName);
1218
+ async pushData(data, datasetIdentifier) {
1219
+ tryCancel();
1220
+ const dataset = await this.getDataset(datasetIdentifier);
699
1221
  return dataset.pushData(data);
700
1222
  }
701
1223
  /**
702
1224
  * Retrieves the specified {@link Dataset}, or the default crawler {@link Dataset}.
703
1225
  */
704
- async getDataset(idOrName) {
705
- return Dataset.open(idOrName, { config: serviceLocator.getConfiguration() });
1226
+ async getDataset(identifier) {
1227
+ return Dataset.open(identifier, {
1228
+ configuration: serviceLocator.getConfiguration(),
1229
+ });
706
1230
  }
707
1231
  /**
708
1232
  * Retrieves data from the default crawler {@link Dataset} by calling {@link Dataset.getData}.
@@ -717,8 +1241,9 @@ export class BasicCrawler {
717
1241
  */
718
1242
  async exportData(path, format, options) {
719
1243
  const supportedFormats = ['json', 'csv'];
720
- if (!format && path.match(/\.(json|csv)$/i)) {
721
- format = path.toLowerCase().match(/\.(json|csv)$/)[1];
1244
+ const formatMatch = /\.(json|csv)$/i.exec(path);
1245
+ if (!format && formatMatch) {
1246
+ format = formatMatch[1].toLowerCase();
722
1247
  }
723
1248
  if (!format) {
724
1249
  throw new Error(`Failed to infer format from the path: '${path}'. Supported formats: ${supportedFormats.join(', ')}`);
@@ -737,6 +1262,7 @@ export class BasicCrawler {
737
1262
  const keys = options?.collectAllKeys
738
1263
  ? Array.from(new Set(items.flatMap(Object.keys)))
739
1264
  : Object.keys(items[0]);
1265
+ const { stringify } = await import('csv-stringify/sync');
740
1266
  value = stringify([
741
1267
  keys,
742
1268
  ...items.map((item) => {
@@ -744,13 +1270,13 @@ export class BasicCrawler {
744
1270
  }),
745
1271
  ]);
746
1272
  }
747
- await ensureDir(dirname(path));
1273
+ await mkdir(dirname(path), { recursive: true });
748
1274
  await writeFile(path, value);
749
1275
  this.log.info(`Export to ${path} finished!`);
750
1276
  }
751
1277
  if (format === 'json') {
752
- await ensureDir(dirname(path));
753
- await writeJSON(path, items, { spaces: 4 });
1278
+ await mkdir(dirname(path), { recursive: true });
1279
+ await writeFile(path, `${JSON.stringify(items, null, 4)}\n`);
754
1280
  this.log.info(`Export to ${path} finished!`);
755
1281
  }
756
1282
  return items;
@@ -758,58 +1284,194 @@ export class BasicCrawler {
758
1284
  /**
759
1285
  * Initializes the crawler.
760
1286
  */
761
- async _init() {
1287
+ async init() {
1288
+ const run = this.#liveRun;
1289
+ if (run === undefined) {
1290
+ throw new Error('There is no crawl in progress - `run()` is the only thing that starts one.');
1291
+ }
762
1292
  const eventManager = serviceLocator.getEventManager();
763
1293
  if (!eventManager.isInitialized()) {
764
1294
  await eventManager.init();
765
- this._closeEvents = true;
766
- }
767
- // Initialize AutoscaledPool before awaiting _loadHandledRequestCount(),
768
- // so that the caller can get a reference to it before awaiting the promise returned from run()
769
- // (otherwise there would be no way)
770
- this.autoscaledPool = new AutoscaledPool(this.autoscaledPoolOptions);
771
- if (this.useSessionPool) {
772
- this.sessionPool = await SessionPool.open(this.sessionPoolOptions);
773
- // Assuming there are not more than 20 browsers running at once;
774
- this.sessionPool.setMaxListeners(20);
1295
+ run.ownsEventManager = true;
1296
+ }
1297
+ // Warn once at startup if the internal timeout is shorter than the phases it is meant to outlast. It is
1298
+ // floored per request so it will not actually cut them short, but the configured value is then effectively
1299
+ // ignored, which is worth flagging. Checked here (not in the constructor) because a subclass sets its
1300
+ // navigation timeout only after `super()`.
1301
+ const phasesMillis = this.getNavigationTimeoutMillis() + this.#resolveRequestHandlerTimeoutMillis(undefined);
1302
+ if (this.internalTimeoutMillis < phasesMillis) {
1303
+ this.log.warning(`CRAWLEE_INTERNAL_TIMEOUT (${this.internalTimeoutMillis / 1000}s) is shorter than the navigation ` +
1304
+ `and request handler timeouts combined (${phasesMillis / 1000}s); it will be raised per request ` +
1305
+ `so it does not cut them short.`);
1306
+ }
1307
+ await this.getRequestManager();
1308
+ }
1309
+ /**
1310
+ * The navigation timeout (pre-navigation hooks, navigation, and post-navigation hooks) in milliseconds, used
1311
+ * to size the internal request timeout. `BasicCrawler` has no navigation phase, so this is 0; the HTTP and
1312
+ * browser crawlers override it with their `navigationTimeoutSecs`.
1313
+ */
1314
+ getNavigationTimeoutMillis() {
1315
+ return 0;
1316
+ }
1317
+ /**
1318
+ * Races the request against the internal timeout (see {@link raceWithTimeout}), sized to outlast the phases
1319
+ * that have their own timeout - the navigation, its hooks, and the request handler - so a legitimately slow
1320
+ * request, a per-route override, or a low `CRAWLEE_INTERNAL_TIMEOUT` is not cut short mid-phase. It takes
1321
+ * whichever is larger: the configured internal timeout, or this request's combined phase budget.
1322
+ */
1323
+ async #withRequestTimeout(crawlingContext, work) {
1324
+ const { request } = crawlingContext;
1325
+ const phasesMillis = this.getNavigationTimeoutMillis() + this.#resolveRequestHandlerTimeoutMillis(request.label);
1326
+ const timeoutMillis = Math.max(this.internalTimeoutMillis, phasesMillis);
1327
+ await raceWithTimeout(crawlingContext, work, { timeoutMillis, requestId: request.id });
1328
+ }
1329
+ /**
1330
+ * The request handler timeout for a request with the given route label. A router route may override the
1331
+ * crawler's own `requestHandlerTimeoutSecs`; anything else falls back to `fallbackMillis`.
1332
+ *
1333
+ * @param label The request's route label, or `undefined` for the default route / no specific request.
1334
+ * @param fallbackMillis Timeout to use when no route overrides it.
1335
+ */
1336
+ #resolveRequestHandlerTimeoutMillis(label, fallbackMillis = this.#requestHandlerTimeoutMillis) {
1337
+ return this.#getRouteTimeoutMillis(label) ?? fallbackMillis;
1338
+ }
1339
+ /**
1340
+ * The timeout the router route with the given label asked for, or `undefined` when it did not override one
1341
+ * (or the request handler is not a router at all).
1342
+ */
1343
+ #getRouteTimeoutMillis(label) {
1344
+ const getTimeoutSecs = this.requestHandler.getTimeoutSecs;
1345
+ if (typeof getTimeoutSecs !== 'function') {
1346
+ return undefined;
775
1347
  }
776
- await this.initializeRequestManager();
777
- await this._loadHandledRequestCount();
1348
+ const timeoutSecs = getTimeoutSecs(label);
1349
+ return timeoutSecs === undefined ? undefined : timeoutSecs * 1000;
778
1350
  }
779
1351
  async runRequestHandler(crawlingContext) {
780
- await this.contextPipeline.call(crawlingContext, async (finalContext) => {
781
- await addTimeoutToPromise(async () => this.requestHandler(finalContext), this.requestHandlerTimeoutMillis, `requestHandler timed out after ${this.requestHandlerTimeoutMillis / 1000} seconds (${finalContext.request.id}).`);
1352
+ const timeoutMillis = this.#resolveRequestHandlerTimeoutMillis(crawlingContext.request.label);
1353
+ await addTimeoutToPromise(async () => this.requestHandler(crawlingContext), timeoutMillis, `requestHandler timed out after ${timeoutMillis / 1000} seconds (${crawlingContext.request.id}).`);
1354
+ }
1355
+ /**
1356
+ * Runs `callback` inside a {@link StorageTransaction}, unless transactional storage is disabled.
1357
+ * Deliberately does **not** commit on return - the pipeline hands failures to `handleRequestFailure`, so a
1358
+ * normal return says nothing about success. `handleRequest` commits on success.
1359
+ */
1360
+ // oxlint-disable-next-line crawlee/prefer-private-fields -- accessed by tests
1361
+ async runInStorageTransaction(callback) {
1362
+ if (!this.#transactionalStorageEnabled) {
1363
+ return callback();
1364
+ }
1365
+ const transaction = createStorageTransaction({
1366
+ policy: this.#storageWritePolicy,
1367
+ commitTimeoutMillis: this.internalTimeoutMillis,
782
1368
  });
1369
+ let threw = true;
1370
+ try {
1371
+ const result = await transaction.run(callback);
1372
+ threw = false;
1373
+ return result;
1374
+ }
1375
+ finally {
1376
+ if (transaction.state === 'open') {
1377
+ // `handleRequest` commits or `handleRequestFailure` rolls back on every normal path, so an open transaction on
1378
+ // a normal return is a wiring bug; on a propagating throw (a pipeline-level failure) it is
1379
+ // expected. Either way, discard the unvalidated writes; only the former is worth flagging.
1380
+ if (!threw) {
1381
+ this.log.error('Internal error: a storage transaction was still open after the request pipeline ' +
1382
+ 'returned normally. Its writes are being discarded. Please report this.');
1383
+ }
1384
+ transaction.rollback();
1385
+ }
1386
+ // Unconditional: `failed` is a terminal state that the branch above never reaches.
1387
+ transaction.dispose();
1388
+ }
783
1389
  }
784
1390
  /**
785
1391
  * Handles blocked request
786
1392
  */
787
- _throwOnBlockedRequest(session, statusCode) {
788
- const isBlocked = session.retireOnBlockedStatusCodes(statusCode);
789
- if (isBlocked) {
790
- throw new Error(`Request blocked - received ${statusCode} status code.`);
1393
+ throwOnBlockedRequest(statusCode) {
1394
+ if (this.retryOnBlocked)
1395
+ return;
1396
+ if (this.blockedStatusCodes.has(statusCode)) {
1397
+ throw new SessionError(`Request blocked - received ${statusCode} status code.`);
791
1398
  }
792
1399
  }
793
- async isAllowedBasedOnRobotsTxtFile(url) {
794
- if (!this.respectRobotsTxtFile) {
1400
+ async #isAllowedBasedOnRobotsTxtFile(url) {
1401
+ if (!this.#respectRobotsTxtFile) {
795
1402
  return true;
796
1403
  }
797
1404
  const robotsTxtFile = await this.getRobotsTxtFileForUrl(url);
798
- const userAgent = typeof this.respectRobotsTxtFile === 'object' ? this.respectRobotsTxtFile?.userAgent : '*';
1405
+ const userAgent = typeof this.#respectRobotsTxtFile === 'object' ? this.#respectRobotsTxtFile?.userAgent : '*';
1406
+ if (robotsTxtFile) {
1407
+ const crawlDelay = robotsTxtFile.getCrawlDelay(userAgent);
1408
+ if (crawlDelay !== undefined) {
1409
+ this.#applyCrawlDelay(url, crawlDelay);
1410
+ }
1411
+ }
799
1412
  return !robotsTxtFile || robotsTxtFile.isAllowed(url, userAgent);
800
1413
  }
1414
+ /**
1415
+ * Records an HTTP 429 against the URL's domain so the request manager can hold the retry back.
1416
+ *
1417
+ * @param retryAfterHeader The raw `Retry-After` response header, if the server sent one.
1418
+ * @returns `true` if the manager took responsibility for the delay, in which case the caller should throw
1419
+ * {@link RequestThrottledError} rather than treating the response as a blocked session.
1420
+ */
1421
+ recordDomainRateLimit(url, retryAfterHeader) {
1422
+ if (this.requestManager?.recordPacingSignal({
1423
+ reason: 'rateLimited',
1424
+ url,
1425
+ waitMs: parseRetryAfterHeader(retryAfterHeader) ?? undefined,
1426
+ })) {
1427
+ return true;
1428
+ }
1429
+ const domain = hostnameOrUrl(url);
1430
+ this.log.warningOnce(`"${domain}" responded with HTTP 429 (Too Many Requests), but the crawler's request manager does not ` +
1431
+ 'pace that domain, so the response is handled like any other, with no per-domain delay. Set ' +
1432
+ `\`sameDomainDelaySecs\`, or pass a \`ThrottlingRequestManager\` covering "${domain}" as ` +
1433
+ '`requestManager`, to honour `Retry-After` and apply exponential backoff instead.');
1434
+ return false;
1435
+ }
1436
+ /** Hands a robots.txt `Crawl-delay` to the request manager, warning if it will not be honoured. */
1437
+ #applyCrawlDelay(url, delaySeconds) {
1438
+ // robots.txt is per-origin; `hostname` is the closest pacing scope and errs wide (http and https to one
1439
+ // host share a clock).
1440
+ if (this.requestManager?.recordPacingSignal({
1441
+ reason: 'minInterval',
1442
+ url,
1443
+ intervalMs: delaySeconds * 1000,
1444
+ scope: 'hostname',
1445
+ })) {
1446
+ return;
1447
+ }
1448
+ const domain = hostnameOrUrl(url);
1449
+ this.log.warningOnce(`robots.txt for "${domain}" defines a crawl-delay of ${delaySeconds}s, but the crawler's request ` +
1450
+ 'manager does not pace that domain, so its requests will not be paced. Set ' +
1451
+ `\`sameDomainDelaySecs\`, or pass a \`ThrottlingRequestManager\` covering "${domain}" as ` +
1452
+ '`requestManager`.');
1453
+ }
1454
+ /** @internal */
801
1455
  async getRobotsTxtFileForUrl(url) {
802
- if (!this.respectRobotsTxtFile) {
1456
+ if (!this.#respectRobotsTxtFile) {
803
1457
  return undefined;
804
1458
  }
805
1459
  try {
806
1460
  const origin = new URL(url).origin;
807
- const cachedRobotsTxtFile = this.robotsTxtFileCache.get(origin);
1461
+ const cachedRobotsTxtFile = this.#robotsTxtFileCache.get(origin);
808
1462
  if (cachedRobotsTxtFile) {
809
1463
  return cachedRobotsTxtFile;
810
1464
  }
811
- const robotsTxtFile = await RobotsTxtFile.find(url);
812
- this.robotsTxtFileCache.add(origin, robotsTxtFile);
1465
+ // Fetch robots.txt the way pages are fetched: through the crawler's HTTP client, with a session's browser
1466
+ // fingerprint and proxy. Sites with bot protection answer a bare request with 4xx or 5xx, and that status
1467
+ // alone would then decide whether everything is allowed or disallowed.
1468
+ const robotsTxtFile = await RobotsTxtFile.find(url, {
1469
+ httpClient: this.httpClient,
1470
+ session: await this.sessionPool.getSession(),
1471
+ timeoutMillis: 30_000,
1472
+ logger: this.log,
1473
+ });
1474
+ this.#robotsTxtFileCache.add(origin, robotsTxtFile);
813
1475
  return robotsTxtFile;
814
1476
  }
815
1477
  catch (e) {
@@ -817,10 +1479,10 @@ export class BasicCrawler {
817
1479
  return undefined;
818
1480
  }
819
1481
  }
820
- async _pauseOnMigration() {
821
- if (this.autoscaledPool) {
822
- // if run wasn't called, this is going to crash
823
- await this.autoscaledPool.pause(SAFE_MIGRATION_WAIT_MILLIS).catch((err) => {
1482
+ async #pauseOnMigration() {
1483
+ const run = this.#dispatchingRun;
1484
+ if (run) {
1485
+ await run.pause(SAFE_MIGRATION_WAIT_MILLIS).catch((err) => {
824
1486
  if (err.message.includes('running tasks did not finish')) {
825
1487
  this.log.error('The crawler was paused due to migration to another host, ' +
826
1488
  "but some requests did not finish in time. Those requests' results may be duplicated.");
@@ -830,233 +1492,115 @@ export class BasicCrawler {
830
1492
  }
831
1493
  });
832
1494
  }
833
- const requestListPersistPromise = (async () => {
834
- if (this.requestList) {
835
- if (await this.requestList.isFinished())
836
- return;
837
- await this.requestList.persistState().catch((err) => {
838
- if (err.message.includes('Cannot persist state.')) {
839
- this.log.error("The crawler attempted to persist its request list's state and failed due to missing or " +
840
- 'invalid config. Make sure to use either RequestList.open() or the "stateKeyPrefix" option of RequestList ' +
841
- 'constructor to ensure your crawling state is persisted through host migrations and restarts.');
842
- }
843
- else {
844
- this.log.exception(err, 'An unexpected error occurred when the crawler ' +
845
- "attempted to persist its request list's state.");
846
- }
847
- });
848
- }
849
- })();
850
- await Promise.all([requestListPersistPromise, this.stats.persistState()]);
851
- }
852
- /**
853
- * Initializes the RequestManager based on the configured requestList and requestQueue.
854
- */
855
- async initializeRequestManager() {
856
- if (this.requestManager !== undefined) {
857
- return;
858
- }
859
- if (this.requestList && this.requestQueue) {
860
- // Create a RequestManagerTandem if both RequestList and RequestQueue are provided
861
- this.requestManager = new RequestManagerTandem(this.requestList, this.requestQueue);
862
- }
863
- else if (this.requestQueue) {
864
- // Use RequestQueue directly if only it is provided
865
- this.requestManager = this.requestQueue;
866
- }
867
- else if (this.requestList) {
868
- // Use RequestList directly if only it is provided
869
- // Make it compatible with the IRequestManager interface
870
- this.requestManager = new RequestListAdapter(this.requestList);
871
- }
872
- // If neither RequestList nor RequestQueue is provided, leave the requestManager uninitialized until `getRequestQueue` is called
1495
+ // Captures the statistics changed while draining above. No PERSIST_STATE event is tied to the drain:
1496
+ // the periodic one is unrelated, and the one the platform emits on migration arrives before any of this.
1497
+ await this.statistics.persistState?.();
873
1498
  }
874
1499
  /**
875
1500
  * Fetches the next request to process from the underlying request provider.
876
1501
  */
877
- async _fetchNextRequest() {
1502
+ async #fetchNextRequest() {
878
1503
  if (this.requestManager === undefined) {
879
- throw new Error(`_fetchNextRequest called on an uninitialized crawler`);
1504
+ throw new Error(`fetchNextRequest called on an uninitialized crawler`);
880
1505
  }
881
1506
  return this.requestManager.fetchNextRequest();
882
1507
  }
883
- /**
884
- * Delays processing of the request based on the `sameDomainDelaySecs` option,
885
- * adding it back to the queue after the timeout passes. Returns `true` if the request
886
- * should be ignored and will be reclaimed to the queue once ready.
887
- */
888
- delayRequest(request, source) {
889
- const domain = getDomain(request.url);
890
- if (!domain || !request) {
891
- return false;
892
- }
893
- const now = Date.now();
894
- const lastAccessTime = this.domainAccessedTime.get(domain);
895
- if (!lastAccessTime || now - lastAccessTime >= this.sameDomainDelayMillis) {
896
- this.domainAccessedTime.set(domain, now);
897
- return false;
898
- }
899
- if (source instanceof RequestQueueV1) {
900
- // eslint-disable-next-line dot-notation
901
- source['inProgress']?.delete(request.id);
1508
+ /** Runs the request handler and marks the request as handled. Failures are left to `handleRequestFailure`. */
1509
+ // oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
1510
+ async handleRequest(crawlingContext, requestSource) {
1511
+ // An earlier phase we cannot cancel (e.g. a slow `extendContext`) may have run past the internal timeout,
1512
+ // which already failed the request in `runTaskFunction`. Bail before running the handler so it does not
1513
+ // execute (and re-report) on top of a request the crawler has already moved past.
1514
+ if (crawlingContext[timeoutExpiredKey]?.()) {
1515
+ return;
902
1516
  }
903
- const delay = lastAccessTime + this.sameDomainDelayMillis - now;
904
- this.log.debug(`Request ${request.url} (${request.id}) will be reclaimed after ${delay} milliseconds due to same domain delay`);
905
- setTimeout(async () => {
906
- this.log.debug(`Adding request ${request.url} (${request.id}) back to the queue`);
907
- if (source instanceof RequestQueueV1) {
908
- // eslint-disable-next-line dot-notation
909
- source['inProgress'].add(request.id);
910
- }
911
- await source.reclaimRequest(request, { forefront: request.userData?.__crawlee?.forefront });
912
- }, delay);
913
- return true;
1517
+ const { request } = crawlingContext;
1518
+ request.state = RequestState.REQUEST_HANDLER;
1519
+ await this.runRequestHandler(crawlingContext);
1520
+ // Commit *before* marking the request as handled, so a commit failure fails the request and
1521
+ // it is retried. This also closes the transaction, so everything below passes through.
1522
+ // Opened by `runInStorageTransaction`; absent when disabled or when the subclass opens its own.
1523
+ await currentStorageTransaction()?.commit();
1524
+ await this.#timeoutAndRetry(async () => requestSource.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${request.url} (${request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
1525
+ this.statistics.recordRequestSuccess(request.id || request.uniqueKey, request.retryCount);
1526
+ // reclaim session if request finishes successfully
1527
+ request.state = RequestState.DONE;
1528
+ crawlingContext.session.markGood();
914
1529
  }
915
1530
  /**
916
- * Wrapper around requestHandler that fetches requests from RequestList/RequestQueue
917
- * then retries them in a case of an error, etc.
1531
+ * Handles a request that failed anywhere in the context pipeline - a middleware (e.g. navigation or its hooks)
1532
+ * or `handleRequest`. Runs inside the pipeline call, so the cleanups (e.g. closing the page) only fire after the
1533
+ * error handler has seen the context.
918
1534
  */
919
- async _runTaskFunction() {
920
- const source = this.requestManager;
921
- if (!source)
922
- throw new Error('Request provider is not initialized!');
923
- const request = await this._timeoutAndRetry(this._fetchNextRequest.bind(this), this.internalTimeoutMillis, `Fetching next request timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
924
- tryCancel();
925
- const session = this.useSessionPool
926
- ? await this._timeoutAndRetry(async () => {
927
- return await this.sessionPool.newSession({
928
- proxyInfo: await this.proxyConfiguration?.newProxyInfo({
929
- request: request ?? undefined,
930
- }),
931
- maxUsageCount: 1,
932
- });
933
- }, this.internalTimeoutMillis, `Fetching session timed out after ${this.internalTimeoutMillis / 1e3} seconds.`)
934
- : undefined;
935
- tryCancel();
936
- if (!request || this.delayRequest(request, source)) {
937
- return;
1535
+ async #handleRequestFailure(rawError, crawlingContext, requestSource) {
1536
+ const { request } = crawlingContext;
1537
+ // Roll back *before* any error handler runs - error handlers write to real storage precisely
1538
+ // because the transaction is already closed. A no-op when the commit in `handleRequest` succeeded.
1539
+ currentStorageTransaction()?.rollback();
1540
+ const resumeRequestTimeout = crawlingContext[suspendTimeoutKey]?.();
1541
+ try {
1542
+ const unwrappedError = this.#unwrapError(rawError);
1543
+ // A failing or timed out `onSkippedRequest` makes this an ordinary failure, so the request gets retried.
1544
+ const err = unwrappedError instanceof ContextPipelineInterruptedError
1545
+ ? await this.#trySkipRequest(unwrappedError, request, requestSource)
1546
+ : unwrappedError;
1547
+ if (err) {
1548
+ await this.#runErrorHandler(err, crawlingContext, requestSource);
1549
+ }
938
1550
  }
939
- if (!(await this.isAllowedBasedOnRobotsTxtFile(request.url))) {
940
- this.log.warning(`Skipping request ${request.url} (${request.id}) because it is disallowed based on robots.txt`);
941
- request.state = RequestState.SKIPPED;
942
- request.noRetry = true;
943
- await source.markRequestHandled(request);
944
- await this.handleSkippedRequest({
945
- url: request.url,
946
- reason: 'robotsTxt',
947
- });
948
- return;
1551
+ finally {
1552
+ resumeRequestTimeout?.();
949
1553
  }
950
- // Reset loadedUrl so an old one is not carried over to retries.
951
- request.loadedUrl = undefined;
952
- const statisticsId = request.id || request.uniqueKey;
953
- this.stats.startJob(statisticsId);
954
- const deferredCleanup = [];
955
- const crawlingContext = {
956
- id: cryptoRandomObjectId(10),
957
- log: this.log,
958
- request,
959
- session,
960
- proxyInfo: session?.proxyInfo,
961
- enqueueLinks: async (options) => {
962
- const requestQueue = await this.getRequestQueue();
963
- return await this.enqueueLinksWithCrawlDepth(options, request, requestQueue);
964
- },
965
- addRequests: async (requests, options = {}) => {
966
- const newCrawlDepth = request.crawlDepth + 1;
967
- const requestsGenerator = this.addCrawlDepthRequestGenerator(requests, newCrawlDepth);
968
- await this.addRequests(requestsGenerator, options);
969
- },
970
- pushData: this.pushData.bind(this),
971
- useState: this.useState.bind(this),
972
- sendRequest: createSendRequest(this.httpClient, request, session),
973
- getKeyValueStore: async (idOrName) => KeyValueStore.open(idOrName, { config: serviceLocator.getConfiguration() }),
974
- registerDeferredCleanup: (cleanup) => {
975
- deferredCleanup.push(cleanup);
976
- },
977
- };
978
- let isRequestLocked = true;
1554
+ }
1555
+ async #runErrorHandler(err, context, requestSource) {
1556
+ const { request } = context;
1557
+ const retryCountBefore = request.retryCount;
979
1558
  try {
980
- request.state = RequestState.REQUEST_HANDLER;
981
- await this.runRequestHandler(crawlingContext);
982
- await this._timeoutAndRetry(async () => source.markRequestHandled(request), this.internalTimeoutMillis, `Marking request ${request.url} (${request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
983
- isRequestLocked = false; // markRequestHandled succeeded and unlocked the request
984
- this.stats.finishJob(statisticsId, request.retryCount);
985
- this.handledRequestsCount++;
986
- // reclaim session if request finishes successfully
987
- request.state = RequestState.DONE;
988
- crawlingContext.session?.markGood();
989
- }
990
- catch (rawError) {
991
- const err = this.unwrapError(rawError);
992
- try {
993
- request.state = RequestState.ERROR_HANDLER;
994
- await addTimeoutToPromise(async () => this._requestFunctionErrorHandler(err, crawlingContext, source), this.internalTimeoutMillis, `Handling request failure of ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
995
- if (!(err instanceof CriticalError)) {
996
- isRequestLocked = false; // _requestFunctionErrorHandler calls either markRequestHandled or reclaimRequest
997
- }
1559
+ request.state = RequestState.ERROR_HANDLER;
1560
+ await addTimeoutToPromise(async () => this.requestFunctionErrorHandler(err, context, requestSource), this.internalTimeoutMillis, `Handling request failure of ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
1561
+ // The error handler may have set `SKIPPED`; TS keeps the narrowing from the assignment above.
1562
+ if (request.state !== RequestState.SKIPPED) {
998
1563
  request.state = RequestState.DONE;
999
1564
  }
1000
- catch (secondaryError) {
1001
- const unwrappedSecondaryError = this.unwrapError(secondaryError);
1002
- if (!unwrappedSecondaryError.triggeredFromUserHandler &&
1003
- // avoid reprinting the same critical error multiple times, as it will be printed by Nodejs at the end anyway
1004
- !(unwrappedSecondaryError instanceof CriticalError)) {
1005
- const apifySpecific = process.env.APIFY_IS_AT_HOME
1006
- ? `This may have happened due to an internal error of Apify's API or due to a misconfigured crawler.`
1007
- : '';
1008
- this.log.exception(unwrappedSecondaryError, 'An exception occurred during handling of failed request. ' +
1009
- `This places the crawler and its underlying storages into an unknown state and crawling will be terminated. ${apifySpecific}`);
1565
+ }
1566
+ catch (secondaryError) {
1567
+ const unwrappedSecondaryError = this.#unwrapError(secondaryError);
1568
+ const timedOut = unwrappedSecondaryError instanceof TimeoutError;
1569
+ request.state = RequestState.ERROR;
1570
+ if (timedOut) {
1571
+ // Not worth crashing the crawler over. The request's state is unknown, so it gets retried - and the
1572
+ // retry is counted, so that an error handler that always times out does not retry it forever.
1573
+ this.log.warning(unwrappedSecondaryError.message);
1574
+ if (request.retryCount === retryCountBefore) {
1575
+ request.retryCount++;
1010
1576
  }
1011
- request.state = RequestState.ERROR;
1577
+ }
1578
+ else if (!(unwrappedSecondaryError instanceof CriticalError)) {
1579
+ // avoid reprinting the same critical error multiple times, as it will be printed by Nodejs at the end anyway
1580
+ const apifySpecific = process.env.APIFY_IS_AT_HOME
1581
+ ? `This may have happened due to an internal error of Apify's API or due to a misconfigured crawler.`
1582
+ : '';
1583
+ this.log.exception(unwrappedSecondaryError, 'An exception occurred during handling of failed request. ' +
1584
+ `This places the crawler and its underlying storages into an unknown state and crawling will be terminated. ${apifySpecific}`);
1585
+ }
1586
+ // Safety net - the error handler neither marked the request as handled nor reclaimed it (e.g. after a
1587
+ // CriticalError). Reclaiming a request that is no longer in progress is a harmless no-op on the storage backend.
1588
+ try {
1589
+ await addTimeoutToPromise(async () => requestSource.reclaimRequest(request), this.internalTimeoutMillis, `Reclaiming request ${request.url} (${request.id}) timed out.`);
1590
+ }
1591
+ catch {
1592
+ // The request was never in progress, or could not be reclaimed. Either way it's fine.
1593
+ }
1594
+ if (!timedOut) {
1012
1595
  throw unwrappedSecondaryError;
1013
1596
  }
1014
- // decrease the session score if the request fails (but the error handler did not throw)
1015
- crawlingContext.session?.markBad();
1016
1597
  }
1017
- finally {
1018
- await Promise.all(deferredCleanup.map((cleanup) => cleanup()));
1019
- // Safety net - release the lock if nobody managed to do it before
1020
- if (isRequestLocked && source instanceof RequestProvider) {
1021
- try {
1022
- await source.client.deleteRequestLock(request.id);
1023
- }
1024
- catch {
1025
- // We don't have the lock, or the request was never locked. Either way it's fine
1026
- }
1027
- }
1598
+ // decrease the session score if the request fails (but the error handler did not throw);
1599
+ // skip when the error is a SessionError, which already retired the session
1600
+ if (!this.#errorAbsolvesSession(err)) {
1601
+ context.session?.markBad();
1028
1602
  }
1029
1603
  }
1030
- /**
1031
- * Wrapper around the crawling context's `enqueueLinks` method:
1032
- * - Injects `crawlDepth` to each request being added based on the crawling context request.
1033
- * - Provides defaults for the `enqueueLinks` options based on the crawler configuration.
1034
- * - These options can be overridden by the user.
1035
- * @internal
1036
- */
1037
- async enqueueLinksWithCrawlDepth(options, request, requestQueue) {
1038
- const transformRequestFunctionWrapper = (requestOptions) => {
1039
- requestOptions.crawlDepth = request.crawlDepth + 1;
1040
- if (this.maxCrawlDepth !== undefined && requestOptions.crawlDepth > this.maxCrawlDepth) {
1041
- // Setting `skippedReason` before returning `false` ensures that `reportSkippedRequests`
1042
- // reports `'depth'` as the reason (via `request.skippedReason ?? reason` fallback),
1043
- // rather than the generic `'transform'` reason.
1044
- requestOptions.skippedReason = 'depth';
1045
- return false;
1046
- }
1047
- // After injecting the crawlDepth, we call the user-provided transform function, if there is one.
1048
- return options.transformRequestFunction?.(requestOptions) ?? requestOptions;
1049
- };
1050
- return await enqueueLinks({
1051
- requestQueue,
1052
- robotsTxtFile: await this.getRobotsTxtFileForUrl(request.url),
1053
- onSkippedRequest: this.handleSkippedRequest,
1054
- limit: this.calculateEnqueuedRequestLimit(options.limit),
1055
- // Allow user options to override defaults set above ⤴
1056
- ...options,
1057
- transformRequestFunction: transformRequestFunctionWrapper,
1058
- });
1059
- }
1060
1604
  /**
1061
1605
  * Generator function that yields requests injected with the given crawl depth.
1062
1606
  * @internal
@@ -1066,6 +1610,11 @@ export class BasicCrawler {
1066
1610
  if (typeof request === 'string') {
1067
1611
  yield { url: request, crawlDepth: newRequestDepth };
1068
1612
  }
1613
+ else if (request instanceof Request) {
1614
+ const crawlee = (request.userData.__crawlee ??= {});
1615
+ crawlee.crawlDepth ??= newRequestDepth;
1616
+ yield request;
1617
+ }
1069
1618
  else {
1070
1619
  request.crawlDepth ??= newRequestDepth;
1071
1620
  yield request;
@@ -1076,7 +1625,7 @@ export class BasicCrawler {
1076
1625
  * Run async callback with given timeout and retry. Returns the result of the callback.
1077
1626
  * @ignore
1078
1627
  */
1079
- async _timeoutAndRetry(handler, timeout, error, maxRetries = 3, retried = 1) {
1628
+ async #timeoutAndRetry(handler, timeout, error, maxRetries = 3, retried = 1) {
1080
1629
  try {
1081
1630
  return await addTimeoutToPromise(handler, timeout, error);
1082
1631
  }
@@ -1084,116 +1633,155 @@ export class BasicCrawler {
1084
1633
  if (retried <= maxRetries) {
1085
1634
  // we retry on any error, not just timeout
1086
1635
  this.log.warning(`${e.message} (retrying ${retried}/${maxRetries})`);
1087
- return this._timeoutAndRetry(handler, timeout, error, maxRetries, retried + 1);
1636
+ return this.#timeoutAndRetry(handler, timeout, error, maxRetries, retried + 1);
1088
1637
  }
1089
1638
  throw e;
1090
1639
  }
1091
1640
  }
1092
1641
  /**
1093
- * Returns true if either RequestList or RequestQueue have a request ready for processing.
1094
- */
1095
- async _isTaskReadyFunction() {
1096
- return this.requestManager !== undefined && !(await this.requestManager.isEmpty());
1097
- }
1098
- /**
1099
- * Returns true if both RequestList and RequestQueue have all requests finished.
1642
+ * Whether the request manager has a request ready for processing. A manager that is only `waiting` also gets a
1643
+ * wake-up scheduled, so a paced crawl resumes on its clock rather than on the task loop's polling interval.
1100
1644
  */
1101
- async _defaultIsFinishedFunction() {
1102
- return !this.requestManager || (await this.requestManager.isFinished());
1103
- }
1104
- async _rotateSession(crawlingContext) {
1105
- const { request } = crawlingContext;
1106
- request.sessionRotationCount ??= 0;
1107
- request.sessionRotationCount++;
1108
- crawlingContext.session?.retire();
1645
+ async #isTaskReadyFunction() {
1646
+ if (this.requestManager === undefined) {
1647
+ return false;
1648
+ }
1649
+ const state = await this.requestManager.checkReadiness();
1650
+ if (state.status === 'waiting' && state.readyAt !== undefined) {
1651
+ this.#run?.scheduleWake(state.readyAt);
1652
+ }
1653
+ return state.status === 'ready';
1109
1654
  }
1110
1655
  /**
1111
1656
  * Unwraps errors thrown by the context pipeline to get the actual user error.
1112
- * RequestHandlerError and ContextPipelineInitializationError wrap the actual error.
1657
+ * RequestHandlerError, ContextPipelineInitializationError and ContextPipelineCleanupError wrap the actual error.
1113
1658
  */
1114
- unwrapError(error) {
1659
+ #unwrapError(error) {
1115
1660
  if (error instanceof RequestHandlerError ||
1116
1661
  error instanceof ContextPipelineInitializationError ||
1117
1662
  error instanceof ContextPipelineCleanupError) {
1118
- return this.unwrapError(error.cause);
1663
+ return this.#unwrapError(error.cause);
1119
1664
  }
1120
1665
  return error;
1121
1666
  }
1122
1667
  /**
1123
1668
  * Handles errors thrown by user provided requestHandler()
1124
1669
  */
1125
- async _requestFunctionErrorHandler(error, crawlingContext, source) {
1670
+ // oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
1671
+ async requestFunctionErrorHandler(error, crawlingContext, source) {
1126
1672
  const { request } = crawlingContext;
1673
+ if (error instanceof RequestThrottledError) {
1674
+ // The domain told us to come back later, so the request was never really attempted. Put it back where
1675
+ // it was, without recording a failure - it costs neither a retry nor session reputation.
1676
+ this.log.debug(`Deferring request because its domain is rate-limiting us. ${error.message}`, {
1677
+ id: request.id,
1678
+ url: request.url,
1679
+ });
1680
+ await source.reclaimRequest(request, { forefront: true });
1681
+ return;
1682
+ }
1127
1683
  request.pushErrorMessage(error);
1128
1684
  if (error instanceof CriticalError) {
1129
1685
  throw error;
1130
1686
  }
1131
- const shouldRetryRequest = this._canRequestBeRetried(request, error);
1687
+ const shouldRetryRequest = this.#canRequestBeRetried(request, error);
1132
1688
  if (shouldRetryRequest) {
1133
- await this.stats.errorTrackerRetry.addAsync(error, crawlingContext);
1134
- await this.errorHandler?.(crawlingContext, // valid cast - ExtendedContext transitively extends CrawlingContext
1135
- error);
1689
+ await this.statistics.errorTrackerRetry.addAsync(error, crawlingContext);
1690
+ try {
1691
+ await this.#errorHandler?.(crawlingContext, // valid cast - ExtendedContext transitively extends CrawlingContext
1692
+ error);
1693
+ }
1694
+ catch (handlerError) {
1695
+ if (handlerError instanceof ContextPipelineInterruptedError) {
1696
+ const skipError = await this.#trySkipRequest(handlerError, request, source);
1697
+ if (!skipError) {
1698
+ return;
1699
+ }
1700
+ // Retry for the original error. Routing `skipError` through `errorHandler` again
1701
+ // would loop forever with a handler that always skips.
1702
+ this.log.warning(`Skipping request ${request.url} failed, retrying it instead.`, {
1703
+ error: skipError.message,
1704
+ });
1705
+ }
1706
+ else {
1707
+ throw handlerError;
1708
+ }
1709
+ }
1136
1710
  if (error instanceof SessionError) {
1137
- await this._rotateSession(crawlingContext);
1711
+ crawlingContext.session?.retire();
1138
1712
  }
1139
1713
  if (!request.noRetry) {
1140
1714
  request.retryCount++;
1715
+ this.statistics.registerRetry?.(request.retryCount);
1141
1716
  const { url, retryCount, id } = request;
1142
1717
  // We don't want to see the stack trace in the logs by default, when we are going to retry the request.
1143
1718
  // Thus, we print the full stack trace only when CRAWLEE_VERBOSE_LOG environment variable is set to true.
1144
- const message = this._getMessageFromError(error);
1719
+ const message = this.getMessageFromError(error);
1145
1720
  this.log.warning(`Reclaiming failed request back to the list or queue. ${message}`, {
1146
1721
  id,
1147
1722
  url,
1148
1723
  retryCount,
1149
1724
  });
1150
- await source.reclaimRequest(request, { forefront: request.userData?.__crawlee?.forefront });
1725
+ await source.reclaimRequest(request);
1151
1726
  return;
1152
1727
  }
1153
1728
  }
1729
+ if (error instanceof SessionError) {
1730
+ crawlingContext.session?.retire();
1731
+ }
1154
1732
  // If the request is non-retryable, the error and snapshot aren't saved in the errorTrackerRetry object.
1155
1733
  // Therefore, we pass the crawlingContext to the errorTracker.add method, enabling snapshot capture.
1156
1734
  // This is to make sure the error snapshot is not duplicated in the errorTrackerRetry and errorTracker objects.
1157
1735
  const { noRetry, maxRetries } = request;
1158
1736
  if (noRetry || !maxRetries) {
1159
- await this.stats.errorTracker.addAsync(error, crawlingContext);
1737
+ await this.statistics.errorTracker.addAsync(error, crawlingContext);
1160
1738
  }
1161
1739
  else {
1162
- this.stats.errorTracker.add(error);
1740
+ this.statistics.errorTracker.add(error);
1163
1741
  }
1164
1742
  // If we get here, the request is either not retryable
1165
1743
  // or failed more than retryCount times and will not be retried anymore.
1166
1744
  // Mark the request as failed and do not retry.
1167
- this.handledRequestsCount++;
1168
- await source.markRequestHandled(request);
1169
- this.stats.failJob(request.id || request.uniqueKey, request.retryCount);
1170
- await this._handleFailedRequestHandler(crawlingContext, error); // This function prints an error message.
1745
+ await source.markRequestAsHandled(request);
1746
+ this.statistics.recordRequestFailure(request.id || request.uniqueKey, request.retryCount);
1747
+ await this.handleFailedRequestHandler(crawlingContext, error); // This function prints an error message.
1171
1748
  }
1172
- async _tagUserHandlerError(cb) {
1749
+ /** Settles a skipped request. Returns the error that prevented that, leaving the request untouched if `onSkippedRequest` failed. */
1750
+ async #trySkipRequest(error, request, source) {
1173
1751
  try {
1174
- return (await cb());
1752
+ await addTimeoutToPromise(async () => this.#handleSkippedRequest({ request, reason: error.reason, message: error.skipMessage }), this.internalTimeoutMillis, `onSkippedRequest for ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
1753
+ request.state = RequestState.SKIPPED;
1754
+ request.noRetry = true;
1755
+ await this.#timeoutAndRetry(async () => source.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${request.url} (${request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
1756
+ this.statistics.discardRequestRecord(request.id || request.uniqueKey);
1757
+ return undefined;
1175
1758
  }
1176
- catch (e) {
1177
- Object.defineProperty(e, 'triggeredFromUserHandler', { value: true });
1178
- throw e;
1759
+ catch (settleError) {
1760
+ return this.#unwrapError(settleError);
1179
1761
  }
1180
1762
  }
1181
- async _handleFailedRequestHandler(crawlingContext, error) {
1763
+ // oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
1764
+ async handleFailedRequestHandler(crawlingContext, error) {
1182
1765
  // Always log the last error regardless if the user provided a failedRequestHandler
1183
1766
  const { id, url, method, uniqueKey } = crawlingContext.request;
1184
- const message = this._getMessageFromError(error, true);
1767
+ const message = this.getMessageFromError(error, true);
1185
1768
  this.log.error(`Request failed and reached maximum retries. ${message}`, { id, url, method, uniqueKey });
1186
- if (this.failedRequestHandler) {
1187
- await this.failedRequestHandler?.(crawlingContext, // valid cast - ExtendedContext transitively extends CrawlingContext
1769
+ try {
1770
+ await this.#failedRequestHandler?.(crawlingContext, // valid cast - ExtendedContext transitively extends CrawlingContext
1188
1771
  error);
1189
1772
  }
1773
+ catch (handlerError) {
1774
+ if (!(handlerError instanceof ContextPipelineInterruptedError))
1775
+ throw handlerError;
1776
+ this.log.warning(`skipRequest() has no effect in failedRequestHandler, request ${url} (${id}) has already failed.`);
1777
+ }
1190
1778
  }
1191
1779
  /**
1192
1780
  * Resolves the most verbose error message from a thrown error
1193
1781
  * @param error The error received
1194
1782
  * @returns The message to be logged
1195
1783
  */
1196
- _getMessageFromError(error, forceStack = false) {
1784
+ getMessageFromError(error, forceStack = false) {
1197
1785
  if ([TypeError, SyntaxError, ReferenceError].some((type) => error instanceof type)) {
1198
1786
  forceStack = true;
1199
1787
  }
@@ -1201,17 +1789,25 @@ export class BasicCrawler {
1201
1789
  const baseDir = process.cwd();
1202
1790
  const userLine = stackLines.find((line) => line.includes(baseDir) && !line.includes('node_modules'));
1203
1791
  if (error instanceof TimeoutError) {
1204
- return process.env.CRAWLEE_VERBOSE_LOG ? error.stack : error.message || error; // stack in timeout errors does not really help
1792
+ // stack in timeout errors does not really help
1793
+ return (process.env.CRAWLEE_VERBOSE_LOG && error.stack) || error.message;
1205
1794
  }
1206
1795
  return process.env.CRAWLEE_VERBOSE_LOG || forceStack
1207
1796
  ? (error.stack ?? [error.message || error, ...stackLines].join('\n'))
1208
1797
  : [error.message || error, userLine].join('\n');
1209
1798
  }
1210
- _canRequestBeRetried(request, error) {
1211
- // Request should never be retried, or the error encountered makes it not able to be retried, or the session rotation limit has been reached
1212
- if (request.noRetry ||
1213
- error instanceof NonRetryableError ||
1214
- (error instanceof SessionError && this.maxSessionRotations <= (request.sessionRotationCount ?? 0))) {
1799
+ /**
1800
+ * Whether the session should be spared for this error - either because it was already retired, or because the
1801
+ * failure says nothing about the session (a rate limit is a property of the domain).
1802
+ */
1803
+ #errorAbsolvesSession(error) {
1804
+ return (error instanceof SessionError ||
1805
+ error instanceof RequestThrottledError ||
1806
+ error instanceof ContextPipelineInterruptedError);
1807
+ }
1808
+ #canRequestBeRetried(request, error) {
1809
+ // Request should never be retried, or the error encountered makes it not able to be retried.
1810
+ if (request.noRetry || error instanceof NonRetryableError) {
1215
1811
  return false;
1216
1812
  }
1217
1813
  // User requested retry (we ignore retry count here as its explicitly told by the user to retry)
@@ -1219,61 +1815,68 @@ export class BasicCrawler {
1219
1815
  return true;
1220
1816
  }
1221
1817
  // Ensure there are more retries available for the request
1222
- const maxRequestRetries = request.maxRetries ?? this.maxRequestRetries;
1818
+ const maxRequestRetries = request.maxRetries ?? this.#maxRequestRetries;
1223
1819
  return request.retryCount < maxRequestRetries;
1224
1820
  }
1225
1821
  /**
1226
- * Updates handledRequestsCount from possibly stored counts, usually after worker migration.
1227
- */
1228
- async _loadHandledRequestCount() {
1229
- if (this.requestManager) {
1230
- this.handledRequestsCount = await this.requestManager.handledCount();
1231
- }
1232
- }
1233
- async _executeHooks(hooks, ...args) {
1234
- if (Array.isArray(hooks) && hooks.length) {
1235
- for (const hook of hooks) {
1236
- await hook(...args);
1237
- }
1238
- }
1239
- }
1240
- /**
1241
- * Stops the crawler immediately.
1822
+ * Ends the run in progress immediately, without waiting for the requests already in flight.
1242
1823
  *
1243
- * This method doesn't wait for currently active requests to finish.
1824
+ * This runs at the end of every {@link BasicCrawler.run|`run()`}, so what it releases is per-run only, and
1825
+ * a repeated `run()` has to find everything it needs either intact or re-establishable. What outlives a run is
1826
+ * released by {@link BasicCrawler.destroy|`destroy()`} instead.
1244
1827
  *
1245
1828
  * To stop the crawler gracefully (waiting for all running requests to finish), use {@link BasicCrawler.stop|`crawler.stop()`} instead.
1246
1829
  */
1247
1830
  async teardown() {
1248
- serviceLocator.getEventManager().emit("persistState" /* EventType.PERSIST_STATE */, { isMigrating: false });
1249
- await this.sessionPool?.teardown();
1250
- if (this._closeEvents) {
1251
- await serviceLocator.getEventManager().close();
1252
- }
1253
- await this.autoscaledPool?.abort();
1831
+ const eventManager = serviceLocator.getEventManager();
1832
+ const ownsEventManager = this.#run?.ownsEventManager === true;
1833
+ // First, so that the pool writes its final state once - its teardown stops it listening before the
1834
+ // persistence event below goes out.
1835
+ await this.#sessionPoolDep.ifOwned(async (pool) => pool.teardown());
1836
+ // When this crawler initialized the event manager, its close() call emits
1837
+ // the final persistence event after the crawler-specific state has been
1838
+ // saved. External event managers still need an explicit event here.
1839
+ if (!ownsEventManager) {
1840
+ eventManager.emit(EventType.PERSIST_STATE, { isMigrating: false });
1841
+ }
1842
+ if (ownsEventManager) {
1843
+ await eventManager.close();
1844
+ }
1845
+ await this.#run?.end();
1254
1846
  }
1255
- _getCookieHeaderFromRequest(request) {
1847
+ /**
1848
+ * Releases what the crawler owns beyond a single run — in the browser crawlers, the browser pool. A finished
1849
+ * `run()` has already released everything a run owns, so this is only needed before dropping a crawler you
1850
+ * could otherwise have run again.
1851
+ */
1852
+ async destroy() {
1853
+ // An abandoned run - one whose `run()` never returned - would otherwise keep dispatching requests.
1854
+ await this.#run?.end();
1855
+ }
1856
+ async [Symbol.asyncDispose]() {
1857
+ await this.destroy();
1858
+ }
1859
+ getCookieHeaderFromRequest(request) {
1256
1860
  if (request.headers?.Cookie && request.headers?.cookie) {
1257
1861
  this.log.warning(`Encountered mixed casing for the cookie headers for request ${request.url} (${request.id}). Their values will be merged.`);
1258
1862
  return mergeCookies(request.url, [request.headers.cookie, request.headers.Cookie]);
1259
1863
  }
1260
1864
  return request.headers?.Cookie || request.headers?.cookie || '';
1261
1865
  }
1262
- async _getRequestQueue() {
1263
- // Check if it's explicitly disabled
1264
- if (this.experiments.requestLocking === false) {
1265
- if (!this._experimentWarnings.requestLocking) {
1266
- this.log.info('Using the old RequestQueue implementation without request locking.');
1267
- this._experimentWarnings.requestLocking = true;
1866
+ #requestMatchesEnqueueStrategy(request) {
1867
+ // If `skipNavigation` was used, just return `true`
1868
+ try {
1869
+ // eslint-disable-next-line @typescript-eslint/no-unused-expressions
1870
+ request.loadedUrl;
1871
+ }
1872
+ catch (err) {
1873
+ if (err instanceof NavigationSkippedError) {
1874
+ return true;
1268
1875
  }
1269
- return RequestQueueV1.open(null, { config: serviceLocator.getConfiguration() });
1876
+ throw err;
1270
1877
  }
1271
- return RequestQueue.open(null, { config: serviceLocator.getConfiguration() });
1272
- }
1273
- requestMatchesEnqueueStrategy(request) {
1274
1878
  const { url, loadedUrl } = request;
1275
- // eslint-disable-next-line dot-notation -- private access
1276
- const strategy = request['enqueueStrategy'];
1879
+ const strategy = request.enqueueStrategy;
1277
1880
  // No strategy set, so we assume it matches, or it was added outside of enqueueLinks
1278
1881
  if (!strategy) {
1279
1882
  return true;
@@ -1282,57 +1885,13 @@ export class BasicCrawler {
1282
1885
  if (!loadedUrl) {
1283
1886
  return true;
1284
1887
  }
1285
- const baseUrl = new URL(url);
1286
- const loadedBaseUrl = new URL(loadedUrl);
1287
- switch (strategy) {
1288
- case EnqueueStrategy.SameHostname: {
1289
- return baseUrl.hostname === loadedBaseUrl.hostname;
1290
- }
1291
- case EnqueueStrategy.SameDomain: {
1292
- const baseUrlHostname = getDomain(baseUrl.hostname, { mixedInputs: false });
1293
- if (baseUrlHostname) {
1294
- const loadedBaseUrlHostname = getDomain(loadedBaseUrl.hostname, { mixedInputs: false });
1295
- return baseUrlHostname === loadedBaseUrlHostname;
1296
- }
1297
- // Can happen for IPs, we just check like same origin
1298
- return baseUrl.origin === loadedBaseUrl.origin;
1299
- }
1300
- case EnqueueStrategy.SameOrigin: {
1301
- // Same as hostname, but also checks protocol
1302
- return baseUrl.origin === loadedBaseUrl.origin;
1303
- }
1304
- case EnqueueStrategy.All:
1305
- default: {
1306
- return baseUrl.protocol === 'http:' || baseUrl.protocol === 'https:';
1307
- }
1308
- }
1888
+ return matchesEnqueueStrategy(strategy, new URL(loadedUrl), new URL(url));
1309
1889
  }
1310
1890
  }
1311
- /**
1312
- * Creates new {@link Router} instance that works based on request labels.
1313
- * This instance can then serve as a {@link BasicCrawlerOptions.requestHandler|`requestHandler`} of our {@link BasicCrawler}.
1314
- * Defaults to the {@link BasicCrawlingContext}.
1315
- *
1316
- * > Serves as a shortcut for using `Router.create<BasicCrawlingContext>()`.
1317
- *
1318
- * ```ts
1319
- * import { BasicCrawler, createBasicRouter } from 'crawlee';
1320
- *
1321
- * const router = createBasicRouter();
1322
- * router.addHandler('label-a', async (ctx) => {
1323
- * ctx.log.info('...');
1324
- * });
1325
- * router.addDefaultHandler(async (ctx) => {
1326
- * ctx.log.info('...');
1327
- * });
1328
- *
1329
- * const crawler = new BasicCrawler({
1330
- * requestHandler: router,
1331
- * });
1332
- * await crawler.run();
1333
- * ```
1334
- */
1891
+ /** The hostname of `url`, falling back to the whole string when it is not parseable - for log messages only. */
1892
+ function hostnameOrUrl(url) {
1893
+ return URL.canParse(url) ? new URL(url).hostname : url;
1894
+ }
1335
1895
  export function createBasicRouter(routes) {
1336
1896
  return Router.create(routes);
1337
1897
  }
1338
- //# sourceMappingURL=basic-crawler.js.map