@crawlee/basic 4.0.0-rc.0 → 4.0.0-rc.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/README.md +1 -1
  2. package/index.d.ts +11 -1
  3. package/index.js +11 -0
  4. package/internals/autoscaling/autoscaled-pool.d.ts +178 -0
  5. package/internals/autoscaling/autoscaled-pool.js +389 -0
  6. package/internals/autoscaling/concurrency-system.d.ts +235 -0
  7. package/internals/autoscaling/concurrency-system.js +357 -0
  8. package/internals/autoscaling/cpu-load-signal.d.ts +43 -0
  9. package/internals/autoscaling/cpu-load-signal.js +46 -0
  10. package/internals/autoscaling/event-loop-load-signal.d.ts +51 -0
  11. package/internals/autoscaling/event-loop-load-signal.js +60 -0
  12. package/internals/autoscaling/index.d.ts +9 -0
  13. package/internals/autoscaling/index.js +9 -0
  14. package/internals/autoscaling/load-signal.d.ts +98 -0
  15. package/internals/autoscaling/load-signal.js +103 -0
  16. package/internals/autoscaling/memory-load-signal.d.ts +45 -0
  17. package/internals/autoscaling/memory-load-signal.js +104 -0
  18. package/internals/autoscaling/snapshotter.d.ts +84 -0
  19. package/internals/autoscaling/snapshotter.js +67 -0
  20. package/internals/autoscaling/storage-backend-load-signal.d.ts +56 -0
  21. package/internals/autoscaling/storage-backend-load-signal.js +73 -0
  22. package/internals/autoscaling/system-status.d.ts +112 -0
  23. package/internals/autoscaling/system-status.js +139 -0
  24. package/internals/autoscaling/weighted-avg.d.ts +5 -0
  25. package/internals/autoscaling/weighted-avg.js +14 -0
  26. package/internals/basic-crawler.d.ts +84 -312
  27. package/internals/basic-crawler.js +709 -598
  28. package/internals/cookie-utils.d.ts +40 -0
  29. package/internals/cookie-utils.js +122 -0
  30. package/internals/crawler-run.d.ts +76 -0
  31. package/internals/crawler-run.js +144 -0
  32. package/internals/crawlers/context-pipeline.d.ts +75 -0
  33. package/internals/crawlers/context-pipeline.js +109 -0
  34. package/internals/crawlers/crawler-commons.d.ts +216 -0
  35. package/internals/crawlers/crawler-commons.js +1 -0
  36. package/internals/crawlers/error-snapshotter.d.ts +29 -0
  37. package/internals/crawlers/error-snapshotter.js +116 -0
  38. package/internals/crawlers/error-tracker.d.ts +51 -0
  39. package/internals/crawlers/error-tracker.js +316 -0
  40. package/internals/crawlers/index.d.ts +4 -0
  41. package/internals/crawlers/index.js +3 -0
  42. package/internals/crawlers/internals/types.d.ts +7 -0
  43. package/internals/crawlers/internals/types.js +1 -0
  44. package/internals/crawlers/statistics.d.ts +326 -0
  45. package/internals/crawlers/statistics.js +559 -0
  46. package/internals/crawling-request.d.ts +86 -0
  47. package/internals/crawling-request.js +127 -0
  48. package/internals/enqueue-links/enqueue-links.d.ts +147 -0
  49. package/internals/enqueue-links/enqueue-links.js +35 -0
  50. package/internals/enqueue-links/index.d.ts +3 -0
  51. package/internals/enqueue-links/index.js +3 -0
  52. package/internals/enqueue-links/shared.d.ts +107 -0
  53. package/internals/enqueue-links/shared.js +228 -0
  54. package/internals/errors.d.ts +73 -0
  55. package/internals/errors.js +108 -0
  56. package/internals/http.d.ts +9 -0
  57. package/internals/http.js +28 -0
  58. package/internals/request-timeout.d.ts +6 -0
  59. package/internals/request-timeout.js +27 -2
  60. package/internals/router.d.ts +299 -0
  61. package/internals/router.js +312 -0
  62. package/internals/send-request.js +17 -8
  63. package/internals/session-pool/consts.d.ts +1 -0
  64. package/internals/session-pool/consts.js +1 -0
  65. package/internals/session-pool/errors.d.ts +7 -0
  66. package/internals/session-pool/errors.js +11 -0
  67. package/internals/session-pool/fingerprint.d.ts +9 -0
  68. package/internals/session-pool/fingerprint.js +30 -0
  69. package/internals/session-pool/index.d.ts +4 -0
  70. package/internals/session-pool/index.js +4 -0
  71. package/internals/session-pool/session-pool.d.ts +193 -0
  72. package/internals/session-pool/session-pool.js +453 -0
  73. package/internals/session-pool/session.d.ts +151 -0
  74. package/internals/session-pool/session.js +225 -0
  75. package/internals/sitemap-request-loader.d.ts +165 -0
  76. package/internals/sitemap-request-loader.js +492 -0
  77. package/internals/throttling-request-manager.d.ts +229 -0
  78. package/internals/throttling-request-manager.js +860 -0
  79. package/package.json +19 -15
@@ -1,16 +1,29 @@
1
1
  import { mkdir, writeFile } from 'node:fs/promises';
2
2
  import { dirname } from 'node:path';
3
- import { applyRequestTransform, AutoscaledPool, bindMethodsToServiceLocator, BLOCKED_STATUS_CODES, buildEnqueueStrategyPatterns, ConcurrencySystem, Configuration, constructUrlPatternObjects, ContextPipeline, ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, createRequestOptions, createStorageTransaction, Request, CriticalError, currentStorageTransaction, Dataset, EnqueueStrategy, EventManager, EventType, filterRequestOptionsByPatterns, getObjectType, KeyValueStore, log, LogLevel, mergeCookies, MissingSessionError, NavigationSkippedError, NonRetryableError, OwnedOrInjected, parseArgument, purgeDefaultStorages, RequestHandlerError, parseRetryAfterHeader, RequestThrottledError, RequestManagerTandem, RequestQueue, RequestState, RetryRequestError, supportsDomainThrottling, Router, schemas, ServiceLocator, serviceLocator, Session, SessionError, SessionPool, Statistics, ThrottlingRequestManager, validateUserData, validators, withDirectStorageAccess, } from '@crawlee/core';
3
+ import { bindMethodsToServiceLocator, Configuration, createStorageTransaction, Request, CriticalError, currentStorageTransaction, Dataset, EnqueueStrategy, EventManager, EventType, getObjectType, KeyValueStore, log, NonRetryableError, OwnedOrInjected, purgeDefaultStorages, RequestManagerTandem, RequestQueue, ServiceLocator, serviceLocator, SessionError, validators, withDirectStorageAccess, } from '@crawlee/core';
4
4
  import { BaseHttpClient, FetchHttpClient } from '@crawlee/http-client';
5
- import { isAsyncIterable, isIterable, ROTATE_PROXY_ERRORS } from '@crawlee/utils/internal';
5
+ import { filterUrl, isAsyncIterable, isIterable, matchesEnqueueStrategy, parseArgument, ROTATE_PROXY_ERRORS, schemas, } from '@crawlee/utils/internal';
6
6
  import { RobotsTxtFile } from '@crawlee/utils';
7
- import { getDomain } from 'tldts';
8
7
  import { z } from 'zod';
9
8
  import { LruCache } from '@apify/datastructures';
10
- import { addTimeoutToPromise, extendTimeout, TimeoutError } from '@apify/timeout';
9
+ import { addTimeoutToPromise, extendTimeout, storage as timeoutStorage, TimeoutError, tryCancel } from '@apify/timeout';
11
10
  import { cryptoRandomObjectId } from '@apify/utilities';
12
- import { extendTimeoutKey, navigationDeadlineKey, raceWithTimeout, timeoutExpiredKey, } from './request-timeout.js';
11
+ import { ConcurrencySystem } from './autoscaling/concurrency-system.js';
12
+ import { mergeCookies } from './cookie-utils.js';
13
+ import { ContextPipeline } from './crawlers/context-pipeline.js';
14
+ import { CrawlingRequest, RequestState } from './crawling-request.js';
15
+ import { ContextPipelineCleanupError, ContextPipelineInterruptedError, MissingSessionError, NavigationSkippedError, PersistentRateLimitError, ContextPipelineInitializationError, RequestHandlerError, RequestThrottledError, RetryRequestError, } from './errors.js';
16
+ import { Statistics } from './crawlers/statistics.js';
17
+ import { applyRequestTransform, constructUrlPatternObjects, createRequestOptions, createSkippedRequestArgs, filterRequestOptionsByPatterns, } from './enqueue-links/index.js';
18
+ import { parseRetryAfterHeader } from './http.js';
19
+ import { CrawlerRun } from './crawler-run.js';
20
+ import { extendTimeoutKey, navigationDeadlineKey, raceWithTimeout, suspendTimeoutKey, timeoutExpiredKey, } from './request-timeout.js';
13
21
  import { createSendRequest } from './send-request.js';
22
+ import { Router, validateUserData } from './router.js';
23
+ import { BLOCKED_STATUS_CODES } from './session-pool/consts.js';
24
+ import { Session } from './session-pool/session.js';
25
+ import { SessionPool } from './session-pool/session-pool.js';
26
+ import { ThrottlingRequestManager } from './throttling-request-manager.js';
14
27
  class LazyDefaultHttpClient extends BaseHttpClient {
15
28
  #delegatePromise;
16
29
  constructor(options) {
@@ -40,7 +53,6 @@ class LazyDefaultHttpClient extends BaseHttpClient {
40
53
  * @ignore
41
54
  */
42
55
  const SAFE_MIGRATION_WAIT_MILLIS = 20000;
43
- const deferredCleanupKey = Symbol('deferredCleanup');
44
56
  // The request timeout plumbing (the window helper, the context symbols, and the race) lives in its own module.
45
57
  export { navigationDeadlineKey, remainingNavigationWindowMillis } from './request-timeout.js';
46
58
  const urlPatternSchema = z.union([
@@ -71,14 +83,13 @@ const addRequestsOptionsSchema = z.looseObject({
71
83
  onSkippedRequest: schemas.anyFunction.optional(),
72
84
  });
73
85
  export class BasicCrawler {
74
- static CRAWLEE_STATE_KEY = 'CRAWLEE_STATE';
86
+ static #CRAWLEE_STATE_KEY = 'CRAWLEE_STATE';
75
87
  /**
76
88
  * Tracks the number of crawler instances created. The first crawler uses the default
77
89
  * request queue; subsequent ones get their own queue via a unique alias so they don't
78
90
  * collide.
79
91
  */
80
- // kept as TS-private: tests reset the counter at runtime
81
- static instanceCount = 0;
92
+ static #instanceCount = 0;
82
93
  /**
83
94
  * Tracks crawler instances that accessed shared state without having an explicit id.
84
95
  * Used to detect and warn about multiple crawlers sharing the same state.
@@ -93,12 +104,15 @@ export class BasicCrawler {
93
104
  get statistics() {
94
105
  return this.#statisticsDep.value;
95
106
  }
107
+ #requestManager;
96
108
  /**
97
109
  * The main request-handling component of the crawler. It manages the requests that the crawler processes,
98
110
  * combining any provided request loader and/or queue. It's initialized during the crawler startup or lazily
99
111
  * via {@link BasicCrawler.getRequestManager|`getRequestManager()`}.
100
112
  */
101
- requestManager;
113
+ get requestManager() {
114
+ return this.#requestManager;
115
+ }
102
116
  /** Backs the {@link BasicCrawler.sessionPool|`sessionPool`} getter. */
103
117
  #sessionPoolDep;
104
118
  /**
@@ -108,12 +122,6 @@ export class BasicCrawler {
108
122
  get sessionPool() {
109
123
  return this.#sessionPoolDep.value;
110
124
  }
111
- /**
112
- * Tracks **only** the queue the crawler opens for itself — not the {@link RequestManagerTandem} that may wrap it
113
- * around a user-supplied `requestList` — so the owned-only purge between repeated `run()` calls never reaches
114
- * through to a borrowed loader. Filled lazily in {@link BasicCrawler.openOwnedRequestQueue|`openOwnedRequestQueue()`}.
115
- */
116
- #ownedRequestQueue = OwnedOrInjected.resolve();
117
125
  /**
118
126
  * Whether the request-processing-time hint has already been forwarded to the request manager. The hint
119
127
  * derives only from `requestHandlerTimeoutMillis` (constant for the crawler's lifetime) and is raise-only,
@@ -126,31 +134,36 @@ export class BasicCrawler {
126
134
  * the concurrency shortcuts folded in (owned, so the crawler starts and stops it).
127
135
  */
128
136
  #resolveConcurrencySystem;
129
- /** As resolved by `init()`. Absent until the first run, so a `teardown()` before it is a no-op. */
130
- #concurrencySystemDep;
137
+ /** The last run started on this instance. It outlives the run, so anything that drives one goes through `#liveRun`. */
138
+ #run;
139
+ #concurrencySystem;
140
+ /** The run in progress, if any — a finished run is history and cannot be driven. */
141
+ get #liveRun() {
142
+ return this.#run?.isLive === true ? this.#run : undefined;
143
+ }
144
+ /** The run whose task loop is dispatching, if any — only that one can be paused and resumed. */
145
+ get #dispatchingRun() {
146
+ return this.#liveRun?.isDispatching === true ? this.#liveRun : undefined;
147
+ }
148
+ /** Whether a {@link BasicCrawler.run|`run()`} is in progress on this instance. */
149
+ get running() {
150
+ return this.#liveRun !== undefined;
151
+ }
131
152
  /**
132
- * The concurrency governor this run is booking its requests against — either the
153
+ * The concurrency governor the run in progress is booking its requests against — either the
133
154
  * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} that was injected, or the default the
134
155
  * crawler built for itself. Read it for telemetry: `desiredConcurrency`, `currentConcurrency`, `isRunning`.
135
156
  *
136
- * > *NOTE:* `undefined` until {@link BasicCrawler.run|`crawler.run()`} has resolved it. A crawler-owned default
137
- * is also rebuilt for every run, so the instance is not stable across runs.
157
+ * > *NOTE:* `undefined` outside a run, and a crawler-owned default is rebuilt for every run — so read it
158
+ * during a run rather than caching it across runs.
138
159
  *
139
160
  * {@link IConcurrencySystem} is deliberately read-only. Tuning concurrency *while a crawl is running* means
140
161
  * owning the instance: build a {@link ConcurrencySystem} yourself and inject it, then set
141
162
  * `minConcurrency`/`maxConcurrency`/`desiredConcurrency` on your own reference.
142
163
  */
143
164
  get concurrencySystem() {
144
- return this.#concurrencySystemDep?.maybeValue;
165
+ return this.#concurrencySystem?.value;
145
166
  }
146
- /**
147
- * The task loop that dispatches this run's requests. Private on purpose — it is a bare parallel task runner with
148
- * no configuration left of its own (see {@link ConcurrencySystem}), and everything a caller legitimately did
149
- * with it now has a crawler-level counterpart: {@link BasicCrawler.pause|`pause()`},
150
- * {@link BasicCrawler.resume|`resume()`}, {@link BasicCrawler.teardown|`teardown()`} and
151
- * {@link BasicCrawler.concurrencySystem|`concurrencySystem`}.
152
- */
153
- #autoscaledPool;
154
167
  /**
155
168
  * A reference to the underlying {@link IProxyConfiguration} instance that manages the crawler's proxies.
156
169
  * Only available if used by the crawler.
@@ -168,40 +181,38 @@ export class BasicCrawler {
168
181
  * pipelines expect the basic crawler fields to already be present in the context at runtime.
169
182
  *
170
183
  * Context built with this pipeline can be passed into multiple crawler pipelines at once.
171
- * This is used e.g. in the {@link AdaptivePlaywrightCrawler|`AdaptivePlaywrightCrawler`}.
172
184
  */
173
- get basicContextPipeline() {
174
- if (this.#basicContextPipeline === undefined) {
175
- this.#basicContextPipeline = this.buildBasicContextPipeline();
176
- }
185
+ get #basicPipeline() {
186
+ this.#basicContextPipeline ??= this.#buildBasicContextPipeline();
177
187
  return this.#basicContextPipeline;
178
188
  }
179
189
  #contextPipeline;
190
+ /** @internal */
180
191
  get contextPipeline() {
181
192
  if (this.#contextPipeline === undefined) {
182
- this.#contextPipeline = this.buildFinalContextPipeline();
193
+ this.#contextPipeline = this.#buildFinalContextPipeline();
183
194
  }
184
195
  return this.#contextPipeline;
185
196
  }
186
- running = false;
187
- hasFinishedBefore = false;
188
- #unexpectedStop = false;
197
+ #hasFinishedBefore = false;
198
+ /** Whether a `run()` on this instance has already finished - a repeated one continues where it left off. */
199
+ get hasFinishedBefore() {
200
+ return this.#hasFinishedBefore;
201
+ }
189
202
  #log;
190
203
  get log() {
191
204
  return this.#log;
192
205
  }
193
206
  requestHandler;
194
- errorHandler;
195
- failedRequestHandler;
196
- // kept as TS-private: tests read it at runtime
197
- requestHandlerTimeoutMillis;
207
+ #errorHandler;
208
+ #failedRequestHandler;
209
+ #requestHandlerTimeoutMillis;
198
210
  internalTimeoutMillis;
199
- maxRequestRetries;
200
- maxCrawlDepth;
201
- #sameDomainDelaySecs;
202
- maxRequestsPerCrawl;
203
- get handledRequestsCount() {
204
- return this.statistics.state.requestsFinished + this.statistics.state.requestsFailed;
211
+ #maxRequestRetries;
212
+ #maxCrawlDepth;
213
+ #maxRequestsPerCrawl;
214
+ get #handledRequestsCount() {
215
+ return this.statistics.state.requestsSucceeded + this.statistics.state.requestsFailed;
205
216
  }
206
217
  #statusMessageLoggingInterval;
207
218
  #statusMessageCallback;
@@ -214,8 +225,7 @@ export class BasicCrawler {
214
225
  * {@link ConcurrencySystem} instead, and the loop's `consumer` identity is the crawler's own, so neither is
215
226
  * settable here.
216
227
  */
217
- // kept as TS-private: tests mutate it at runtime
218
- taskLoopOptions;
228
+ #taskLoopOptions;
219
229
  httpClient;
220
230
  retryOnBlocked;
221
231
  #respectRobotsTxtFile;
@@ -223,12 +233,13 @@ export class BasicCrawler {
223
233
  #transactionalStorageEnabled;
224
234
  /** The resolved per-storage-type write policy overrides forwarded to each request's transaction. */
225
235
  #storageWritePolicy;
226
- onSkippedRequest;
227
- #closeEvents;
228
- #loggedPerRun = new Set();
236
+ #onSkippedRequest;
229
237
  #robotsTxtFileCache;
230
- identity;
238
+ #identity;
231
239
  #contextPipelineOptions;
240
+ /**
241
+ * @internal
242
+ */
232
243
  static optionsShape = {
233
244
  contextPipelineBuilder: schemas.anyObject.optional(),
234
245
  extendContext: schemas.anyFunction.optional(),
@@ -270,6 +281,7 @@ export class BasicCrawler {
270
281
  // AutoscaledPool shorthands
271
282
  minConcurrency: schemas.anyNumber.optional(),
272
283
  maxConcurrency: schemas.anyNumber.optional(),
284
+ initialConcurrency: schemas.anyNumber.optional(),
273
285
  maxRequestsPerMinute: schemas.anyNumber
274
286
  .refine((value) => Number.isInteger(value) || value === Infinity, 'Expected an integer or infinite number')
275
287
  .refine((value) => value >= 1, 'Expected a number greater than or equal to 1')
@@ -278,12 +290,12 @@ export class BasicCrawler {
278
290
  statistics: schemas.anyObject.optional(),
279
291
  id: z.string().optional(),
280
292
  };
281
- static optionsSchema = z.strictObject(BasicCrawler.optionsShape);
293
+ static #optionsSchema = z.strictObject(BasicCrawler.optionsShape);
282
294
  /**
283
295
  * All `BasicCrawler` parameters are passed via an options object.
284
296
  */
285
297
  constructor(options = {}) {
286
- const parsedOptions = parseArgument(options, BasicCrawler.optionsSchema, 'BasicCrawlerOptions');
298
+ const parsedOptions = parseArgument(options, BasicCrawler.#optionsSchema, 'BasicCrawlerOptions');
287
299
  const {
288
300
  // oxlint-disable-next-line typescript/no-deprecated -- still accepted and folded into `requestManager` for back-compat
289
301
  requestList,
@@ -292,15 +304,18 @@ export class BasicCrawler {
292
304
  // Service locator options
293
305
  configuration, storageBackend, eventManager, logger,
294
306
  // AutoscaledPool shorthands
295
- minConcurrency, maxConcurrency, maxRequestsPerMinute, blockedStatusCodes: blockedStatusCodesInput, retryOnBlocked, respectRobotsTxtFile, transactionalStorage, onSkippedRequest, requestHandler, requestHandlerTimeoutSecs, errorHandler, failedRequestHandler, statusMessageLoggingInterval, statusMessageCallback, statistics, httpClient, id, } = parsedOptions;
307
+ minConcurrency, maxConcurrency, initialConcurrency, maxRequestsPerMinute, blockedStatusCodes: blockedStatusCodesInput, retryOnBlocked, respectRobotsTxtFile, transactionalStorage, onSkippedRequest, requestHandler, requestHandlerTimeoutSecs, errorHandler, failedRequestHandler, statusMessageLoggingInterval, statusMessageCallback, statistics, httpClient, id, } = parsedOptions;
296
308
  // All concurrency configuration lives on the `ConcurrencySystem`, so the shortcuts have nowhere to go once
297
309
  // one is supplied - and silently dropping a `maxConcurrency` the user asked for is how crawls end up
298
310
  // hammering a site.
299
311
  if (concurrencySystem !== undefined &&
300
- (minConcurrency !== undefined || maxConcurrency !== undefined || maxRequestsPerMinute !== undefined)) {
301
- throw new Error('The `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute` shortcuts cannot be combined with ' +
302
- '`concurrencySystem` - they configure the default `ConcurrencySystem` that a supplied one ' +
303
- 'replaces. Pass them to the `ConcurrencySystem` constructor instead.');
312
+ (minConcurrency !== undefined ||
313
+ maxConcurrency !== undefined ||
314
+ initialConcurrency !== undefined ||
315
+ maxRequestsPerMinute !== undefined)) {
316
+ throw new Error('The `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute` shortcuts ' +
317
+ 'cannot be combined with `concurrencySystem` - they configure the default `ConcurrencySystem` ' +
318
+ 'that a supplied one replaces. Pass them to the `ConcurrencySystem` constructor instead.');
304
319
  }
305
320
  // Create per-crawler service locator if custom services were provided.
306
321
  // This wraps every method on the crawler instance so that calls to the global `serviceLocator`
@@ -328,50 +343,58 @@ export class BasicCrawler {
328
343
  this.#log = serviceLocator.getLogger().child({ prefix: this.constructor.name });
329
344
  // Initialize the Configuration instance to avoid lazy loading in the components
330
345
  serviceLocator.getConfiguration();
331
- const instanceIndex = BasicCrawler.instanceCount++;
332
- this.identity = { instanceIndex, hasExplicitId: id !== undefined, id: id ?? String(instanceIndex) };
333
- if (requestManager !== undefined) {
334
- if (requestList !== undefined || requestQueue !== undefined) {
335
- throw new Error('The `requestManager` option cannot be used in conjunction with `requestList` and/or `requestQueue`');
336
- }
337
- // Both would pace the same domains, from different keys and with no idea of one another.
338
- if (sameDomainDelaySecs > 0 && supportsDomainThrottling(requestManager)) {
339
- throw new Error('The `sameDomainDelaySecs` option cannot be combined with a `requestManager` that throttles ' +
340
- 'per domain on its own. Configure the delay on the manager instead, via the ' +
341
- '`minCrawlDelaySecs` option of `ThrottlingRequestManager`.');
342
- }
343
- this.requestManager = requestManager;
346
+ const instanceIndex = BasicCrawler.#instanceCount++;
347
+ this.#identity = { instanceIndex, hasExplicitId: id !== undefined, id: id ?? String(instanceIndex) };
348
+ if (requestManager !== undefined && (requestList !== undefined || requestQueue !== undefined)) {
349
+ throw new Error('The `requestManager` option cannot be used in conjunction with `requestList` and/or `requestQueue`');
344
350
  }
345
- else if (requestList !== undefined && requestQueue !== undefined) {
346
- // Combine the read-only list with the writable queue into a tandem.
347
- this.requestManager = new RequestManagerTandem(requestList, requestQueue);
351
+ const suppliedManager = requestManager ?? requestQueue;
352
+ // Offered before building a pacer of our own: anything that paces takes the floor - through any
353
+ // number of wrappers, since they all forward - so no domain ends up with two clocks.
354
+ const floorTaken = sameDomainDelaySecs > 0 &&
355
+ (suppliedManager?.recordPacingSignal({
356
+ reason: 'minIntervalEverywhere',
357
+ intervalMs: sameDomainDelaySecs * 1000,
358
+ // What `sameDomainDelaySecs` has always meant: one clock per site, subdomains included.
359
+ scope: 'registrableDomain',
360
+ }) ??
361
+ false);
362
+ const pacerNeeded = sameDomainDelaySecs > 0 && !floorTaken;
363
+ // Built here rather than at first use so it can sit *inside* the tandem below, which is where a
364
+ // loader's transferred requests pass through it.
365
+ const writableManager = pacerNeeded
366
+ ? new ThrottlingRequestManager({
367
+ domains: 'all',
368
+ minCrawlDelaySecs: sameDomainDelaySecs,
369
+ throttleBy: 'registrableDomain',
370
+ persistStateKey: `CRAWLEE_THROTTLED_DOMAINS_${this.#identity.id}`,
371
+ // A factory, because the default queue is only opened on first use.
372
+ inner: suppliedManager ?? (() => this.#openOwnedRequestQueue()),
373
+ })
374
+ : suppliedManager;
375
+ if (requestList !== undefined) {
376
+ // The list is read first, while new requests still have somewhere writable to go.
377
+ this.#requestManager = new RequestManagerTandem(requestList, writableManager ?? (() => this.#openOwnedRequestQueue()));
348
378
  }
349
- else if (requestQueue !== undefined) {
379
+ else if (writableManager !== undefined) {
350
380
  // A RequestQueue is itself a request manager.
351
- this.requestManager = requestQueue;
352
- }
353
- else if (requestList !== undefined) {
354
- // A lone read-only `requestList` (deprecated option) is combined with a lazily-opened default queue
355
- // into a tandem, so that its requests are read first and new ones can still be enqueued during the
356
- // crawl. The queue is opened on first use; the tandem also forwards `persistState()` to the loader.
357
- this.requestManager = new RequestManagerTandem(requestList, () => this.openOwnedRequestQueue());
381
+ this.#requestManager = writableManager;
358
382
  }
359
383
  this.httpClient = httpClient ?? new LazyDefaultHttpClient({ logger: this.log });
360
384
  this.proxyConfiguration = proxyConfiguration;
361
385
  this.#statusMessageLoggingInterval = statusMessageLoggingInterval;
362
386
  this.#statusMessageCallback = statusMessageCallback;
363
387
  this.#robotsTxtFileCache = new LruCache({ maxLength: 1000 });
364
- this.handleSkippedRequest = this.handleSkippedRequest.bind(this);
365
388
  this.additionalHttpErrorStatusCodes = new Set([...additionalHttpErrorStatusCodes]);
366
389
  this.#ignoreHttpErrorStatusCodes = new Set([...ignoreHttpErrorStatusCodes]);
367
390
  this.requestHandler = requestHandler ?? this.router;
368
- this.failedRequestHandler = failedRequestHandler;
369
- this.errorHandler = errorHandler;
391
+ this.#failedRequestHandler = failedRequestHandler;
392
+ this.#errorHandler = errorHandler;
370
393
  if (requestHandlerTimeoutSecs) {
371
- this.requestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
394
+ this.#requestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
372
395
  }
373
396
  else {
374
- this.requestHandlerTimeoutMillis = 60_000;
397
+ this.#requestHandlerTimeoutMillis = 60_000;
375
398
  }
376
399
  this.retryOnBlocked = retryOnBlocked;
377
400
  this.#respectRobotsTxtFile = respectRobotsTxtFile;
@@ -379,14 +402,13 @@ export class BasicCrawler {
379
402
  const transactionalStorageOption = transactionalStorage;
380
403
  this.#transactionalStorageEnabled = transactionalStorageOption !== false;
381
404
  this.#storageWritePolicy = typeof transactionalStorageOption === 'object' ? transactionalStorageOption : {};
382
- this.onSkippedRequest = onSkippedRequest;
405
+ this.#onSkippedRequest = onSkippedRequest;
383
406
  // allow at least 5min for internal timeouts
384
407
  this.internalTimeoutMillis =
385
408
  serviceLocator.getConfiguration().internalTimeoutMillis ??
386
- Math.max(this.requestHandlerTimeoutMillis * 2, 300e3);
387
- this.maxRequestRetries = maxRequestRetries;
388
- this.maxCrawlDepth = maxCrawlDepth;
389
- this.#sameDomainDelaySecs = sameDomainDelaySecs;
409
+ Math.max(this.#requestHandlerTimeoutMillis * 2, 300e3);
410
+ this.#maxRequestRetries = maxRequestRetries;
411
+ this.#maxCrawlDepth = maxCrawlDepth;
390
412
  this.#statisticsDep = OwnedOrInjected.resolve(statistics,
391
413
  // A crawler-built default tracks the built-in fields only. A non-empty `StatisticStateExtension` can
392
414
  // only be satisfied by an injected instance carrying the matching `state`, so this branch does
@@ -394,7 +416,7 @@ export class BasicCrawler {
394
416
  () => new Statistics({
395
417
  logMessage: `${this.constructor.name} request statistics:`,
396
418
  log: this.log,
397
- id: this.identity.id,
419
+ id: this.#identity.id,
398
420
  }));
399
421
  if (sessionPool && proxyConfiguration) {
400
422
  this.log.warning('Both `sessionPool` and `proxyConfiguration` were provided to the crawler. ' +
@@ -405,19 +427,21 @@ export class BasicCrawler {
405
427
  this.#sessionPoolDep = OwnedOrInjected.resolve(sessionPool, () => new SessionPool({
406
428
  createSessionFunction: async (opts) => new Session({
407
429
  ...opts?.sessionOptions,
408
- proxyInfo: opts?.sessionOptions?.proxyInfo ?? (await this.proxyConfiguration?.newProxyInfo()),
430
+ proxyInfo: this.proxyConfiguration
431
+ ? await this.proxyConfiguration.newProxyInfo(opts?.sessionOptions?.proxyInfo)
432
+ : opts?.sessionOptions?.proxyInfo,
409
433
  }),
410
434
  }));
411
435
  this.blockedStatusCodes = new Set(blockedStatusCodesInput ?? BLOCKED_STATUS_CODES);
412
436
  const maxSignedInteger = 2 ** 31 - 1;
413
- if (this.requestHandlerTimeoutMillis > maxSignedInteger) {
414
- this.log.warning(`requestHandlerTimeoutMillis ${this.requestHandlerTimeoutMillis}` +
437
+ if (this.#requestHandlerTimeoutMillis > maxSignedInteger) {
438
+ this.log.warning(`requestHandlerTimeoutMillis ${this.#requestHandlerTimeoutMillis}` +
415
439
  ` does not fit a signed 32-bit integer. Limiting the value to ${maxSignedInteger}`);
416
- this.requestHandlerTimeoutMillis = maxSignedInteger;
440
+ this.#requestHandlerTimeoutMillis = maxSignedInteger;
417
441
  }
418
442
  this.internalTimeoutMillis = Math.min(this.internalTimeoutMillis, maxSignedInteger);
419
- this.maxRequestsPerCrawl = maxRequestsPerCrawl;
420
- const isMaxPagesExceeded = () => this.maxRequestsPerCrawl && this.maxRequestsPerCrawl <= this.handledRequestsCount;
443
+ this.#maxRequestsPerCrawl = maxRequestsPerCrawl;
444
+ const isMaxPagesExceeded = () => this.#maxRequestsPerCrawl && this.#maxRequestsPerCrawl <= this.#handledRequestsCount;
421
445
  // eslint-disable-next-line prefer-const
422
446
  let { isFinishedFunction, isTaskReadyFunction } = taskLoopOptions;
423
447
  // override even if `isFinishedFunction` provided by user - `keepAlive` has higher priority
@@ -429,90 +453,74 @@ export class BasicCrawler {
429
453
  const source = this.requestManager;
430
454
  if (!source)
431
455
  throw new Error('Request provider is not initialized!');
432
- const request = await this.resolveRequest();
456
+ const request = await this.#resolveRequest();
433
457
  if (!request) {
434
458
  return;
435
459
  }
436
460
  // Started here, rather than in `handleRequest`, so that a failure during context pipeline
437
461
  // initialization (e.g. a browser page timing out before the request handler ever runs) is
438
- // still accounted for by `failJob` below - which is a no-op without a matching `startJob`.
439
- this.statistics.startJob(request.id || request.uniqueKey);
462
+ // still accounted for by `recordRequestFailure` below - which is a no-op without a
463
+ // matching `recordRequestStart`.
464
+ this.statistics.recordRequestStart(request.id || request.uniqueKey);
440
465
  const crawlingContext = { request };
441
466
  try {
442
467
  // The transaction spans the whole pipeline call, covering the navigation hooks
443
- // and `extendContext` too; `handleRequest` drives its outcome explicitly.
468
+ // and `extendContext` too; `handleRequest` commits it and `handleRequestFailure` rolls it back.
444
469
  await this.runInStorageTransaction(async () =>
445
470
  // Navigation, the navigation hooks and the request handler are timed individually, but the
446
471
  // phases between them are not, so a request could still get stuck indefinitely. This is the
447
472
  // catch-all for that - see `raceWithTimeout` for why it is a bare timer, not a timeout frame.
448
- await this.withRequestTimeout(crawlingContext, this.basicContextPipeline
449
- .chain(this.contextPipeline)
450
- .call(crawlingContext, (ctx) => this.handleRequest(ctx, source, request))));
473
+ await this.#withRequestTimeout(crawlingContext, this.#basicPipeline.chain(this.contextPipeline).call(crawlingContext, (ctx) => this.handleRequest(ctx, source), (error) => this.#handleRequestFailure(error, crawlingContext, source))));
451
474
  }
452
475
  catch (error) {
453
- // ContextPipelineInterruptedError means the request was intentionally skipped
454
- // (e.g., doesn't match enqueue strategy after redirect). Just return gracefully.
455
- if (error instanceof ContextPipelineInterruptedError) {
456
- this.statistics.discardJob(request.id || request.uniqueKey);
457
- await this.timeoutAndRetry(async () => this.requestManager?.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${crawlingContext.request.url} (${crawlingContext.request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
458
- return;
459
- }
460
- // If the error happened during pipeline initialization (e.g., navigation timeout, session/proxy error,
461
- // i.e. not in user's requestHandler), handle it through the normal error flow. A bare `TimeoutError`
462
- // here is the internal timeout above firing - anything else thrown inside the pipeline arrives wrapped.
463
- const isPipelineError = error instanceof ContextPipelineInitializationError ||
464
- error instanceof SessionError ||
465
- error instanceof TimeoutError;
466
- if (isPipelineError) {
467
- const unwrappedError = this.unwrapError(error);
468
- await this.requestFunctionErrorHandler(unwrappedError, crawlingContext, request, this.requestManager);
469
- // SessionError already retired the session in `requestFunctionErrorHandler`;
470
- // skip `markBad` to avoid double-counting usage/error score.
471
- if (!this.errorAbsolvesSession(unwrappedError)) {
476
+ // The internal timeout above fired, or `handleRequestFailure` threw a `SessionError` - both
477
+ // take the normal error flow. Anything else is fatal.
478
+ if (error instanceof SessionError || crawlingContext[timeoutExpiredKey]?.()) {
479
+ const unwrappedError = this.#unwrapError(error);
480
+ await this.requestFunctionErrorHandler(unwrappedError, crawlingContext, this.requestManager);
481
+ if (!this.#errorAbsolvesSession(unwrappedError)) {
472
482
  crawlingContext.session?.markBad();
473
483
  }
474
484
  return;
475
485
  }
476
- throw this.unwrapError(error);
477
- }
478
- finally {
479
- // Run request-scoped deferred cleanups only after the whole request lifecycle - including the user's error handler - has finished.
480
- const deferredCleanup = crawlingContext[deferredCleanupKey] ?? [];
481
- await Promise.all(deferredCleanup.map((fn) => fn().catch((cleanupError) => this.log.debug('Error in deferred cleanup', { error: cleanupError }))));
486
+ throw this.#unwrapError(error);
482
487
  }
483
488
  },
484
489
  isTaskReadyFunction: async () => {
485
490
  if (isMaxPagesExceeded()) {
486
- this.logOncePerRun('shuttingDown', 'Crawler reached the maxRequestsPerCrawl limit of ' +
487
- `${this.maxRequestsPerCrawl} requests and will shut down soon. Requests that are in progress will be allowed to finish.`);
491
+ this.#liveRun?.logOnce('shuttingDown', 'Crawler reached the maxRequestsPerCrawl limit of ' +
492
+ `${this.#maxRequestsPerCrawl} requests and will shut down soon. Requests that are in progress will be allowed to finish.`);
488
493
  return false;
489
494
  }
490
- if (this.#unexpectedStop) {
491
- this.logOncePerRun('shuttingDown', 'No new requests are allowed because the `stop()` method has been called. ' +
495
+ if (this.#run?.stopRequested) {
496
+ this.#liveRun?.logOnce('shuttingDown', 'No new requests are allowed because the `stop()` method has been called. ' +
492
497
  'Ongoing requests will be allowed to complete.');
493
498
  return false;
494
499
  }
495
- return isTaskReadyFunction ? await isTaskReadyFunction() : await this.isTaskReadyFunction();
500
+ return isTaskReadyFunction ? await isTaskReadyFunction() : await this.#isTaskReadyFunction();
496
501
  },
497
502
  isFinishedFunction: async () => {
498
503
  if (isMaxPagesExceeded()) {
499
- this.log.info(`Earlier, the crawler reached the maxRequestsPerCrawl limit of ${this.maxRequestsPerCrawl} requests ` +
504
+ this.log.info(`Earlier, the crawler reached the maxRequestsPerCrawl limit of ${this.#maxRequestsPerCrawl} requests ` +
500
505
  'and all requests that were in progress at that time have now finished. ' +
501
- `In total, the crawler processed ${this.handledRequestsCount} requests and will shut down.`);
506
+ `In total, the crawler processed ${this.#handledRequestsCount} requests and will shut down.`);
502
507
  return true;
503
508
  }
504
- if (this.#unexpectedStop) {
509
+ if (this.#run?.stopRequested) {
505
510
  this.log.info('The crawler has finished all the remaining ongoing requests and will shut down now.');
506
511
  return true;
507
512
  }
508
- // Checked here because this runs only once nothing is in flight, which is exactly when a
509
- // crawl that cannot progress looks indistinguishable from one that is merely waiting.
510
- if (!keepAlive && supportsDomainThrottling(this.requestManager)) {
511
- await this.requestManager.assertNoStalledDomains();
513
+ // `maybeFinish()` calls this only once nothing is in flight (`autoscaled-pool.ts`) - the point
514
+ // where a crawl that cannot progress becomes distinguishable from one that is merely waiting,
515
+ // and the only place where throwing does not abandon requests mid-processing.
516
+ const state = await this.requestManager?.checkReadiness();
517
+ // Under `keepAlive`, outliving a domain that will not let us through is the whole point.
518
+ if (state?.status === 'stalled' && !keepAlive) {
519
+ throw new PersistentRateLimitError(`Giving up: ${state.reason}`);
512
520
  }
513
521
  const isFinished = isFinishedFunction
514
522
  ? await isFinishedFunction()
515
- : await this.defaultIsFinishedFunction();
523
+ : state === undefined || state.status === 'finished';
516
524
  if (isFinished) {
517
525
  const reason = isFinishedFunction
518
526
  ? "Crawler's custom isFinishedFunction() returned true, the crawler will shut down."
@@ -523,11 +531,14 @@ export class BasicCrawler {
523
531
  },
524
532
  log: this.log,
525
533
  };
526
- this.taskLoopOptions = { ...taskLoopOptions, ...crawlerOwnedTaskLoopConfiguration };
534
+ this.#taskLoopOptions = { ...taskLoopOptions, ...crawlerOwnedTaskLoopConfiguration };
527
535
  this.#resolveConcurrencySystem = () => OwnedOrInjected.resolve(concurrencySystem, () => this.createDefaultConcurrencySystem({
528
536
  minConcurrency,
529
537
  maxConcurrency,
530
538
  maxTasksPerMinute: maxRequestsPerMinute,
539
+ // Spread conditionally - an explicit `undefined` would clobber a subclass default, see
540
+ // `HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS`.
541
+ ...(initialConcurrency !== undefined && { desiredConcurrency: initialConcurrency }),
531
542
  log: this.log,
532
543
  }));
533
544
  }
@@ -537,7 +548,7 @@ export class BasicCrawler {
537
548
  }
538
549
  /**
539
550
  * Builds the crawler-owned default {@link ConcurrencySystem} from the resolved
540
- * `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute` shortcuts. Not called when a
551
+ * `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute` shortcuts. Not called when a
541
552
  * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} was injected.
542
553
  *
543
554
  * Subclasses may override this to tune the default system (e.g. {@link HttpCrawler} raises the starting
@@ -561,35 +572,21 @@ export class BasicCrawler {
561
572
  * Builds the basic context pipeline that transforms `{ request }` into a full `CrawlingContext`.
562
573
  * This handles base context creation, session resolution, and context helpers.
563
574
  */
564
- buildBasicContextPipeline() {
575
+ #buildBasicContextPipeline() {
565
576
  return ContextPipeline.create()
566
- .compose({ action: this.checkRobotsTxt.bind(this) })
567
- .compose({ action: (context) => this.createBaseContext(context) })
568
- .compose({ action: this.resolveSession.bind(this) })
569
- .compose({ action: this.createContextHelpers.bind(this) });
577
+ .compose(this.#checkRobotsTxt.bind(this))
578
+ .compose(this.#createBaseContext.bind(this))
579
+ .compose(this.#resolveSession.bind(this))
580
+ .compose(this.#createContextHelpers.bind(this));
570
581
  }
571
- async checkRobotsTxt({ request }) {
572
- if (!(await this.isAllowedBasedOnRobotsTxtFile(request.url))) {
582
+ async #checkRobotsTxt({ request }) {
583
+ if (!(await this.#isAllowedBasedOnRobotsTxtFile(request.url))) {
573
584
  this.log.warning(`Skipping request ${request.url} (${request.id}) because it is disallowed based on robots.txt`);
574
- request.state = RequestState.SKIPPED;
575
- request.noRetry = true;
576
- await this.handleSkippedRequest({
577
- url: request.url,
578
- reason: 'robotsTxt',
579
- });
580
- throw new ContextPipelineInterruptedError(`Skipping request ${request.url} as disallowed by robots.txt`);
585
+ throw new ContextPipelineInterruptedError('robotsTxt');
581
586
  }
582
587
  return {};
583
588
  }
584
- /**
585
- * Builds the subclass-specific context pipeline that transforms a `CrawlingContext` into the crawler's target context type.
586
- * Subclasses should override this to add their own pipeline stages.
587
- */
588
- buildContextPipeline() {
589
- return ContextPipeline.create();
590
- }
591
- createBaseContext(context) {
592
- const deferredCleanup = [];
589
+ #createBaseContext(context, onCleanup) {
593
590
  return {
594
591
  id: cryptoRandomObjectId(10),
595
592
  log: this.log,
@@ -597,7 +594,23 @@ export class BasicCrawler {
597
594
  useState: this.useState.bind(this),
598
595
  getKeyValueStore: async (identifier) => KeyValueStore.open(identifier),
599
596
  registerDeferredCleanup: (cleanup) => {
600
- deferredCleanup.push(cleanup);
597
+ onCleanup(async () => {
598
+ try {
599
+ await cleanup();
600
+ }
601
+ catch (error) {
602
+ this.log.debug('Error in deferred cleanup', { error });
603
+ }
604
+ });
605
+ },
606
+ afterStorageCommit: (callback) => {
607
+ const transaction = currentStorageTransaction();
608
+ if (transaction === undefined) {
609
+ throw new Error('afterStorageCommit() needs an active storage transaction, and there is none. ' +
610
+ "With `transactionalStorage: false` the request's writes are applied as they are " +
611
+ 'made, so they throw at the call site instead of at commit time.');
612
+ }
613
+ transaction.afterCommit(callback);
601
614
  },
602
615
  extendTimeout: (secs) => {
603
616
  const extraMillis = secs * 1000;
@@ -610,29 +623,43 @@ export class BasicCrawler {
610
623
  if (context[navigationDeadlineKey] !== undefined) {
611
624
  context[navigationDeadlineKey] += extraMillis;
612
625
  }
626
+ // Extension failure must not fail the request that asked for more time.
627
+ this.requestManager?.extendRequestProcessingTime?.(context.request, secs)?.catch((error) => {
628
+ this.log.debug('Extending the request processing time failed', { url: context.request.url, error });
629
+ });
630
+ },
631
+ skipRequest: (message) => {
632
+ this.log.debug(`Skipping request ${context.request.url} (${context.request.id})${message ? `: ${message}` : ''}`);
633
+ throw new ContextPipelineInterruptedError('manual', message);
613
634
  },
614
- [deferredCleanupKey]: deferredCleanup,
615
635
  };
616
636
  }
617
- async resolveRequest() {
618
- const request = await this.timeoutAndRetry(this.fetchNextRequest.bind(this), this.internalTimeoutMillis, `Fetching next request timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
619
- // Reset loadedUrl so an old one is not carried over to retries.
620
- if (request) {
621
- request.loadedUrl = undefined;
637
+ /**
638
+ * Fetches the next request and rebuilds it as a {@link CrawlingRequest}. A copy rather than the manager's
639
+ * instance: managers key on `uniqueKey`/`id`, and it is this copy that flows back into `reclaimRequest` /
640
+ * `markRequestAsHandled`, so nothing observes the difference.
641
+ */
642
+ async #resolveRequest() {
643
+ const fetched = await this.#timeoutAndRetry(this.#fetchNextRequest.bind(this), this.internalTimeoutMillis, `Fetching next request timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
644
+ if (!fetched) {
645
+ return null;
622
646
  }
647
+ const request = CrawlingRequest.fromSchema(fetched);
648
+ // Reset loadedUrl so an old one is not carried over to retries.
649
+ request.loadedUrl = undefined;
623
650
  return request;
624
651
  }
625
- async resolveSession({ request }) {
626
- const session = await this.timeoutAndRetry(async () => {
652
+ async #resolveSession({ request }) {
653
+ const session = await this.#timeoutAndRetry(async () => {
627
654
  const existingSession = await this.sessionPool.getSession(request.sessionId);
628
655
  if (!existingSession) {
629
- throw new ContextPipelineInitializationError(new MissingSessionError(request.sessionId));
656
+ throw new MissingSessionError(request.sessionId);
630
657
  }
631
658
  return existingSession;
632
659
  }, this.internalTimeoutMillis, `Fetching session timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
633
660
  return { session, proxyInfo: session?.proxyInfo };
634
661
  }
635
- async createContextHelpers({ request, session }) {
662
+ async #createContextHelpers({ request, session }) {
636
663
  const addRequests = async (requests, options = {}) => {
637
664
  const newCrawlDepth = request.crawlDepth + 1;
638
665
  const requestsGenerator = this.addCrawlDepthRequestGenerator(requests, newCrawlDepth);
@@ -641,9 +668,9 @@ export class BasicCrawler {
641
668
  const sendRequest = createSendRequest(this.httpClient, request, session);
642
669
  return { addRequests, sendRequest };
643
670
  }
644
- buildFinalContextPipeline() {
671
+ #buildFinalContextPipeline() {
645
672
  const subclassPipeline = (this.#contextPipelineOptions.contextPipelineBuilder?.() ??
646
- this.buildContextPipeline());
673
+ ContextPipeline.create());
647
674
  // `extendContext` runs *before* the subclass navigation pipeline (which includes the
648
675
  // pre/post-navigation hooks). This makes the extension visible to those hooks and to the
649
676
  // request handler alike. The trade-off is that `extendContext` cannot access
@@ -658,26 +685,21 @@ export class BasicCrawler {
658
685
  let contextPipeline;
659
686
  if (extendContext !== undefined) {
660
687
  contextPipeline = ContextPipeline.create()
661
- .compose({ action: async (context) => await extendContext(context) })
688
+ .compose(extendContext)
662
689
  .chain(subclassPipeline);
663
690
  }
664
691
  else {
665
692
  contextPipeline = subclassPipeline;
666
693
  }
667
- contextPipeline = contextPipeline.compose({
668
- action: async (context) => {
669
- const { request } = context;
670
- if (request && !this.requestMatchesEnqueueStrategy(request)) {
671
- // eslint-disable-next-line dot-notation
672
- const message = `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`;
673
- this.log.debug(message);
674
- request.noRetry = true;
675
- request.state = RequestState.SKIPPED;
676
- await this.handleSkippedRequest({ url: request.url, reason: 'redirect' });
677
- throw new ContextPipelineInterruptedError(message);
678
- }
679
- return context;
680
- },
694
+ contextPipeline = contextPipeline.compose(async (context) => {
695
+ const { request } = context;
696
+ if (request && !this.#requestMatchesEnqueueStrategy(request)) {
697
+ // eslint-disable-next-line dot-notation
698
+ const message = `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`;
699
+ this.log.debug(message);
700
+ throw new ContextPipelineInterruptedError('redirect');
701
+ }
702
+ return context;
681
703
  });
682
704
  return contextPipeline;
683
705
  }
@@ -688,7 +710,7 @@ export class BasicCrawler {
688
710
  * @param error The error to check.
689
711
  */
690
712
  isProxyError(error) {
691
- return ROTATE_PROXY_ERRORS.some((x) => this.getMessageFromError(error)?.includes(x));
713
+ return ROTATE_PROXY_ERRORS.some((x) => this.getMessageFromError(error).includes(x));
692
714
  }
693
715
  /**
694
716
  * Sets the status message for the current crawler run.
@@ -701,19 +723,21 @@ export class BasicCrawler {
701
723
  */
702
724
  setStatusMessage(message, options = {}) {
703
725
  const data = options.isStatusMessageTerminal != null ? { terminal: options.isStatusMessageTerminal } : undefined;
704
- this.log.logWithLevel(LogLevel[options.level ?? 'DEBUG'], message, data);
726
+ // Each allowed level has its own method on the logger, so this goes through them rather than through
727
+ // `logWithLevel`, which is abstract and therefore cannot be instrumented.
728
+ this.log[{ DEBUG: 'debug', INFO: 'info', WARNING: 'warning', ERROR: 'error' }[options.level ?? 'DEBUG']](message, data);
705
729
  // Broadcast the status message through the event system. Consumers (e.g. the Apify SDK) can
706
730
  // subscribe to `EventType.STATUS_MESSAGE` and propagate it to their status-reporting backend.
707
731
  // Setting the status message is not a storage concern, so we intentionally don't route it
708
732
  // through the storage client anymore.
709
733
  serviceLocator.getEventManager().emit(EventType.STATUS_MESSAGE, {
710
- crawlerId: this.identity.id,
734
+ crawlerId: this.#identity.id,
711
735
  message,
712
736
  isStatusMessageTerminal: options.isStatusMessageTerminal,
713
737
  level: options.level,
714
738
  });
715
739
  }
716
- getPeriodicLogger() {
740
+ #getPeriodicLogger() {
717
741
  let previousState = { ...this.statistics.state };
718
742
  const getOperationMode = () => {
719
743
  const { requestsFailed } = this.statistics.state;
@@ -733,7 +757,7 @@ export class BasicCrawler {
733
757
  }
734
758
  else {
735
759
  const total = await this.requestManager?.getTotalCount();
736
- message = `Crawled ${this.statistics.state.requestsFinished}${total ? `/${total}` : ''} pages, ${this.statistics.state.requestsFailed} failed requests, desired concurrency ${this.concurrencySystem?.desiredConcurrency ?? 0}.`;
760
+ message = `Crawled ${this.statistics.state.requestsSucceeded}${total ? `/${total}` : ''} pages, ${this.statistics.state.requestsFailed} failed requests, desired concurrency ${this.concurrencySystem?.desiredConcurrency ?? 0}.`;
737
761
  }
738
762
  if (this.#statusMessageCallback) {
739
763
  await this.#statusMessageCallback({
@@ -757,120 +781,141 @@ export class BasicCrawler {
757
781
  * We can use the `requests` parameter to enqueue the initial requests — it is a shortcut for
758
782
  * running {@link BasicCrawler.addRequests|`crawler.addRequests()`} before {@link BasicCrawler.run|`crawler.run()`}.
759
783
  *
784
+ * Calling `run()` again on the same instance keeps crawling the same request manager - requests the previous
785
+ * run handled (a failed one counts as handled) are not processed again. Purge the queue or open a fresh one
786
+ * if that is what you want.
787
+ *
760
788
  * @param [requests] The requests to add.
761
- * @param [options] Options for the request queue.
789
+ * @param [options] Options for adding the initial requests.
762
790
  */
763
791
  async run(requests, options) {
764
- if (this.running) {
765
- throw new Error('This crawler instance is already running, you can add more requests to it via `crawler.addRequests()`.');
766
- }
767
- const { purgeRequestQueue, ...addRequestsOptions } = options ?? {};
768
- if (this.hasFinishedBefore) {
769
- // When executing the run method for the second time explicitly,
770
- // we need to purge the RQ to allow processing the same requests again — this is important so users can
771
- // pass in failed requests back to the `crawler.run()`, otherwise they would be considered as handled and
772
- // ignored — as a failed request is still handled.
773
- // By default (`purgeRequestQueue` unset), only the queue we opened ourselves is purged.
774
- // When `purgeRequestQueue` is explicitly `true`, we also purge a user-supplied manager.
775
- // When `purgeRequestQueue` is explicitly `false`, nothing is purged.
776
- const shouldPurge = purgeRequestQueue !== false;
777
- const managerToPurge = this.#ownedRequestQueue.maybeValue ?? (purgeRequestQueue === true ? this.requestManager : undefined);
778
- if (shouldPurge) {
779
- await managerToPurge?.purge?.();
780
- // The per-domain queues a `sameDomainDelaySecs` wrapper created are the crawler's own, whatever
781
- // sits underneath them - so they are emptied even when the manager they wrap is spared. Purging
782
- // the wrapper itself has already covered them.
783
- if (this.requestManager instanceof ThrottlingRequestManager && managerToPurge !== this.requestManager) {
784
- await this.requestManager.purgeDomainQueues();
785
- }
792
+ // A crawl is the top level of its own transaction and timeout scope, not a participant in the caller's.
793
+ return withDirectStorageAccess(async () => timeoutStorage.exit(async () => {
794
+ if (this.running) {
795
+ throw new Error('This crawler instance is already running, you can add more requests to it via `crawler.addRequests()`.');
786
796
  }
787
- // A supplied statistics instance keeps whatever state it was handed - only wipe a default we built.
788
- await this.#statisticsDep.ifOwned(async (stats) => {
789
- stats.reset();
790
- await stats.resetStore();
791
- });
792
- await this.#sessionPoolDep.ifOwned((pool) => pool.resetStore());
793
- }
794
- this.#unexpectedStop = false;
795
- this.running = true;
796
- this.#loggedPerRun.clear();
797
- await purgeDefaultStorages({
798
- onlyPurgeOnce: true,
799
- storageBackend: serviceLocator.getStorageBackend(),
800
- configuration: serviceLocator.getConfiguration(),
801
- });
802
- if (requests) {
803
- await this.addRequests(requests, addRequestsOptions);
804
- }
805
- try {
806
- await this.init();
807
- await this.statistics.startCapturing();
808
- }
809
- catch (error) {
810
- // Clean up here before propagating, otherwise a failed startup would leave the process hanging.
811
- await this.teardown().catch((teardownError) => {
812
- this.log.exception(teardownError, 'Cleaning up after a failed crawler startup failed.');
797
+ // An owned governor is rebuilt for every run, so it always starts from a clean slate — stale
798
+ // resource snapshots or a previous run's scaled desired concurrency would otherwise distort this
799
+ // run's scaling. An injected one is long-lived and its lifecycle belongs to the caller.
800
+ this.#concurrencySystem = this.#resolveConcurrencySystem();
801
+ const run = new CrawlerRun({
802
+ log: this.log,
803
+ concurrencySystem: this.#concurrencySystem.value,
804
+ taskLoopOptions: this.#taskLoopOptions,
805
+ consumer: this.#identity,
813
806
  });
814
- // The run never began, so let the instance be run again instead of leaving it wedged as `running`.
815
- this.running = false;
816
- throw error;
817
- }
818
- const periodicLogger = this.getPeriodicLogger();
819
- this.setStatusMessage('Starting the crawler.', { level: 'INFO' });
820
- const sigintHandler = async () => {
821
- this.log.warning('Pausing... Press CTRL+C again to force exit. To resume, do: CRAWLEE_PURGE_ON_START=0 npm start');
822
- await this.pauseOnMigration();
823
- await this.#autoscaledPool.abort();
824
- };
825
- // Attach a listener to handle migration and aborting events gracefully.
826
- const boundPauseOnMigration = this.pauseOnMigration.bind(this);
827
- process.once('SIGINT', sigintHandler);
828
- const eventManager = serviceLocator.getEventManager();
829
- eventManager.on(EventType.MIGRATING, boundPauseOnMigration);
830
- eventManager.on(EventType.ABORTING, boundPauseOnMigration);
831
- let stats = {};
832
- try {
833
- await this.#autoscaledPool.run();
834
- }
835
- finally {
836
- await this.statistics.stopCapturing();
837
- await this.teardown();
838
- process.off('SIGINT', sigintHandler);
839
- eventManager.off(EventType.MIGRATING, boundPauseOnMigration);
840
- eventManager.off(EventType.ABORTING, boundPauseOnMigration);
841
- const finalStats = this.statistics.calculate();
842
- stats = {
843
- requestsFinished: this.statistics.state.requestsFinished,
844
- requestsFailed: this.statistics.state.requestsFailed,
845
- retryHistogram: this.statistics.requestRetryHistogram,
846
- ...finalStats,
847
- };
848
- this.log.info('Final request statistics:', stats);
849
- if (this.statistics.errorTracker.total !== 0) {
850
- const prettify = ([count, info]) => `${count}x: ${info.at(-1).trim()} (${info[0]})`;
851
- this.log.info(`Error analysis:`, {
852
- totalErrors: this.statistics.errorTracker.total,
853
- uniqueErrors: this.statistics.errorTracker.getUniqueErrorCount(),
854
- mostCommonErrors: this.statistics.errorTracker.getMostPopularErrors(3).map(prettify),
807
+ this.#run = run;
808
+ try {
809
+ if (this.#hasFinishedBefore) {
810
+ // A supplied statistics instance keeps whatever state it was handed - only wipe a default we built.
811
+ await this.#statisticsDep.ifOwned(async (stats) => {
812
+ stats.reset();
813
+ await stats.resetStore();
814
+ });
815
+ // A session pool the crawler built follows the same blank-slate rule as its statistics.
816
+ await this.#sessionPoolDep.ifOwned(async (pool) => {
817
+ pool.reset();
818
+ await pool.resetStore();
819
+ });
820
+ }
821
+ await purgeDefaultStorages({
822
+ onlyPurgeOnce: true,
823
+ storageBackend: serviceLocator.getStorageBackend(),
824
+ configuration: serviceLocator.getConfiguration(),
855
825
  });
826
+ if (requests) {
827
+ await this.addRequests(requests, options);
828
+ }
829
+ // An injected governor is the caller's to run; an owned one is started here and stopped again
830
+ // below if the startup fails. Either way the task loop rejects one that is not running.
831
+ await this.#concurrencySystem.ifOwned((system) => system.start());
832
+ await this.init();
833
+ await this.statistics.startCapturing();
834
+ }
835
+ catch (error) {
836
+ // Clean up here before propagating, otherwise a failed startup would leave the process hanging.
837
+ await this.teardown().catch((teardownError) => {
838
+ this.log.exception(teardownError, 'Cleaning up after a failed crawler startup failed.');
839
+ });
840
+ // The run never began, so let the instance be run again instead of leaving it wedged as `running`.
841
+ await run.finish();
842
+ await this.#concurrencySystem.ifOwned((system) => system.stop());
843
+ throw error;
844
+ }
845
+ const periodicLogger = this.#getPeriodicLogger();
846
+ this.setStatusMessage('Starting the crawler.', { level: 'INFO' });
847
+ const sigintHandler = async () => {
848
+ this.log.warning('Pausing... Press CTRL+C again to force exit. To resume, do: CRAWLEE_PURGE_ON_START=0 npm start');
849
+ await this.#pauseOnMigration();
850
+ await run.abort();
851
+ };
852
+ // Attach a listener to handle migration and aborting events gracefully.
853
+ const boundPauseOnMigration = this.#pauseOnMigration.bind(this);
854
+ process.once('SIGINT', sigintHandler);
855
+ const eventManager = serviceLocator.getEventManager();
856
+ eventManager.on(EventType.MIGRATING, boundPauseOnMigration);
857
+ eventManager.on(EventType.ABORTING, boundPauseOnMigration);
858
+ let stats = {};
859
+ try {
860
+ await run.dispatchRequests();
856
861
  }
857
- const client = serviceLocator.getStorageBackend();
858
- if (client.teardown) {
859
- let finished = false;
860
- setTimeout(() => {
861
- if (!finished) {
862
- this.log.info('Waiting for the storage to write its state to file system.');
862
+ finally {
863
+ await this.statistics.stopCapturing();
864
+ await this.teardown();
865
+ process.off('SIGINT', sigintHandler);
866
+ eventManager.off(EventType.MIGRATING, boundPauseOnMigration);
867
+ eventManager.off(EventType.ABORTING, boundPauseOnMigration);
868
+ const finalStats = this.statistics.calculate();
869
+ stats = {
870
+ requestsSucceeded: this.statistics.state.requestsSucceeded,
871
+ requestsFailed: this.statistics.state.requestsFailed,
872
+ retryHistogram: this.statistics.requestRetryHistogram,
873
+ ...finalStats,
874
+ };
875
+ this.log.info('Final request statistics:', stats);
876
+ // A crawl that did nothing while the manager holds only handled requests is a mistake whoever
877
+ // handled them - this run, another crawler on the same queue, or a previous process. Starting
878
+ // against handled requests is not: that is what resuming a crawl looks like.
879
+ if (stats.requestsSucceeded + stats.requestsFailed === 0) {
880
+ // Never let the diagnostic itself break the run.
881
+ const alreadyHandled = (await this.requestManager?.getHandledCount().catch(() => 0)) ?? 0;
882
+ if (alreadyHandled > 0) {
883
+ this.log.warningOnce('This crawl processed no requests - the request manager holds ' +
884
+ `${alreadyHandled} request${alreadyHandled === 1 ? '' : 's'}, all of them ` +
885
+ 'already handled, and a failed request counts as handled too. Nothing ' +
886
+ 'empties a queue between runs, so to crawl them again, purge it ' +
887
+ '(`await queue.purge()`) or use a fresh one (e.g. ' +
888
+ '`RequestQueue.open({ alias: "second-run" })`) with a freshly created ' +
889
+ 'crawler instance.');
863
890
  }
864
- }, 1000);
865
- await client.teardown();
866
- finished = true;
891
+ }
892
+ if (this.statistics.errorTracker.total !== 0) {
893
+ const prettify = ([count, info]) => `${count}x: ${info.at(-1).trim()} (${info[0]})`;
894
+ this.log.info(`Error analysis:`, {
895
+ totalErrors: this.statistics.errorTracker.total,
896
+ uniqueErrors: this.statistics.errorTracker.getUniqueErrorCount(),
897
+ mostCommonErrors: this.statistics.errorTracker.getMostPopularErrors(3).map(prettify),
898
+ });
899
+ }
900
+ const client = serviceLocator.getStorageBackend();
901
+ if (client.teardown) {
902
+ let finished = false;
903
+ setTimeout(() => {
904
+ if (!finished) {
905
+ this.log.info('Waiting for the storage to write its state to file system.');
906
+ }
907
+ }, 1000);
908
+ await client.teardown();
909
+ finished = true;
910
+ }
911
+ periodicLogger.stop();
912
+ this.setStatusMessage(`Finished! Total ${this.statistics.state.requestsSucceeded + this.statistics.state.requestsFailed} requests: ${this.statistics.state.requestsSucceeded} succeeded, ${this.statistics.state.requestsFailed} failed.`, { isStatusMessageTerminal: true, level: 'INFO' });
913
+ await run.finish();
914
+ await this.#concurrencySystem.ifOwned((system) => system.stop());
915
+ this.#hasFinishedBefore = true;
867
916
  }
868
- periodicLogger.stop();
869
- this.setStatusMessage(`Finished! Total ${this.statistics.state.requestsFinished + this.statistics.state.requestsFailed} requests: ${this.statistics.state.requestsFinished} succeeded, ${this.statistics.state.requestsFailed} failed.`, { isStatusMessageTerminal: true, level: 'INFO' });
870
- this.running = false;
871
- this.hasFinishedBefore = true;
872
- }
873
- return stats;
917
+ return stats;
918
+ }));
874
919
  }
875
920
  /**
876
921
  * Gracefully stops the current run of the crawler.
@@ -880,11 +925,12 @@ export class BasicCrawler {
880
925
  * To stop the crawler immediately, use {@link BasicCrawler.teardown|`crawler.teardown()`} instead.
881
926
  */
882
927
  stop(reason = 'The crawler has been gracefully stopped.') {
883
- if (this.#unexpectedStop) {
928
+ const run = this.#liveRun;
929
+ if (run === undefined) {
930
+ this.log.warning('Cannot stop a crawler that is not running.');
884
931
  return;
885
932
  }
886
- this.log.info(reason);
887
- this.#unexpectedStop = true;
933
+ run.stop(reason);
888
934
  }
889
935
  /**
890
936
  * Stops dispatching new requests, letting the in-progress ones finish. Resolves once they have settled, or rejects
@@ -895,52 +941,41 @@ export class BasicCrawler {
895
941
  * throughout, since a shared one may still be serving other crawlers.
896
942
  */
897
943
  async pause(timeoutSecs) {
898
- if (!this.#autoscaledPool) {
944
+ const run = this.#dispatchingRun;
945
+ if (run === undefined) {
899
946
  this.log.warning('Cannot pause a crawler that is not running.');
900
947
  return;
901
948
  }
902
- await this.#autoscaledPool.pause(timeoutSecs);
949
+ await run.pause(timeoutSecs);
903
950
  }
904
951
  /**
905
952
  * Resumes a run suspended with {@link BasicCrawler.pause|`pause()`}, letting the crawler dispatch requests
906
953
  * again. A no-op on a crawler that is not paused.
907
954
  */
908
955
  resume() {
909
- if (!this.#autoscaledPool) {
956
+ const run = this.#dispatchingRun;
957
+ if (run === undefined) {
910
958
  this.log.warning('Cannot resume a crawler that is not running.');
911
959
  return;
912
960
  }
913
- this.#autoscaledPool.resume();
961
+ run.resume();
914
962
  }
915
963
  /**
916
964
  * Returns the crawler's {@link IRequestManager|request manager}, opening the default {@link RequestQueue}
917
965
  * if none has been configured or opened yet.
918
966
  */
919
967
  async getRequestManager() {
920
- if (!this.requestManager) {
921
- this.requestManager = await this.openOwnedRequestQueue();
922
- }
923
- // Wrapped here rather than in the constructor, because the manager being wrapped may only be opened at
924
- // this point - and because everything that enqueues goes through here first, so nothing slips past the
925
- // wrapper into the queue it hides.
926
- if (this.#sameDomainDelaySecs > 0 && !supportsDomainThrottling(this.requestManager)) {
927
- this.requestManager = new ThrottlingRequestManager({
928
- inner: this.requestManager,
929
- domains: 'all',
930
- minCrawlDelaySecs: this.#sameDomainDelaySecs,
931
- // What `sameDomainDelaySecs` has always meant: one clock for a site, subdomains included.
932
- throttleBy: 'registrableDomain',
933
- persistStateKey: `CRAWLEE_THROTTLED_DOMAINS_${this.identity.id}`,
934
- });
968
+ if (!this.#requestManager) {
969
+ this.#requestManager = await this.#openOwnedRequestQueue();
935
970
  }
936
971
  // Apply the processing-time hint here (an async lifecycle point) rather than in the constructor,
937
- // now that `setExpectedRequestProcessingTimeSecs` is async. The hint is raise-only and idempotent,
972
+ // now that `setExpectedRequestProcessingTime` is async. The hint is raise-only and idempotent,
938
973
  // but guard so we do not re-issue it on every call.
939
974
  if (!this.#requestManagerTimeoutsApplied) {
940
975
  this.#requestManagerTimeoutsApplied = true;
941
- await this.applyRequestManagerTimeouts(this.requestManager);
976
+ await this.#applyRequestManagerTimeouts(this.#requestManager);
942
977
  }
943
- return this.requestManager;
978
+ return this.#requestManager;
944
979
  }
945
980
  /**
946
981
  * @deprecated Use {@link BasicCrawler.getRequestManager|`getRequestManager()`} instead. This returns the
@@ -950,16 +985,13 @@ export class BasicCrawler {
950
985
  return this.getRequestManager();
951
986
  }
952
987
  /**
953
- * Opens the default {@link RequestQueue}, applies the crawler's timeouts to it and records it as the
954
- * crawler-owned queue (so it gets purged between repeated `run()` calls).
955
- * @private
988
+ * Opens the default {@link RequestQueue} — the crawler's own, read from when the caller supplied nothing.
956
989
  */
957
- async openOwnedRequestQueue() {
990
+ async #openOwnedRequestQueue() {
958
991
  // The first crawler instance uses the default queue (null identifier);
959
992
  // subsequent instances get their own queue via a unique alias so they don't collide.
960
- const identifier = this.identity.instanceIndex === 0 ? null : { alias: `__default_${this.identity.id}__` };
961
- const requestQueue = await RequestQueue.open(identifier, { configuration: serviceLocator.getConfiguration() });
962
- return this.#ownedRequestQueue.set(requestQueue);
993
+ const identifier = this.#identity.instanceIndex === 0 ? null : { alias: `__default_${this.#identity.id}__` };
994
+ return RequestQueue.open(identifier, { configuration: serviceLocator.getConfiguration() });
963
995
  }
964
996
  /**
965
997
  * Tells a request manager how long we expect to hold a fetched request, so that one backed by a
@@ -968,13 +1000,13 @@ export class BasicCrawler {
968
1000
  * request from being handed out a second time while it is still being processed — and it works
969
1001
  * regardless of whether the manager is a plain {@link RequestQueue} or a `RequestManagerTandem`.
970
1002
  */
971
- async applyRequestManagerTimeouts(requestManager) {
1003
+ async #applyRequestManagerTimeouts(requestManager) {
972
1004
  // A router route may hold a request for longer than the crawler's own timeout, and we cannot know
973
1005
  // which routes a run will hit, so reserve for the longest one any route asked for. The hint is
974
1006
  // raise-only, so erring high here is safe.
975
1007
  const maxRouteTimeoutSecs = this.requestHandler.getMaxTimeoutSecs?.() ?? 0;
976
- const handlerTimeoutSecs = Math.max(this.requestHandlerTimeoutMillis / 1000, maxRouteTimeoutSecs);
977
- await requestManager.setExpectedRequestProcessingTimeSecs?.(Math.max(handlerTimeoutSecs + 5, 60));
1008
+ const handlerTimeoutSecs = Math.max(this.#requestHandlerTimeoutMillis / 1000, maxRouteTimeoutSecs);
1009
+ await requestManager.setExpectedRequestProcessingTime?.(Math.max(handlerTimeoutSecs + 5, 60));
978
1010
  }
979
1011
  /**
980
1012
  * Validates a request source's `userData` against the {@link RouteSchemas|Standard Schema} registered
@@ -983,7 +1015,7 @@ export class BasicCrawler {
983
1015
  * the request's label. Applied by the crawler on the add paths it owns — `crawler.addRequests`,
984
1016
  * `crawler.run`, `context.addRequests` and `context.enqueueLinks`.
985
1017
  */
986
- async validateRequestUserData(source) {
1018
+ async #validateRequestUserData(source) {
987
1019
  if (typeof source === 'string') {
988
1020
  return;
989
1021
  }
@@ -1004,11 +1036,11 @@ export class BasicCrawler {
1004
1036
  }
1005
1037
  async useState(defaultValue = {}) {
1006
1038
  const kvs = await KeyValueStore.open(null, { configuration: serviceLocator.getConfiguration() });
1007
- if (this.identity.hasExplicitId) {
1008
- const stateKey = `${BasicCrawler.CRAWLEE_STATE_KEY}_${this.identity.id}`;
1039
+ if (this.#identity.hasExplicitId) {
1040
+ const stateKey = `${BasicCrawler.#CRAWLEE_STATE_KEY}_${this.#identity.id}`;
1009
1041
  return kvs.getAutoSavedValue(stateKey, defaultValue);
1010
1042
  }
1011
- BasicCrawler.#useStateAnonymousIndices.add(this.identity.instanceIndex);
1043
+ BasicCrawler.#useStateAnonymousIndices.add(this.#identity.instanceIndex);
1012
1044
  if (BasicCrawler.#useStateAnonymousIndices.size > 1) {
1013
1045
  serviceLocator
1014
1046
  .getLogger()
@@ -1017,38 +1049,36 @@ export class BasicCrawler {
1017
1049
  'To fix this, provide a unique `id` option to each crawler instance. \n' +
1018
1050
  'Example: new BasicCrawler({ id: "my-crawler-1", ... })');
1019
1051
  }
1020
- return kvs.getAutoSavedValue(BasicCrawler.CRAWLEE_STATE_KEY, defaultValue);
1052
+ return kvs.getAutoSavedValue(BasicCrawler.#CRAWLEE_STATE_KEY, defaultValue);
1021
1053
  }
1022
- async getPendingRequestCountApproximation() {
1054
+ async #getPendingRequestCountApproximation() {
1023
1055
  return (await this.requestManager?.getPendingCount()) ?? 0;
1024
1056
  }
1025
- async calculateEnqueuedRequestLimit(explicitLimit) {
1026
- if (this.maxRequestsPerCrawl === undefined) {
1057
+ async #calculateEnqueuedRequestLimit(explicitLimit) {
1058
+ if (this.#maxRequestsPerCrawl === undefined) {
1027
1059
  return explicitLimit;
1028
1060
  }
1029
- const limit = Math.max(0, this.maxRequestsPerCrawl - this.handledRequestsCount - (await this.getPendingRequestCountApproximation()));
1061
+ const limit = Math.max(0, this.#maxRequestsPerCrawl -
1062
+ this.#handledRequestsCount -
1063
+ (await this.#getPendingRequestCountApproximation()));
1030
1064
  return Math.min(limit, explicitLimit ?? Infinity);
1031
1065
  }
1032
- async handleSkippedRequest(options) {
1066
+ async #handleSkippedRequest(options) {
1033
1067
  // A skipped request is a *successful* outcome, but the interrupt still unwinds through the
1034
1068
  // transaction scope, which rolls back - so the skip bookkeeping must write directly.
1035
1069
  await withDirectStorageAccess(async () => {
1070
+ // Enqueueing happens outside a run too - `addRequests()` before the first `run()` - so these dedupe
1071
+ // against the logger for the crawler's lifetime rather than per run.
1036
1072
  if (options.reason === 'limit') {
1037
- this.logOncePerRun('maxRequestsPerCrawl', 'The number of requests enqueued by the crawler reached the maxRequestsPerCrawl limit of ' +
1038
- `${this.maxRequestsPerCrawl} requests and no further requests will be added.`);
1073
+ this.log.info('The number of requests enqueued by the crawler reached the maxRequestsPerCrawl limit of ' +
1074
+ `${this.#maxRequestsPerCrawl} requests and no further requests will be added.`, undefined, { once: true });
1039
1075
  }
1040
1076
  if (options.reason === 'depth') {
1041
- this.logOncePerRun('maxCrawlDepth', `The crawler reached the maxCrawlDepth limit of ${this.maxCrawlDepth} and no further requests will be enqueued.`);
1077
+ this.log.info(`The crawler reached the maxCrawlDepth limit of ${this.#maxCrawlDepth} and no further requests will be enqueued.`, undefined, { once: true });
1042
1078
  }
1043
- await this.onSkippedRequest?.(options);
1079
+ await this.#onSkippedRequest?.(options);
1044
1080
  });
1045
1081
  }
1046
- logOncePerRun(key, message, level = 'info') {
1047
- if (!this.#loggedPerRun.has(key)) {
1048
- this.log[level](message);
1049
- this.#loggedPerRun.add(key);
1050
- }
1051
- }
1052
1082
  /**
1053
1083
  * Adds requests to the queue in batches. By default, it will resolve after the initial batch is added, and continue
1054
1084
  * adding the rest in background. You can configure the batch size via `batchSize` option and the sleep time in between
@@ -1077,25 +1107,47 @@ export class BasicCrawler {
1077
1107
  // its own per-item validation below, and validating an absent label/userData here would spuriously
1078
1108
  // check them against a registered default-route schema.
1079
1109
  if (options.label !== undefined || options.userData !== undefined) {
1080
- await this.validateRequestUserData({ label: options.label, userData: options.userData });
1110
+ await this.#validateRequestUserData({ label: options.label, userData: options.userData });
1081
1111
  }
1082
- const requestLimit = await this.calculateEnqueuedRequestLimit(options.limit);
1112
+ const requestLimit = await this.#calculateEnqueuedRequestLimit(options.limit);
1083
1113
  const strategy = options.strategy ?? EnqueueStrategy.All;
1084
- const urlExcludePatternObjects = options.exclude?.length
1085
- ? constructUrlPatternObjects(options.exclude)
1086
- : [];
1087
- const urlPatternObjects = options.include?.length
1088
- ? constructUrlPatternObjects(options.include)
1089
- : [];
1114
+ const urlExcludePatternObjects = options.exclude?.length ? constructUrlPatternObjects(options.exclude) : [];
1115
+ const urlPatternObjects = options.include?.length ? constructUrlPatternObjects(options.include) : [];
1090
1116
  // The strategy always applies, even when `include` patterns are provided - the two are AND-ed together
1091
1117
  // (a URL must match an `include` pattern *and* satisfy the strategy). This mirrors crawlee-python.
1092
- const enqueueStrategyPatterns = options.baseUrl
1093
- ? buildEnqueueStrategyPatterns(options.baseUrl, strategy)
1094
- : [];
1095
- const isAllowedBasedOnRobotsTxtFile = this.isAllowedBasedOnRobotsTxtFile.bind(this);
1096
- const maxCrawlDepth = this.maxCrawlDepth;
1097
- const validateRequestUserData = this.validateRequestUserData.bind(this);
1118
+ const strategyBaseUrl = options.baseUrl ? new URL(options.baseUrl) : undefined;
1119
+ const isAllowedBasedOnRobotsTxtFile = this.#isAllowedBasedOnRobotsTxtFile.bind(this);
1120
+ const maxCrawlDepth = this.#maxCrawlDepth;
1121
+ const validateRequestUserData = this.#validateRequestUserData.bind(this);
1098
1122
  const allSkipped = [];
1123
+ // A skipped source (which can carry arbitrary userData) is only retained if something reads it -
1124
+ // otherwise the URL alone is enough to build the callback argument and to log with.
1125
+ const hasSkippedRequestCallback = this.#onSkippedRequest !== undefined || options.onSkippedRequest !== undefined;
1126
+ const keepSkippedSource = (source) => (hasSkippedRequestCallback ? source : source.url);
1127
+ const reportSkippedRequests = async () => {
1128
+ const skippedRequests = allSkipped.splice(0);
1129
+ if (skippedRequests.length === 0) {
1130
+ return;
1131
+ }
1132
+ const skippedRobotsUrls = skippedRequests
1133
+ .filter((s) => s.reason === 'robotsTxt')
1134
+ .map(({ source }) => (typeof source === 'string' ? source : source.url));
1135
+ if (skippedRobotsUrls.length > 0) {
1136
+ this.log.warning(`Some requests were skipped because they were disallowed based on the robots.txt file`, { skipped: skippedRobotsUrls });
1137
+ }
1138
+ // Only log the limit message when an explicit `limit` was passed (not the internal
1139
+ // `maxRequestsPerCrawl`-derived one), and only once per call.
1140
+ if (options.limit !== undefined && skippedRequests.some((s) => s.reason === 'limit')) {
1141
+ this.log.info(requestLimit === options.limit
1142
+ ? `Skipping requests in this call due to the enqueueLinks limit of ${options.limit}.`
1143
+ : `Skipping requests in this call due to the remaining maxRequestsPerCrawl budget of ${requestLimit}, which is lower than the enqueueLinks limit of ${options.limit}.`);
1144
+ }
1145
+ await Promise.all(skippedRequests.map(async ({ source, reason }) => {
1146
+ const args = createSkippedRequestArgs(source, reason);
1147
+ await this.#handleSkippedRequest(args);
1148
+ await options.onSkippedRequest?.(args);
1149
+ }));
1150
+ };
1099
1151
  async function* filteredRequests() {
1100
1152
  for await (const request of requests) {
1101
1153
  const [requestOptions] = createRequestOptions([typeof request === 'string' ? request : request], { ...options, strategy });
@@ -1103,24 +1155,31 @@ export class BasicCrawler {
1103
1155
  continue; // invalid URL, silently dropped (matches `createRequestOptions`'s own filtering)
1104
1156
  }
1105
1157
  if (maxCrawlDepth !== undefined && requestOptions.crawlDepth > maxCrawlDepth) {
1106
- allSkipped.push({ url: requestOptions.url, reason: 'depth' });
1158
+ allSkipped.push({ source: keepSkippedSource(requestOptions), reason: 'depth' });
1107
1159
  continue;
1108
1160
  }
1109
1161
  if (!(await isAllowedBasedOnRobotsTxtFile(requestOptions.url))) {
1110
- allSkipped.push({ url: requestOptions.url, reason: 'robotsTxt' });
1162
+ allSkipped.push({ source: keepSkippedSource(requestOptions), reason: 'robotsTxt' });
1111
1163
  continue;
1112
1164
  }
1113
- const onSkippedFilterUrl = (url) => allSkipped.push({ url, reason: 'filters' });
1165
+ const onSkippedByFilter = (opts) => allSkipped.push({ source: keepSkippedSource(opts), reason: 'filters' });
1114
1166
  // Filter by user patterns first (with exclude)...
1115
- let filtered = filterRequestOptionsByPatterns([requestOptions], urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, strategy, onSkippedFilterUrl);
1167
+ let filtered = filterRequestOptionsByPatterns([requestOptions], urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, strategy, onSkippedByFilter);
1116
1168
  // ...then filter by the enqueue strategy (making this an AND check)
1117
- filtered = filterRequestOptionsByPatterns(filtered, enqueueStrategyPatterns.length > 0 ? enqueueStrategyPatterns : undefined, [], strategy, onSkippedFilterUrl);
1169
+ if (strategyBaseUrl) {
1170
+ filtered = filtered.filter((opts) => {
1171
+ const { allowed } = filterUrl(opts.url, strategyBaseUrl, strategy);
1172
+ if (!allowed)
1173
+ onSkippedByFilter(opts);
1174
+ return allowed;
1175
+ });
1176
+ }
1118
1177
  if (filtered.length === 0) {
1119
1178
  continue;
1120
1179
  }
1121
1180
  let [finalOptions] = filtered;
1122
1181
  if (options.transformRequestFunction) {
1123
- const transformed = applyRequestTransform([finalOptions], options.transformRequestFunction, (r) => allSkipped.push({ url: r.url, reason: r.skippedReason ?? 'transform' }));
1182
+ const transformed = applyRequestTransform([finalOptions], options.transformRequestFunction, (r) => allSkipped.push({ source: keepSkippedSource(r), reason: 'transform' }));
1124
1183
  if (transformed.length === 0) {
1125
1184
  continue;
1126
1185
  }
@@ -1132,7 +1191,7 @@ export class BasicCrawler {
1132
1191
  }
1133
1192
  const result = await this.requestManager.addRequestsBatched(filteredRequests(), {
1134
1193
  forefront: options.forefront,
1135
- waitForAllRequestsToBeAdded: options.waitForAllRequestsToBeAdded,
1194
+ waitForAllRequestsToBeAdded: options.waitForAllRequestsToBeAdded || requestLimit !== undefined,
1136
1195
  batchSize: options.batchSize,
1137
1196
  waitBetweenBatchesMillis: options.waitBetweenBatchesMillis,
1138
1197
  maxNewRequests: requestLimit,
@@ -1140,31 +1199,24 @@ export class BasicCrawler {
1140
1199
  // Report requests skipped due to the maxNewRequests budget (i.e. maxRequestsPerCrawl limit, or an
1141
1200
  // explicit `limit` option)
1142
1201
  for (const request of result.requestsOverLimit ?? []) {
1143
- allSkipped.push({ url: typeof request === 'string' ? request : request.url, reason: 'limit' });
1144
- }
1145
- if (allSkipped.length > 0) {
1146
- const skippedRobotsUrls = allSkipped.filter((s) => s.reason === 'robotsTxt').map((s) => s.url);
1147
- if (skippedRobotsUrls.length > 0) {
1148
- this.log.warning(`Some requests were skipped because they were disallowed based on the robots.txt file`, { skipped: skippedRobotsUrls });
1149
- }
1150
- // Only log the limit message when an explicit `limit` was passed (not the internal
1151
- // `maxRequestsPerCrawl`-derived one), and only once per call.
1152
- if (options.limit !== undefined && allSkipped.some((s) => s.reason === 'limit')) {
1153
- this.log.info(requestLimit === options.limit
1154
- ? `Skipping requests in this call due to the enqueueLinks limit of ${options.limit}.`
1155
- : `Skipping requests in this call due to the remaining maxRequestsPerCrawl budget of ${requestLimit}, which is lower than the enqueueLinks limit of ${options.limit}.`);
1156
- }
1157
- await Promise.all(allSkipped.map(async ({ url, reason }) => {
1158
- await this.handleSkippedRequest({ url, reason });
1159
- await options.onSkippedRequest?.({ url, reason });
1160
- }));
1202
+ allSkipped.push({ source: request, reason: 'limit' });
1161
1203
  }
1162
- return result;
1204
+ await reportSkippedRequests();
1205
+ const waitForAllRequestsToBeAdded = result.waitForAllRequestsToBeAdded.then(async (addedRequests) => {
1206
+ await reportSkippedRequests();
1207
+ return addedRequests;
1208
+ });
1209
+ // Keep callback failures observable to callers that await this promise without emitting an unhandled rejection
1210
+ // when callers intentionally leave background additions running, matching `drainRequestBatches` behavior.
1211
+ // Background failures are already logged by `drainRequestBatches`.
1212
+ void waitForAllRequestsToBeAdded.catch(() => { });
1213
+ return { ...result, waitForAllRequestsToBeAdded };
1163
1214
  }
1164
1215
  /**
1165
1216
  * Pushes data to the specified {@link Dataset}, or the default crawler {@link Dataset} by calling {@link Dataset.pushData}.
1166
1217
  */
1167
1218
  async pushData(data, datasetIdentifier) {
1219
+ tryCancel();
1168
1220
  const dataset = await this.getDataset(datasetIdentifier);
1169
1221
  return dataset.pushData(data);
1170
1222
  }
@@ -1233,31 +1285,25 @@ export class BasicCrawler {
1233
1285
  * Initializes the crawler.
1234
1286
  */
1235
1287
  async init() {
1288
+ const run = this.#liveRun;
1289
+ if (run === undefined) {
1290
+ throw new Error('There is no crawl in progress - `run()` is the only thing that starts one.');
1291
+ }
1236
1292
  const eventManager = serviceLocator.getEventManager();
1237
1293
  if (!eventManager.isInitialized()) {
1238
1294
  await eventManager.init();
1239
- this.#closeEvents = true;
1295
+ run.ownsEventManager = true;
1240
1296
  }
1241
1297
  // Warn once at startup if the internal timeout is shorter than the phases it is meant to outlast. It is
1242
1298
  // floored per request so it will not actually cut them short, but the configured value is then effectively
1243
1299
  // ignored, which is worth flagging. Checked here (not in the constructor) because a subclass sets its
1244
1300
  // navigation timeout only after `super()`.
1245
- const phasesMillis = this.getNavigationTimeoutMillis() + this.resolveRequestHandlerTimeoutMillis(undefined);
1301
+ const phasesMillis = this.getNavigationTimeoutMillis() + this.#resolveRequestHandlerTimeoutMillis(undefined);
1246
1302
  if (this.internalTimeoutMillis < phasesMillis) {
1247
1303
  this.log.warning(`CRAWLEE_INTERNAL_TIMEOUT (${this.internalTimeoutMillis / 1000}s) is shorter than the navigation ` +
1248
1304
  `and request handler timeouts combined (${phasesMillis / 1000}s); it will be raised per request ` +
1249
1305
  `so it does not cut them short.`);
1250
1306
  }
1251
- // An owned governor is rebuilt (and started) for every run, so it always starts from a clean slate — stale
1252
- // resource snapshots or a previous run's scaled desired concurrency would otherwise distort this run's
1253
- // scaling. An injected one is long-lived and its lifecycle belongs to the caller.
1254
- this.#concurrencySystemDep = this.#resolveConcurrencySystem();
1255
- await this.#concurrencySystemDep.ifOwned((system) => system.start());
1256
- this.#autoscaledPool = new AutoscaledPool({
1257
- ...this.taskLoopOptions,
1258
- concurrencySystem: this.#concurrencySystemDep.value,
1259
- consumer: this.identity,
1260
- });
1261
1307
  await this.getRequestManager();
1262
1308
  }
1263
1309
  /**
@@ -1274,9 +1320,9 @@ export class BasicCrawler {
1274
1320
  * request, a per-route override, or a low `CRAWLEE_INTERNAL_TIMEOUT` is not cut short mid-phase. It takes
1275
1321
  * whichever is larger: the configured internal timeout, or this request's combined phase budget.
1276
1322
  */
1277
- async withRequestTimeout(crawlingContext, work) {
1323
+ async #withRequestTimeout(crawlingContext, work) {
1278
1324
  const { request } = crawlingContext;
1279
- const phasesMillis = this.getNavigationTimeoutMillis() + this.resolveRequestHandlerTimeoutMillis(request.label);
1325
+ const phasesMillis = this.getNavigationTimeoutMillis() + this.#resolveRequestHandlerTimeoutMillis(request.label);
1280
1326
  const timeoutMillis = Math.max(this.internalTimeoutMillis, phasesMillis);
1281
1327
  await raceWithTimeout(crawlingContext, work, { timeoutMillis, requestId: request.id });
1282
1328
  }
@@ -1287,14 +1333,14 @@ export class BasicCrawler {
1287
1333
  * @param label The request's route label, or `undefined` for the default route / no specific request.
1288
1334
  * @param fallbackMillis Timeout to use when no route overrides it.
1289
1335
  */
1290
- resolveRequestHandlerTimeoutMillis(label, fallbackMillis = this.requestHandlerTimeoutMillis) {
1291
- return this.getRouteTimeoutMillis(label) ?? fallbackMillis;
1336
+ #resolveRequestHandlerTimeoutMillis(label, fallbackMillis = this.#requestHandlerTimeoutMillis) {
1337
+ return this.#getRouteTimeoutMillis(label) ?? fallbackMillis;
1292
1338
  }
1293
1339
  /**
1294
1340
  * The timeout the router route with the given label asked for, or `undefined` when it did not override one
1295
1341
  * (or the request handler is not a router at all).
1296
1342
  */
1297
- getRouteTimeoutMillis(label) {
1343
+ #getRouteTimeoutMillis(label) {
1298
1344
  const getTimeoutSecs = this.requestHandler.getTimeoutSecs;
1299
1345
  if (typeof getTimeoutSecs !== 'function') {
1300
1346
  return undefined;
@@ -1303,14 +1349,15 @@ export class BasicCrawler {
1303
1349
  return timeoutSecs === undefined ? undefined : timeoutSecs * 1000;
1304
1350
  }
1305
1351
  async runRequestHandler(crawlingContext) {
1306
- const timeoutMillis = this.resolveRequestHandlerTimeoutMillis(crawlingContext.request.label);
1352
+ const timeoutMillis = this.#resolveRequestHandlerTimeoutMillis(crawlingContext.request.label);
1307
1353
  await addTimeoutToPromise(async () => this.requestHandler(crawlingContext), timeoutMillis, `requestHandler timed out after ${timeoutMillis / 1000} seconds (${crawlingContext.request.id}).`);
1308
1354
  }
1309
1355
  /**
1310
1356
  * Runs `callback` inside a {@link StorageTransaction}, unless transactional storage is disabled.
1311
- * Deliberately does **not** commit on return - `handleRequest` swallows request handler failures, so
1312
- * a normal return says nothing about success. `handleRequest` owns the outcome.
1357
+ * Deliberately does **not** commit on return - the pipeline hands failures to `handleRequestFailure`, so a
1358
+ * normal return says nothing about success. `handleRequest` commits on success.
1313
1359
  */
1360
+ // oxlint-disable-next-line crawlee/prefer-private-fields -- accessed by tests
1314
1361
  async runInStorageTransaction(callback) {
1315
1362
  if (!this.#transactionalStorageEnabled) {
1316
1363
  return callback();
@@ -1327,7 +1374,7 @@ export class BasicCrawler {
1327
1374
  }
1328
1375
  finally {
1329
1376
  if (transaction.state === 'open') {
1330
- // `handleRequest` commits or rolls back on every normal path, so an open transaction on
1377
+ // `handleRequest` commits or `handleRequestFailure` rolls back on every normal path, so an open transaction on
1331
1378
  // a normal return is a wiring bug; on a propagating throw (a pipeline-level failure) it is
1332
1379
  // expected. Either way, discard the unvalidated writes; only the former is worth flagging.
1333
1380
  if (!threw) {
@@ -1350,7 +1397,7 @@ export class BasicCrawler {
1350
1397
  throw new SessionError(`Request blocked - received ${statusCode} status code.`);
1351
1398
  }
1352
1399
  }
1353
- async isAllowedBasedOnRobotsTxtFile(url) {
1400
+ async #isAllowedBasedOnRobotsTxtFile(url) {
1354
1401
  if (!this.#respectRobotsTxtFile) {
1355
1402
  return true;
1356
1403
  }
@@ -1359,45 +1406,52 @@ export class BasicCrawler {
1359
1406
  if (robotsTxtFile) {
1360
1407
  const crawlDelay = robotsTxtFile.getCrawlDelay(userAgent);
1361
1408
  if (crawlDelay !== undefined) {
1362
- this.applyCrawlDelay(url, crawlDelay);
1409
+ this.#applyCrawlDelay(url, crawlDelay);
1363
1410
  }
1364
1411
  }
1365
1412
  return !robotsTxtFile || robotsTxtFile.isAllowed(url, userAgent);
1366
1413
  }
1367
1414
  /**
1368
- * Records an HTTP 429 against the URL's domain so the request manager can pace the retry.
1415
+ * Records an HTTP 429 against the URL's domain so the request manager can hold the retry back.
1369
1416
  *
1370
1417
  * @param retryAfterHeader The raw `Retry-After` response header, if the server sent one.
1371
- * @returns `true` if a manager took responsibility for the delay, in which case the caller should throw
1418
+ * @returns `true` if the manager took responsibility for the delay, in which case the caller should throw
1372
1419
  * {@link RequestThrottledError} rather than treating the response as a blocked session.
1373
1420
  */
1374
1421
  recordDomainRateLimit(url, retryAfterHeader) {
1375
- if (supportsDomainThrottling(this.requestManager) &&
1376
- this.requestManager.recordDomainDelay(url, parseRetryAfterHeader(retryAfterHeader))) {
1422
+ if (this.requestManager?.recordPacingSignal({
1423
+ reason: 'rateLimited',
1424
+ url,
1425
+ waitMs: parseRetryAfterHeader(retryAfterHeader) ?? undefined,
1426
+ })) {
1377
1427
  return true;
1378
1428
  }
1379
1429
  const domain = hostnameOrUrl(url);
1380
- this.logOncePerRun(`rateLimitNotThrottled:${domain}`, `"${domain}" responded with HTTP 429 (Too Many Requests), but nothing is set up to back off from it, ` +
1381
- 'so the response is handled like any other, with no per-domain delay. ' +
1382
- `Pass a \`ThrottlingRequestManager\` as \`requestManager\` and include "${domain}" in its \`domains\` ` +
1383
- 'option to honour `Retry-After` and apply exponential backoff instead.', 'warning');
1430
+ this.log.warningOnce(`"${domain}" responded with HTTP 429 (Too Many Requests), but the crawler's request manager does not ` +
1431
+ 'pace that domain, so the response is handled like any other, with no per-domain delay. Set ' +
1432
+ `\`sameDomainDelaySecs\`, or pass a \`ThrottlingRequestManager\` covering "${domain}" as ` +
1433
+ '`requestManager`, to honour `Retry-After` and apply exponential backoff instead.');
1384
1434
  return false;
1385
1435
  }
1386
- /**
1387
- * Hands a robots.txt `Crawl-delay` to the request manager, warning if nothing is able to honour it.
1388
- *
1389
- * The warning is driven by whether the delay was actually accepted rather than by the type of the manager,
1390
- * because a manager that does throttle still drops the delay for a domain missing from its `domains` list.
1391
- */
1392
- applyCrawlDelay(url, delaySeconds) {
1393
- if (supportsDomainThrottling(this.requestManager) && this.requestManager.setCrawlDelay(url, delaySeconds)) {
1436
+ /** Hands a robots.txt `Crawl-delay` to the request manager, warning if it will not be honoured. */
1437
+ #applyCrawlDelay(url, delaySeconds) {
1438
+ // robots.txt is per-origin; `hostname` is the closest pacing scope and errs wide (http and https to one
1439
+ // host share a clock).
1440
+ if (this.requestManager?.recordPacingSignal({
1441
+ reason: 'minInterval',
1442
+ url,
1443
+ intervalMs: delaySeconds * 1000,
1444
+ scope: 'hostname',
1445
+ })) {
1394
1446
  return;
1395
1447
  }
1396
1448
  const domain = hostnameOrUrl(url);
1397
- this.logOncePerRun(`crawlDelayIgnored:${domain}`, `robots.txt for "${domain}" defines a crawl-delay of ${delaySeconds}s, but nothing is set up to honour it, ` +
1398
- 'so requests to that domain will not be paced. Pass a `ThrottlingRequestManager` as `requestManager` ' +
1399
- `and include "${domain}" in its \`domains\` option to enforce the delay.`, 'warning');
1449
+ this.log.warningOnce(`robots.txt for "${domain}" defines a crawl-delay of ${delaySeconds}s, but the crawler's request ` +
1450
+ 'manager does not pace that domain, so its requests will not be paced. Set ' +
1451
+ `\`sameDomainDelaySecs\`, or pass a \`ThrottlingRequestManager\` covering "${domain}" as ` +
1452
+ '`requestManager`.');
1400
1453
  }
1454
+ /** @internal */
1401
1455
  async getRobotsTxtFileForUrl(url) {
1402
1456
  if (!this.#respectRobotsTxtFile) {
1403
1457
  return undefined;
@@ -1408,7 +1462,15 @@ export class BasicCrawler {
1408
1462
  if (cachedRobotsTxtFile) {
1409
1463
  return cachedRobotsTxtFile;
1410
1464
  }
1411
- const robotsTxtFile = await RobotsTxtFile.find(url, { logger: this.log });
1465
+ // Fetch robots.txt the way pages are fetched: through the crawler's HTTP client, with a session's browser
1466
+ // fingerprint and proxy. Sites with bot protection answer a bare request with 4xx or 5xx, and that status
1467
+ // alone would then decide whether everything is allowed or disallowed.
1468
+ const robotsTxtFile = await RobotsTxtFile.find(url, {
1469
+ httpClient: this.httpClient,
1470
+ session: await this.sessionPool.getSession(),
1471
+ timeoutMillis: 30_000,
1472
+ logger: this.log,
1473
+ });
1412
1474
  this.#robotsTxtFileCache.add(origin, robotsTxtFile);
1413
1475
  return robotsTxtFile;
1414
1476
  }
@@ -1417,10 +1479,10 @@ export class BasicCrawler {
1417
1479
  return undefined;
1418
1480
  }
1419
1481
  }
1420
- async pauseOnMigration() {
1421
- if (this.#autoscaledPool) {
1422
- // if run wasn't called, this is going to crash
1423
- await this.#autoscaledPool.pause(SAFE_MIGRATION_WAIT_MILLIS).catch((err) => {
1482
+ async #pauseOnMigration() {
1483
+ const run = this.#dispatchingRun;
1484
+ if (run) {
1485
+ await run.pause(SAFE_MIGRATION_WAIT_MILLIS).catch((err) => {
1424
1486
  if (err.message.includes('running tasks did not finish')) {
1425
1487
  this.log.error('The crawler was paused due to migration to another host, ' +
1426
1488
  "but some requests did not finish in time. Those requests' results may be duplicated.");
@@ -1430,106 +1492,113 @@ export class BasicCrawler {
1430
1492
  }
1431
1493
  });
1432
1494
  }
1433
- const requestManagerPersistPromise = (async () => {
1434
- // The request manager persists its read-only loader's state, if it has one that supports persistence
1435
- // (e.g. a tandem wrapping a `RequestList`). For a plain `RequestQueue`, this is a no-op.
1436
- if (this.requestManager?.persistState) {
1437
- if (await this.requestManager.isFinished())
1438
- return;
1439
- await this.requestManager.persistState().catch((err) => {
1440
- if (err.message.includes('Cannot persist state.')) {
1441
- this.log.error("The crawler attempted to persist its request list's state and failed due to missing or " +
1442
- 'invalid configuration. Make sure to use either RequestList.open() or the "stateKeyPrefix" option of RequestList ' +
1443
- 'constructor to ensure your crawling state is persisted through host migrations and restarts.');
1444
- }
1445
- else {
1446
- this.log.exception(err, 'An unexpected error occurred when the crawler ' +
1447
- "attempted to persist its request list's state.");
1448
- }
1449
- });
1450
- }
1451
- })();
1452
- await Promise.all([requestManagerPersistPromise, this.statistics.persistState?.()]);
1495
+ // Captures the statistics changed while draining above. No PERSIST_STATE event is tied to the drain:
1496
+ // the periodic one is unrelated, and the one the platform emits on migration arrives before any of this.
1497
+ await this.statistics.persistState?.();
1453
1498
  }
1454
1499
  /**
1455
1500
  * Fetches the next request to process from the underlying request provider.
1456
1501
  */
1457
- async fetchNextRequest() {
1502
+ async #fetchNextRequest() {
1458
1503
  if (this.requestManager === undefined) {
1459
1504
  throw new Error(`fetchNextRequest called on an uninitialized crawler`);
1460
1505
  }
1461
1506
  return this.requestManager.fetchNextRequest();
1462
1507
  }
1463
- /** Handles a single request - runs the request handler with retries, error handling, and lifecycle management. */
1464
- async handleRequest(crawlingContext, requestSource, request) {
1508
+ /** Runs the request handler and marks the request as handled. Failures are left to `handleRequestFailure`. */
1509
+ // oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
1510
+ async handleRequest(crawlingContext, requestSource) {
1465
1511
  // An earlier phase we cannot cancel (e.g. a slow `extendContext`) may have run past the internal timeout,
1466
1512
  // which already failed the request in `runTaskFunction`. Bail before running the handler so it does not
1467
1513
  // execute (and re-report) on top of a request the crawler has already moved past.
1468
1514
  if (crawlingContext[timeoutExpiredKey]?.()) {
1469
1515
  return;
1470
1516
  }
1471
- const statisticsId = request.id || request.uniqueKey;
1517
+ const { request } = crawlingContext;
1518
+ request.state = RequestState.REQUEST_HANDLER;
1519
+ await this.runRequestHandler(crawlingContext);
1520
+ // Commit *before* marking the request as handled, so a commit failure fails the request and
1521
+ // it is retried. This also closes the transaction, so everything below passes through.
1472
1522
  // Opened by `runInStorageTransaction`; absent when disabled or when the subclass opens its own.
1473
- const transaction = currentStorageTransaction();
1474
- let isRequestLocked = true;
1523
+ await currentStorageTransaction()?.commit();
1524
+ await this.#timeoutAndRetry(async () => requestSource.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${request.url} (${request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
1525
+ this.statistics.recordRequestSuccess(request.id || request.uniqueKey, request.retryCount);
1526
+ // reclaim session if request finishes successfully
1527
+ request.state = RequestState.DONE;
1528
+ crawlingContext.session.markGood();
1529
+ }
1530
+ /**
1531
+ * Handles a request that failed anywhere in the context pipeline - a middleware (e.g. navigation or its hooks)
1532
+ * or `handleRequest`. Runs inside the pipeline call, so the cleanups (e.g. closing the page) only fire after the
1533
+ * error handler has seen the context.
1534
+ */
1535
+ async #handleRequestFailure(rawError, crawlingContext, requestSource) {
1536
+ const { request } = crawlingContext;
1537
+ // Roll back *before* any error handler runs - error handlers write to real storage precisely
1538
+ // because the transaction is already closed. A no-op when the commit in `handleRequest` succeeded.
1539
+ currentStorageTransaction()?.rollback();
1540
+ const resumeRequestTimeout = crawlingContext[suspendTimeoutKey]?.();
1475
1541
  try {
1476
- request.state = RequestState.REQUEST_HANDLER;
1477
- await this.runRequestHandler(crawlingContext);
1478
- // Commit *before* marking the request as handled, so a commit failure fails the request and
1479
- // it is retried. This also closes the transaction, so everything below passes through.
1480
- await transaction?.commit();
1481
- await this.timeoutAndRetry(async () => requestSource.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${request.url} (${request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
1482
- isRequestLocked = false; // markRequestAsHandled succeeded and unlocked the request
1483
- this.statistics.finishJob(statisticsId, request.retryCount);
1484
- // reclaim session if request finishes successfully
1485
- request.state = RequestState.DONE;
1486
- crawlingContext.session.markGood();
1487
- }
1488
- catch (rawError) {
1489
- // Roll back *before* any error handler runs - error handlers write to real storage precisely
1490
- // because the transaction is already closed. A no-op when the commit above succeeded.
1491
- transaction?.rollback();
1492
- const err = this.unwrapError(rawError);
1493
- try {
1494
- request.state = RequestState.ERROR_HANDLER;
1495
- await addTimeoutToPromise(async () => this.requestFunctionErrorHandler(err, crawlingContext, request, requestSource), this.internalTimeoutMillis, `Handling request failure of ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
1496
- if (!(err instanceof CriticalError)) {
1497
- isRequestLocked = false; // requestFunctionErrorHandler calls either markRequestAsHandled or reclaimRequest
1498
- }
1542
+ const unwrappedError = this.#unwrapError(rawError);
1543
+ // A failing or timed out `onSkippedRequest` makes this an ordinary failure, so the request gets retried.
1544
+ const err = unwrappedError instanceof ContextPipelineInterruptedError
1545
+ ? await this.#trySkipRequest(unwrappedError, request, requestSource)
1546
+ : unwrappedError;
1547
+ if (err) {
1548
+ await this.#runErrorHandler(err, crawlingContext, requestSource);
1549
+ }
1550
+ }
1551
+ finally {
1552
+ resumeRequestTimeout?.();
1553
+ }
1554
+ }
1555
+ async #runErrorHandler(err, context, requestSource) {
1556
+ const { request } = context;
1557
+ const retryCountBefore = request.retryCount;
1558
+ try {
1559
+ request.state = RequestState.ERROR_HANDLER;
1560
+ await addTimeoutToPromise(async () => this.requestFunctionErrorHandler(err, context, requestSource), this.internalTimeoutMillis, `Handling request failure of ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
1561
+ // The error handler may have set `SKIPPED`; TS keeps the narrowing from the assignment above.
1562
+ if (request.state !== RequestState.SKIPPED) {
1499
1563
  request.state = RequestState.DONE;
1500
1564
  }
1501
- catch (secondaryError) {
1502
- const unwrappedSecondaryError = this.unwrapError(secondaryError);
1503
- if (!unwrappedSecondaryError.triggeredFromUserHandler &&
1504
- // avoid reprinting the same critical error multiple times, as it will be printed by Nodejs at the end anyway
1505
- !(unwrappedSecondaryError instanceof CriticalError)) {
1506
- const apifySpecific = process.env.APIFY_IS_AT_HOME
1507
- ? `This may have happened due to an internal error of Apify's API or due to a misconfigured crawler.`
1508
- : '';
1509
- this.log.exception(unwrappedSecondaryError, 'An exception occurred during handling of failed request. ' +
1510
- `This places the crawler and its underlying storages into an unknown state and crawling will be terminated. ${apifySpecific}`);
1565
+ }
1566
+ catch (secondaryError) {
1567
+ const unwrappedSecondaryError = this.#unwrapError(secondaryError);
1568
+ const timedOut = unwrappedSecondaryError instanceof TimeoutError;
1569
+ request.state = RequestState.ERROR;
1570
+ if (timedOut) {
1571
+ // Not worth crashing the crawler over. The request's state is unknown, so it gets retried - and the
1572
+ // retry is counted, so that an error handler that always times out does not retry it forever.
1573
+ this.log.warning(unwrappedSecondaryError.message);
1574
+ if (request.retryCount === retryCountBefore) {
1575
+ request.retryCount++;
1511
1576
  }
1512
- request.state = RequestState.ERROR;
1513
- throw unwrappedSecondaryError;
1514
1577
  }
1515
- // decrease the session score if the request fails (but the error handler did not throw);
1516
- // skip when the error is a SessionError, which already retired the session
1517
- if (!this.errorAbsolvesSession(err)) {
1518
- crawlingContext.session.markBad();
1578
+ else if (!(unwrappedSecondaryError instanceof CriticalError)) {
1579
+ // avoid reprinting the same critical error multiple times, as it will be printed by Nodejs at the end anyway
1580
+ const apifySpecific = process.env.APIFY_IS_AT_HOME
1581
+ ? `This may have happened due to an internal error of Apify's API or due to a misconfigured crawler.`
1582
+ : '';
1583
+ this.log.exception(unwrappedSecondaryError, 'An exception occurred during handling of failed request. ' +
1584
+ `This places the crawler and its underlying storages into an unknown state and crawling will be terminated. ${apifySpecific}`);
1519
1585
  }
1520
- }
1521
- finally {
1522
- // Safety net - return the request to the queue if nobody managed to mark it as handled
1523
- // or reclaim it before (e.g. after a CriticalError). Reclaiming a request that is no longer
1524
- // in progress is a harmless no-op on the storage backend.
1525
- if (isRequestLocked && requestSource instanceof RequestQueue) {
1526
- try {
1527
- await requestSource.reclaimRequest(request);
1528
- }
1529
- catch {
1530
- // The request was never in progress, or could not be reclaimed. Either way it's fine.
1531
- }
1586
+ // Safety net - the error handler neither marked the request as handled nor reclaimed it (e.g. after a
1587
+ // CriticalError). Reclaiming a request that is no longer in progress is a harmless no-op on the storage backend.
1588
+ try {
1589
+ await addTimeoutToPromise(async () => requestSource.reclaimRequest(request), this.internalTimeoutMillis, `Reclaiming request ${request.url} (${request.id}) timed out.`);
1590
+ }
1591
+ catch {
1592
+ // The request was never in progress, or could not be reclaimed. Either way it's fine.
1532
1593
  }
1594
+ if (!timedOut) {
1595
+ throw unwrappedSecondaryError;
1596
+ }
1597
+ }
1598
+ // decrease the session score if the request fails (but the error handler did not throw);
1599
+ // skip when the error is a SessionError, which already retired the session
1600
+ if (!this.#errorAbsolvesSession(err)) {
1601
+ context.session?.markBad();
1533
1602
  }
1534
1603
  }
1535
1604
  /**
@@ -1541,6 +1610,11 @@ export class BasicCrawler {
1541
1610
  if (typeof request === 'string') {
1542
1611
  yield { url: request, crawlDepth: newRequestDepth };
1543
1612
  }
1613
+ else if (request instanceof Request) {
1614
+ const crawlee = (request.userData.__crawlee ??= {});
1615
+ crawlee.crawlDepth ??= newRequestDepth;
1616
+ yield request;
1617
+ }
1544
1618
  else {
1545
1619
  request.crawlDepth ??= newRequestDepth;
1546
1620
  yield request;
@@ -1551,7 +1625,7 @@ export class BasicCrawler {
1551
1625
  * Run async callback with given timeout and retry. Returns the result of the callback.
1552
1626
  * @ignore
1553
1627
  */
1554
- async timeoutAndRetry(handler, timeout, error, maxRetries = 3, retried = 1) {
1628
+ async #timeoutAndRetry(handler, timeout, error, maxRetries = 3, retried = 1) {
1555
1629
  try {
1556
1630
  return await addTimeoutToPromise(handler, timeout, error);
1557
1631
  }
@@ -1559,65 +1633,86 @@ export class BasicCrawler {
1559
1633
  if (retried <= maxRetries) {
1560
1634
  // we retry on any error, not just timeout
1561
1635
  this.log.warning(`${e.message} (retrying ${retried}/${maxRetries})`);
1562
- return this.timeoutAndRetry(handler, timeout, error, maxRetries, retried + 1);
1636
+ return this.#timeoutAndRetry(handler, timeout, error, maxRetries, retried + 1);
1563
1637
  }
1564
1638
  throw e;
1565
1639
  }
1566
1640
  }
1567
1641
  /**
1568
- * Returns true if either RequestList or RequestQueue have a request ready for processing.
1642
+ * Whether the request manager has a request ready for processing. A manager that is only `waiting` also gets a
1643
+ * wake-up scheduled, so a paced crawl resumes on its clock rather than on the task loop's polling interval.
1569
1644
  */
1570
- async isTaskReadyFunction() {
1571
- return this.requestManager !== undefined && !(await this.requestManager.isEmpty());
1572
- }
1573
- /**
1574
- * Returns true if both RequestList and RequestQueue have all requests finished.
1575
- */
1576
- async defaultIsFinishedFunction() {
1577
- return !this.requestManager || (await this.requestManager.isFinished());
1645
+ async #isTaskReadyFunction() {
1646
+ if (this.requestManager === undefined) {
1647
+ return false;
1648
+ }
1649
+ const state = await this.requestManager.checkReadiness();
1650
+ if (state.status === 'waiting' && state.readyAt !== undefined) {
1651
+ this.#run?.scheduleWake(state.readyAt);
1652
+ }
1653
+ return state.status === 'ready';
1578
1654
  }
1579
1655
  /**
1580
1656
  * Unwraps errors thrown by the context pipeline to get the actual user error.
1581
- * RequestHandlerError and ContextPipelineInitializationError wrap the actual error.
1657
+ * RequestHandlerError, ContextPipelineInitializationError and ContextPipelineCleanupError wrap the actual error.
1582
1658
  */
1583
- unwrapError(error) {
1659
+ #unwrapError(error) {
1584
1660
  if (error instanceof RequestHandlerError ||
1585
1661
  error instanceof ContextPipelineInitializationError ||
1586
1662
  error instanceof ContextPipelineCleanupError) {
1587
- return this.unwrapError(error.cause);
1663
+ return this.#unwrapError(error.cause);
1588
1664
  }
1589
1665
  return error;
1590
1666
  }
1591
1667
  /**
1592
1668
  * Handles errors thrown by user provided requestHandler()
1593
- *
1594
- * @param request The request object, passed separately to circumvent potential dynamic logic in crawlingContext.request
1595
1669
  */
1596
- async requestFunctionErrorHandler(error, crawlingContext, request, source) {
1670
+ // oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
1671
+ async requestFunctionErrorHandler(error, crawlingContext, source) {
1672
+ const { request } = crawlingContext;
1597
1673
  if (error instanceof RequestThrottledError) {
1598
- // The domain told us to come back later, so the request was never really attempted. Put it back
1599
- // without recording a failure - it costs neither a retry nor session reputation.
1674
+ // The domain told us to come back later, so the request was never really attempted. Put it back where
1675
+ // it was, without recording a failure - it costs neither a retry nor session reputation.
1600
1676
  this.log.debug(`Deferring request because its domain is rate-limiting us. ${error.message}`, {
1601
1677
  id: request.id,
1602
1678
  url: request.url,
1603
1679
  });
1604
- await source.reclaimRequest(request, { forefront: request.userData?.__crawlee?.forefront });
1680
+ await source.reclaimRequest(request, { forefront: true });
1605
1681
  return;
1606
1682
  }
1607
1683
  request.pushErrorMessage(error);
1608
1684
  if (error instanceof CriticalError) {
1609
1685
  throw error;
1610
1686
  }
1611
- const shouldRetryRequest = this.canRequestBeRetried(request, error);
1687
+ const shouldRetryRequest = this.#canRequestBeRetried(request, error);
1612
1688
  if (shouldRetryRequest) {
1613
1689
  await this.statistics.errorTrackerRetry.addAsync(error, crawlingContext);
1614
- await this.errorHandler?.(crawlingContext, // valid cast - ExtendedContext transitively extends CrawlingContext
1615
- error);
1690
+ try {
1691
+ await this.#errorHandler?.(crawlingContext, // valid cast - ExtendedContext transitively extends CrawlingContext
1692
+ error);
1693
+ }
1694
+ catch (handlerError) {
1695
+ if (handlerError instanceof ContextPipelineInterruptedError) {
1696
+ const skipError = await this.#trySkipRequest(handlerError, request, source);
1697
+ if (!skipError) {
1698
+ return;
1699
+ }
1700
+ // Retry for the original error. Routing `skipError` through `errorHandler` again
1701
+ // would loop forever with a handler that always skips.
1702
+ this.log.warning(`Skipping request ${request.url} failed, retrying it instead.`, {
1703
+ error: skipError.message,
1704
+ });
1705
+ }
1706
+ else {
1707
+ throw handlerError;
1708
+ }
1709
+ }
1616
1710
  if (error instanceof SessionError) {
1617
1711
  crawlingContext.session?.retire();
1618
1712
  }
1619
1713
  if (!request.noRetry) {
1620
1714
  request.retryCount++;
1715
+ this.statistics.registerRetry?.(request.retryCount);
1621
1716
  const { url, retryCount, id } = request;
1622
1717
  // We don't want to see the stack trace in the logs by default, when we are going to retry the request.
1623
1718
  // Thus, we print the full stack trace only when CRAWLEE_VERBOSE_LOG environment variable is set to true.
@@ -1627,7 +1722,7 @@ export class BasicCrawler {
1627
1722
  url,
1628
1723
  retryCount,
1629
1724
  });
1630
- await source.reclaimRequest(request, { forefront: request.userData?.__crawlee?.forefront });
1725
+ await source.reclaimRequest(request);
1631
1726
  return;
1632
1727
  }
1633
1728
  }
@@ -1648,18 +1743,38 @@ export class BasicCrawler {
1648
1743
  // or failed more than retryCount times and will not be retried anymore.
1649
1744
  // Mark the request as failed and do not retry.
1650
1745
  await source.markRequestAsHandled(request);
1651
- this.statistics.failJob(request.id || request.uniqueKey, request.retryCount);
1746
+ this.statistics.recordRequestFailure(request.id || request.uniqueKey, request.retryCount);
1652
1747
  await this.handleFailedRequestHandler(crawlingContext, error); // This function prints an error message.
1653
1748
  }
1749
+ /** Settles a skipped request. Returns the error that prevented that, leaving the request untouched if `onSkippedRequest` failed. */
1750
+ async #trySkipRequest(error, request, source) {
1751
+ try {
1752
+ await addTimeoutToPromise(async () => this.#handleSkippedRequest({ request, reason: error.reason, message: error.skipMessage }), this.internalTimeoutMillis, `onSkippedRequest for ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
1753
+ request.state = RequestState.SKIPPED;
1754
+ request.noRetry = true;
1755
+ await this.#timeoutAndRetry(async () => source.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${request.url} (${request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
1756
+ this.statistics.discardRequestRecord(request.id || request.uniqueKey);
1757
+ return undefined;
1758
+ }
1759
+ catch (settleError) {
1760
+ return this.#unwrapError(settleError);
1761
+ }
1762
+ }
1763
+ // oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
1654
1764
  async handleFailedRequestHandler(crawlingContext, error) {
1655
1765
  // Always log the last error regardless if the user provided a failedRequestHandler
1656
1766
  const { id, url, method, uniqueKey } = crawlingContext.request;
1657
1767
  const message = this.getMessageFromError(error, true);
1658
1768
  this.log.error(`Request failed and reached maximum retries. ${message}`, { id, url, method, uniqueKey });
1659
- if (this.failedRequestHandler) {
1660
- await this.failedRequestHandler?.(crawlingContext, // valid cast - ExtendedContext transitively extends CrawlingContext
1769
+ try {
1770
+ await this.#failedRequestHandler?.(crawlingContext, // valid cast - ExtendedContext transitively extends CrawlingContext
1661
1771
  error);
1662
1772
  }
1773
+ catch (handlerError) {
1774
+ if (!(handlerError instanceof ContextPipelineInterruptedError))
1775
+ throw handlerError;
1776
+ this.log.warning(`skipRequest() has no effect in failedRequestHandler, request ${url} (${id}) has already failed.`);
1777
+ }
1663
1778
  }
1664
1779
  /**
1665
1780
  * Resolves the most verbose error message from a thrown error
@@ -1674,7 +1789,8 @@ export class BasicCrawler {
1674
1789
  const baseDir = process.cwd();
1675
1790
  const userLine = stackLines.find((line) => line.includes(baseDir) && !line.includes('node_modules'));
1676
1791
  if (error instanceof TimeoutError) {
1677
- return process.env.CRAWLEE_VERBOSE_LOG ? error.stack : error.message || error; // stack in timeout errors does not really help
1792
+ // stack in timeout errors does not really help
1793
+ return (process.env.CRAWLEE_VERBOSE_LOG && error.stack) || error.message;
1678
1794
  }
1679
1795
  return process.env.CRAWLEE_VERBOSE_LOG || forceStack
1680
1796
  ? (error.stack ?? [error.message || error, ...stackLines].join('\n'))
@@ -1684,10 +1800,12 @@ export class BasicCrawler {
1684
1800
  * Whether the session should be spared for this error - either because it was already retired, or because the
1685
1801
  * failure says nothing about the session (a rate limit is a property of the domain).
1686
1802
  */
1687
- errorAbsolvesSession(error) {
1688
- return error instanceof SessionError || error instanceof RequestThrottledError;
1803
+ #errorAbsolvesSession(error) {
1804
+ return (error instanceof SessionError ||
1805
+ error instanceof RequestThrottledError ||
1806
+ error instanceof ContextPipelineInterruptedError);
1689
1807
  }
1690
- canRequestBeRetried(request, error) {
1808
+ #canRequestBeRetried(request, error) {
1691
1809
  // Request should never be retried, or the error encountered makes it not able to be retried.
1692
1810
  if (request.noRetry || error instanceof NonRetryableError) {
1693
1811
  return false;
@@ -1697,29 +1815,46 @@ export class BasicCrawler {
1697
1815
  return true;
1698
1816
  }
1699
1817
  // Ensure there are more retries available for the request
1700
- const maxRequestRetries = request.maxRetries ?? this.maxRequestRetries;
1818
+ const maxRequestRetries = request.maxRetries ?? this.#maxRequestRetries;
1701
1819
  return request.retryCount < maxRequestRetries;
1702
1820
  }
1703
1821
  /**
1704
- * Stops the crawler immediately.
1822
+ * Ends the run in progress immediately, without waiting for the requests already in flight.
1705
1823
  *
1706
- * This method doesn't wait for currently active requests to finish.
1824
+ * This runs at the end of every {@link BasicCrawler.run|`run()`}, so what it releases is per-run only, and
1825
+ * a repeated `run()` has to find everything it needs either intact or re-establishable. What outlives a run is
1826
+ * released by {@link BasicCrawler.destroy|`destroy()`} instead.
1707
1827
  *
1708
1828
  * To stop the crawler gracefully (waiting for all running requests to finish), use {@link BasicCrawler.stop|`crawler.stop()`} instead.
1709
1829
  */
1710
1830
  async teardown() {
1831
+ const eventManager = serviceLocator.getEventManager();
1832
+ const ownsEventManager = this.#run?.ownsEventManager === true;
1833
+ // First, so that the pool writes its final state once - its teardown stops it listening before the
1834
+ // persistence event below goes out.
1835
+ await this.#sessionPoolDep.ifOwned(async (pool) => pool.teardown());
1711
1836
  // When this crawler initialized the event manager, its close() call emits
1712
1837
  // the final persistence event after the crawler-specific state has been
1713
1838
  // saved. External event managers still need an explicit event here.
1714
- if (!this.#closeEvents) {
1715
- serviceLocator.getEventManager().emit(EventType.PERSIST_STATE, { isMigrating: false });
1839
+ if (!ownsEventManager) {
1840
+ eventManager.emit(EventType.PERSIST_STATE, { isMigrating: false });
1716
1841
  }
1717
- await this.#sessionPoolDep.ifOwned(async (pool) => pool.teardown({ persistState: this.#closeEvents ?? false }));
1718
- if (this.#closeEvents) {
1719
- await serviceLocator.getEventManager().close();
1842
+ if (ownsEventManager) {
1843
+ await eventManager.close();
1720
1844
  }
1721
- await this.#autoscaledPool?.abort();
1722
- await this.#concurrencySystemDep?.ifOwned((system) => system.stop());
1845
+ await this.#run?.end();
1846
+ }
1847
+ /**
1848
+ * Releases what the crawler owns beyond a single run — in the browser crawlers, the browser pool. A finished
1849
+ * `run()` has already released everything a run owns, so this is only needed before dropping a crawler you
1850
+ * could otherwise have run again.
1851
+ */
1852
+ async destroy() {
1853
+ // An abandoned run - one whose `run()` never returned - would otherwise keep dispatching requests.
1854
+ await this.#run?.end();
1855
+ }
1856
+ async [Symbol.asyncDispose]() {
1857
+ await this.destroy();
1723
1858
  }
1724
1859
  getCookieHeaderFromRequest(request) {
1725
1860
  if (request.headers?.Cookie && request.headers?.cookie) {
@@ -1728,7 +1863,7 @@ export class BasicCrawler {
1728
1863
  }
1729
1864
  return request.headers?.Cookie || request.headers?.cookie || '';
1730
1865
  }
1731
- requestMatchesEnqueueStrategy(request) {
1866
+ #requestMatchesEnqueueStrategy(request) {
1732
1867
  // If `skipNavigation` was used, just return `true`
1733
1868
  try {
1734
1869
  // eslint-disable-next-line @typescript-eslint/no-unused-expressions
@@ -1741,8 +1876,7 @@ export class BasicCrawler {
1741
1876
  throw err;
1742
1877
  }
1743
1878
  const { url, loadedUrl } = request;
1744
- // eslint-disable-next-line dot-notation -- private access
1745
- const strategy = request['enqueueStrategy'];
1879
+ const strategy = request.enqueueStrategy;
1746
1880
  // No strategy set, so we assume it matches, or it was added outside of enqueueLinks
1747
1881
  if (!strategy) {
1748
1882
  return true;
@@ -1751,30 +1885,7 @@ export class BasicCrawler {
1751
1885
  if (!loadedUrl) {
1752
1886
  return true;
1753
1887
  }
1754
- const baseUrl = new URL(url);
1755
- const loadedBaseUrl = new URL(loadedUrl);
1756
- switch (strategy) {
1757
- case EnqueueStrategy.SameHostname: {
1758
- return baseUrl.hostname === loadedBaseUrl.hostname;
1759
- }
1760
- case EnqueueStrategy.SameDomain: {
1761
- const baseUrlHostname = getDomain(baseUrl.hostname, { mixedInputs: false });
1762
- if (baseUrlHostname) {
1763
- const loadedBaseUrlHostname = getDomain(loadedBaseUrl.hostname, { mixedInputs: false });
1764
- return baseUrlHostname === loadedBaseUrlHostname;
1765
- }
1766
- // Can happen for IPs, we just check like same origin
1767
- return baseUrl.origin === loadedBaseUrl.origin;
1768
- }
1769
- case EnqueueStrategy.SameOrigin: {
1770
- // Same as hostname, but also checks protocol
1771
- return baseUrl.origin === loadedBaseUrl.origin;
1772
- }
1773
- case EnqueueStrategy.All:
1774
- default: {
1775
- return baseUrl.protocol === 'http:' || baseUrl.protocol === 'https:';
1776
- }
1777
- }
1888
+ return matchesEnqueueStrategy(strategy, new URL(loadedUrl), new URL(url));
1778
1889
  }
1779
1890
  }
1780
1891
  /** The hostname of `url`, falling back to the whole string when it is not parseable - for log messages only. */