@crawlee/basic 4.0.0-rc.1 → 4.0.0-rc.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/index.d.ts +6 -6
  2. package/index.js +6 -6
  3. package/internals/autoscaling/{autoscaled_pool.d.ts → autoscaled-pool.d.ts} +6 -6
  4. package/internals/autoscaling/{autoscaled_pool.js → autoscaled-pool.js} +3 -1
  5. package/internals/autoscaling/{concurrency_system.js → concurrency-system.js} +1 -1
  6. package/internals/autoscaling/{cpu_load_signal.d.ts → cpu-load-signal.d.ts} +1 -1
  7. package/internals/autoscaling/{cpu_load_signal.js → cpu-load-signal.js} +1 -1
  8. package/internals/autoscaling/{event_loop_load_signal.d.ts → event-loop-load-signal.d.ts} +1 -1
  9. package/internals/autoscaling/{event_loop_load_signal.js → event-loop-load-signal.js} +1 -1
  10. package/internals/autoscaling/index.d.ts +8 -8
  11. package/internals/autoscaling/index.js +8 -8
  12. package/internals/autoscaling/{load_signal.js → load-signal.js} +1 -1
  13. package/internals/autoscaling/{memory_load_signal.d.ts → memory-load-signal.d.ts} +1 -1
  14. package/internals/autoscaling/{memory_load_signal.js → memory-load-signal.js} +1 -1
  15. package/internals/autoscaling/snapshotter.d.ts +5 -5
  16. package/internals/autoscaling/snapshotter.js +4 -4
  17. package/internals/autoscaling/{storage_backend_load_signal.d.ts → storage-backend-load-signal.d.ts} +1 -1
  18. package/internals/autoscaling/{storage_backend_load_signal.js → storage-backend-load-signal.js} +1 -1
  19. package/internals/autoscaling/{system_status.d.ts → system-status.d.ts} +1 -1
  20. package/internals/autoscaling/{system_status.js → system-status.js} +1 -1
  21. package/internals/basic-crawler.d.ts +16 -11
  22. package/internals/basic-crawler.js +61 -66
  23. package/internals/{cookie_utils.js → cookie-utils.js} +1 -1
  24. package/internals/crawler-run.d.ts +2 -2
  25. package/internals/crawler-run.js +3 -1
  26. package/internals/crawlers/{crawler_commons.d.ts → crawler-commons.d.ts} +2 -2
  27. package/internals/crawlers/{error_snapshotter.d.ts → error-snapshotter.d.ts} +2 -2
  28. package/internals/crawlers/{error_tracker.d.ts → error-tracker.d.ts} +1 -1
  29. package/internals/crawlers/{error_tracker.js → error-tracker.js} +1 -1
  30. package/internals/crawlers/index.d.ts +3 -3
  31. package/internals/crawlers/index.js +2 -2
  32. package/internals/crawlers/statistics.d.ts +11 -1
  33. package/internals/crawlers/statistics.js +39 -5
  34. package/internals/{crawling_request.js → crawling-request.js} +1 -1
  35. package/internals/{enqueue_links/enqueue_links.d.ts → enqueue-links/enqueue-links.d.ts} +1 -6
  36. package/internals/{enqueue_links/enqueue_links.js → enqueue-links/enqueue-links.js} +0 -41
  37. package/internals/{enqueue_links → enqueue-links}/index.d.ts +1 -1
  38. package/internals/{enqueue_links → enqueue-links}/index.js +1 -1
  39. package/internals/{enqueue_links → enqueue-links}/shared.d.ts +5 -4
  40. package/internals/{enqueue_links → enqueue-links}/shared.js +15 -45
  41. package/internals/errors.d.ts +6 -1
  42. package/internals/errors.js +25 -0
  43. package/internals/router.d.ts +2 -2
  44. package/internals/send-request.js +17 -8
  45. package/internals/{session_pool → session-pool}/index.d.ts +1 -1
  46. package/internals/{session_pool → session-pool}/index.js +1 -1
  47. package/internals/{sitemap_request_loader.d.ts → sitemap-request-loader.d.ts} +8 -1
  48. package/internals/{sitemap_request_loader.js → sitemap-request-loader.js} +116 -56
  49. package/internals/{throttling_request_manager.d.ts → throttling-request-manager.d.ts} +7 -7
  50. package/internals/{throttling_request_manager.js → throttling-request-manager.js} +61 -44
  51. package/package.json +15 -15
  52. /package/internals/autoscaling/{concurrency_system.d.ts → concurrency-system.d.ts} +0 -0
  53. /package/internals/autoscaling/{load_signal.d.ts → load-signal.d.ts} +0 -0
  54. /package/internals/autoscaling/{weighted_avg.d.ts → weighted-avg.d.ts} +0 -0
  55. /package/internals/autoscaling/{weighted_avg.js → weighted-avg.js} +0 -0
  56. /package/internals/{cookie_utils.d.ts → cookie-utils.d.ts} +0 -0
  57. /package/internals/crawlers/{context_pipeline.d.ts → context-pipeline.d.ts} +0 -0
  58. /package/internals/crawlers/{context_pipeline.js → context-pipeline.js} +0 -0
  59. /package/internals/crawlers/{crawler_commons.js → crawler-commons.js} +0 -0
  60. /package/internals/crawlers/{error_snapshotter.js → error-snapshotter.js} +0 -0
  61. /package/internals/{crawling_request.d.ts → crawling-request.d.ts} +0 -0
  62. /package/internals/{session_pool → session-pool}/consts.d.ts +0 -0
  63. /package/internals/{session_pool → session-pool}/consts.js +0 -0
  64. /package/internals/{session_pool → session-pool}/errors.d.ts +0 -0
  65. /package/internals/{session_pool → session-pool}/errors.js +0 -0
  66. /package/internals/{session_pool → session-pool}/fingerprint.d.ts +0 -0
  67. /package/internals/{session_pool → session-pool}/fingerprint.js +0 -0
  68. /package/internals/{session_pool/session_pool.d.ts → session-pool/session-pool.d.ts} +0 -0
  69. /package/internals/{session_pool/session_pool.js → session-pool/session-pool.js} +0 -0
  70. /package/internals/{session_pool → session-pool}/session.d.ts +0 -0
  71. /package/internals/{session_pool → session-pool}/session.js +0 -0
package/index.d.ts CHANGED
@@ -1,13 +1,13 @@
1
1
  export * from '@crawlee/core';
2
2
  export * from './internals/autoscaling/index.js';
3
3
  export * from './internals/basic-crawler.js';
4
- export * from './internals/cookie_utils.js';
4
+ export * from './internals/cookie-utils.js';
5
5
  export * from './internals/crawlers/index.js';
6
- export * from './internals/crawling_request.js';
7
- export * from './internals/enqueue_links/index.js';
6
+ export * from './internals/crawling-request.js';
7
+ export * from './internals/enqueue-links/index.js';
8
8
  export * from './internals/errors.js';
9
9
  export * from './internals/http.js';
10
10
  export * from './internals/router.js';
11
- export * from './internals/session_pool/index.js';
12
- export * from './internals/sitemap_request_loader.js';
13
- export * from './internals/throttling_request_manager.js';
11
+ export * from './internals/session-pool/index.js';
12
+ export * from './internals/sitemap-request-loader.js';
13
+ export * from './internals/throttling-request-manager.js';
package/index.js CHANGED
@@ -1,13 +1,13 @@
1
1
  export * from '@crawlee/core';
2
2
  export * from './internals/autoscaling/index.js';
3
3
  export * from './internals/basic-crawler.js';
4
- export * from './internals/cookie_utils.js';
4
+ export * from './internals/cookie-utils.js';
5
5
  export * from './internals/crawlers/index.js';
6
- export * from './internals/crawling_request.js';
7
- export * from './internals/enqueue_links/index.js';
6
+ export * from './internals/crawling-request.js';
7
+ export * from './internals/enqueue-links/index.js';
8
8
  export * from './internals/errors.js';
9
9
  export * from './internals/http.js';
10
10
  export * from './internals/router.js';
11
- export * from './internals/session_pool/index.js';
12
- export * from './internals/sitemap_request_loader.js';
13
- export * from './internals/throttling_request_manager.js';
11
+ export * from './internals/session-pool/index.js';
12
+ export * from './internals/sitemap-request-loader.js';
13
+ export * from './internals/throttling-request-manager.js';
@@ -1,5 +1,5 @@
1
1
  import type { CrawleeLogger } from '@crawlee/core';
2
- import type { ConcurrencyConsumer, IConcurrencySystem } from './concurrency_system.js';
2
+ import type { ConcurrencyConsumer, IConcurrencySystem } from './concurrency-system.js';
3
3
  /**
4
4
  * The two predicates that steer a task loop: *is there work ready?* and *are we done?* These are the parts of the loop
5
5
  * a caller legitimately overrides, as opposed to the task itself (`runTaskFunction`), which the loop's driver owns.
@@ -20,15 +20,13 @@ export interface TaskLoopPredicates {
20
20
  */
21
21
  isFinishedFunction?: () => Promise<boolean>;
22
22
  }
23
- export interface TaskLoopOptions extends TaskLoopPredicates {
23
+ /** @internal */
24
+ export interface AutoscaledPoolOptions extends TaskLoopPredicates {
24
25
  /**
25
26
  * How often the pool should check if a new task is ready, in seconds.
26
27
  * @default 0.5
27
28
  */
28
29
  maybeRunIntervalSecs?: number;
29
- }
30
- /** @internal */
31
- export interface AutoscaledPoolOptions extends TaskLoopOptions {
32
30
  /**
33
31
  * The governor that decides whether there is free compute for one more task. Typically a
34
32
  * {@link ConcurrencySystem}, but any {@link IConcurrencySystem} works. Share a single instance across
@@ -60,7 +58,9 @@ export interface AutoscaledPoolOptions extends TaskLoopOptions {
60
58
  /**
61
59
  * Manages a pool of asynchronous resource-intensive tasks that are executed in parallel.
62
60
  * The pool only starts new tasks while its {@link IConcurrencySystem|concurrency system} reports free capacity —
63
- * that governor is what monitors CPU, memory and event loop load and autoscales the concurrency budget.
61
+ * that governor is what monitors CPU, memory and event loop load and autoscales the concurrency budget. The default
62
+ * {@link ConcurrencySystem} keeps `minConcurrency` tasks running even while the system is overloaded, and only
63
+ * starts additional ones when there is enough free CPU and memory.
64
64
  *
65
65
  * Before running the pool, you need to implement the following three functions:
66
66
  * {@link AutoscaledPoolOptions.runTaskFunction|`runTaskFunction`},
@@ -23,7 +23,9 @@ const autoscaledPoolOptionsSchema = z.strictObject({
23
23
  /**
24
24
  * Manages a pool of asynchronous resource-intensive tasks that are executed in parallel.
25
25
  * The pool only starts new tasks while its {@link IConcurrencySystem|concurrency system} reports free capacity —
26
- * that governor is what monitors CPU, memory and event loop load and autoscales the concurrency budget.
26
+ * that governor is what monitors CPU, memory and event loop load and autoscales the concurrency budget. The default
27
+ * {@link ConcurrencySystem} keeps `minConcurrency` tasks running even while the system is overloaded, and only
28
+ * starts additional ones when there is enough free CPU and memory.
27
29
  *
28
30
  * Before running the pool, you need to implement the following three functions:
29
31
  * {@link AutoscaledPoolOptions.runTaskFunction|`runTaskFunction`},
@@ -3,7 +3,7 @@ import { parseArgument, schemas } from '@crawlee/utils/internal';
3
3
  import { z } from 'zod';
4
4
  import { betterClearInterval, betterSetInterval } from '@apify/utilities';
5
5
  import { Snapshotter } from './snapshotter.js';
6
- import { SystemStatus } from './system_status.js';
6
+ import { SystemStatus } from './system-status.js';
7
7
  const concurrencySchema = z.number().int().gte(1).optional();
8
8
  // `schemas.anyObject` passes values through by reference, so the load signal instances inside
9
9
  // `loadSignals` and class instances like loggers keep their prototype.
@@ -1,5 +1,5 @@
1
1
  import type { SystemInfo } from '@crawlee/core';
2
- import type { LoadSignal, LoadSignalStartContext, LoadSnapshot } from './load_signal.js';
2
+ import type { LoadSignal, LoadSignalStartContext, LoadSnapshot } from './load-signal.js';
3
3
  /**
4
4
  * A snapshot produced by the built-in CPU signal.
5
5
  * @internal
@@ -1,5 +1,5 @@
1
1
  import { EventType, serviceLocator } from '@crawlee/core';
2
- import { SnapshotStore } from './load_signal.js';
2
+ import { SnapshotStore } from './load-signal.js';
3
3
  /**
4
4
  * Tracks CPU usage via `SYSTEM_INFO` events and reports overload when the platform or local OS metrics indicate the
5
5
  * CPU is overloaded.
@@ -1,4 +1,4 @@
1
- import type { LoadSignal, LoadSignalStartContext, LoadSnapshot } from './load_signal.js';
1
+ import type { LoadSignal, LoadSignalStartContext, LoadSnapshot } from './load-signal.js';
2
2
  /**
3
3
  * A snapshot produced by the built-in event loop signal.
4
4
  * @internal
@@ -1,5 +1,5 @@
1
1
  import { betterClearInterval, betterSetInterval } from '@apify/utilities';
2
- import { SnapshotStore } from './load_signal.js';
2
+ import { SnapshotStore } from './load-signal.js';
3
3
  /**
4
4
  * Periodically measures event loop delay and reports overload when the delay exceeds a configured threshold.
5
5
  *
@@ -1,9 +1,9 @@
1
- export * from './autoscaled_pool.js';
2
- export * from './concurrency_system.js';
3
- export * from './cpu_load_signal.js';
4
- export * from './event_loop_load_signal.js';
5
- export * from './load_signal.js';
6
- export * from './memory_load_signal.js';
1
+ export * from './autoscaled-pool.js';
2
+ export * from './concurrency-system.js';
3
+ export * from './cpu-load-signal.js';
4
+ export * from './event-loop-load-signal.js';
5
+ export * from './load-signal.js';
6
+ export * from './memory-load-signal.js';
7
7
  export * from './snapshotter.js';
8
- export * from './storage_backend_load_signal.js';
9
- export * from './system_status.js';
8
+ export * from './storage-backend-load-signal.js';
9
+ export * from './system-status.js';
@@ -1,9 +1,9 @@
1
- export * from './autoscaled_pool.js';
2
- export * from './concurrency_system.js';
3
- export * from './cpu_load_signal.js';
4
- export * from './event_loop_load_signal.js';
5
- export * from './load_signal.js';
6
- export * from './memory_load_signal.js';
1
+ export * from './autoscaled-pool.js';
2
+ export * from './concurrency-system.js';
3
+ export * from './cpu-load-signal.js';
4
+ export * from './event-loop-load-signal.js';
5
+ export * from './load-signal.js';
6
+ export * from './memory-load-signal.js';
7
7
  export * from './snapshotter.js';
8
- export * from './storage_backend_load_signal.js';
9
- export * from './system_status.js';
8
+ export * from './storage-backend-load-signal.js';
9
+ export * from './system-status.js';
@@ -1,4 +1,4 @@
1
- import { weightedAvg } from './weighted_avg.js';
1
+ import { weightedAvg } from './weighted-avg.js';
2
2
  /**
3
3
  * A time-pruning, time-windowed store for `LoadSnapshot` values. All four built-in signals compose with one of these,
4
4
  * and so can yours — it is the only part of their machinery worth reusing.
@@ -1,5 +1,5 @@
1
1
  import type { SystemInfo } from '@crawlee/core';
2
- import type { LoadSignal, LoadSignalStartContext, LoadSnapshot } from './load_signal.js';
2
+ import type { LoadSignal, LoadSignalStartContext, LoadSnapshot } from './load-signal.js';
3
3
  /**
4
4
  * A snapshot produced by the built-in memory signal.
5
5
  * @internal
@@ -1,6 +1,6 @@
1
1
  import { EventType, serviceLocator } from '@crawlee/core';
2
2
  import { getMemoryInfo, isContainerized } from '@crawlee/core/internal';
3
- import { SnapshotStore } from './load_signal.js';
3
+ import { SnapshotStore } from './load-signal.js';
4
4
  const RESERVE_MEMORY_RATIO = 0.5;
5
5
  const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10_000;
6
6
  /**
@@ -1,8 +1,8 @@
1
- import type { CpuLoadSignalOptions } from './cpu_load_signal.js';
2
- import type { EventLoopLoadSignalOptions } from './event_loop_load_signal.js';
3
- import type { LoadSignal, LoadSignalStartContext } from './load_signal.js';
4
- import type { MemoryLoadSignalOptions } from './memory_load_signal.js';
5
- import type { StorageBackendLoadSignalOptions } from './storage_backend_load_signal.js';
1
+ import type { CpuLoadSignalOptions } from './cpu-load-signal.js';
2
+ import type { EventLoopLoadSignalOptions } from './event-loop-load-signal.js';
3
+ import type { LoadSignal, LoadSignalStartContext } from './load-signal.js';
4
+ import type { MemoryLoadSignalOptions } from './memory-load-signal.js';
5
+ import type { StorageBackendLoadSignalOptions } from './storage-backend-load-signal.js';
6
6
  /**
7
7
  * The load signals a {@link ConcurrencySystem} watches to decide whether the machine is overloaded.
8
8
  *
@@ -1,7 +1,7 @@
1
- import { CpuLoadSignal } from './cpu_load_signal.js';
2
- import { EventLoopLoadSignal } from './event_loop_load_signal.js';
3
- import { MemoryLoadSignal } from './memory_load_signal.js';
4
- import { StorageBackendLoadSignal } from './storage_backend_load_signal.js';
1
+ import { CpuLoadSignal } from './cpu-load-signal.js';
2
+ import { EventLoopLoadSignal } from './event-loop-load-signal.js';
3
+ import { MemoryLoadSignal } from './memory-load-signal.js';
4
+ import { StorageBackendLoadSignal } from './storage-backend-load-signal.js';
5
5
  /**
6
6
  * Owns the four built-in {@link LoadSignal} instances — {@link MemoryLoadSignal},
7
7
  * {@link EventLoopLoadSignal}, {@link CpuLoadSignal} and {@link StorageBackendLoadSignal} — constructing
@@ -1,4 +1,4 @@
1
- import type { LoadSignal, LoadSignalStartContext, LoadSnapshot } from './load_signal.js';
1
+ import type { LoadSignal, LoadSignalStartContext, LoadSnapshot } from './load-signal.js';
2
2
  /**
3
3
  * A snapshot produced by the built-in storage backend (rate-limit) signal.
4
4
  * @internal
@@ -1,6 +1,6 @@
1
1
  import { serviceLocator } from '@crawlee/core';
2
2
  import { betterClearInterval, betterSetInterval } from '@apify/utilities';
3
- import { SnapshotStore } from './load_signal.js';
3
+ import { SnapshotStore } from './load-signal.js';
4
4
  const RATE_LIMIT_ERROR_RETRY_COUNT = 2;
5
5
  /**
6
6
  * Periodically checks the storage backend for rate-limit errors (HTTP 429) and reports overload when the error delta
@@ -1,5 +1,5 @@
1
1
  import type { SystemInfo } from '@crawlee/core';
2
- import type { LoadSignal } from './load_signal.js';
2
+ import type { LoadSignal } from './load-signal.js';
3
3
  import type { Snapshotter } from './snapshotter.js';
4
4
  /**
5
5
  * How far back the *current* system status looks by default — the window that gates task dispatch.
@@ -1,4 +1,4 @@
1
- import { evaluateLoadSignalSample } from './load_signal.js';
1
+ import { evaluateLoadSignalSample } from './load-signal.js';
2
2
  /**
3
3
  * How far back the *current* system status looks by default — the window that gates task dispatch.
4
4
  * @internal
@@ -5,14 +5,15 @@ import type { Awaitable, Dictionary, ISessionPool, SetStatusMessageOptions, Stor
5
5
  import { RobotsTxtFile } from '@crawlee/utils';
6
6
  import type { ReadonlyDeep } from 'type-fest';
7
7
  import { z } from 'zod';
8
- import type { TaskLoopOptions } from './autoscaling/autoscaled_pool.js';
9
- import type { ConcurrencySystemOptions, IConcurrencySystem } from './autoscaling/concurrency_system.js';
10
- import { ConcurrencySystem } from './autoscaling/concurrency_system.js';
11
- import type { FinalStatistics } from './autoscaling/system_status.js';
12
- import { ContextPipeline } from './crawlers/context_pipeline.js';
13
- import type { CrawlingContext, TypedRequestsLike } from './crawlers/crawler_commons.js';
8
+ import type { TaskLoopPredicates } from './autoscaling/autoscaled-pool.js';
9
+ import type { ConcurrencySystemOptions, IConcurrencySystem } from './autoscaling/concurrency-system.js';
10
+ import { ConcurrencySystem } from './autoscaling/concurrency-system.js';
11
+ import type { FinalStatistics } from './autoscaling/system-status.js';
12
+ import type { ContextMiddleware } from './crawlers/context-pipeline.js';
13
+ import { ContextPipeline } from './crawlers/context-pipeline.js';
14
+ import type { CrawlingContext, TypedRequestsLike } from './crawlers/crawler-commons.js';
14
15
  import type { IStatistics, StatisticState } from './crawlers/statistics.js';
15
- import type { EnqueueUrlsOptions, SkippedRequestCallback } from './enqueue_links/index.js';
16
+ import type { EnqueueUrlsOptions, SkippedRequestCallback } from './enqueue-links/index.js';
16
17
  import type { GetUserDataFromRequest, RouterHandler, RouterRoutes } from './router.js';
17
18
  export interface BasicCrawlingContext<UserData extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
18
19
  }
@@ -65,6 +66,10 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
65
66
  * **not** include navigation-dependent members (e.g. `page`, `response`, `$`, `body`). If you need
66
67
  * those, use a `postNavigationHook` or the `requestHandler` instead.
67
68
  *
69
+ * The second argument is the same {@link CleanupRegistrar|`onCleanup`} registrar used by
70
+ * {@link ContextPipeline.compose}. Use it to register teardown callbacks that run after the
71
+ * request handler or a pipeline failure, in reverse registration order.
72
+ *
68
73
  * **Example usage:**
69
74
  *
70
75
  * ```javascript
@@ -83,7 +88,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
83
88
  * });
84
89
  * ```
85
90
  */
86
- extendContext?: (context: CrawlingContext) => Awaitable<ContextExtension>;
91
+ extendContext?: ContextMiddleware<CrawlingContext, ContextExtension>;
87
92
  /**
88
93
  * *Intended for BasicCrawler subclasses*. Prepares a context pipeline that transforms the initial crawling context into the shape given by the `Context` type parameter.
89
94
  *
@@ -174,7 +179,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
174
179
  * Concurrency is configured elsewhere — through the `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute`
175
180
  * shortcuts, or a {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} for finer control.
176
181
  */
177
- taskLoopOptions?: TaskLoopOptions;
182
+ taskLoopOptions?: TaskLoopPredicates;
178
183
  /**
179
184
  * A pre-configured concurrency governor — the component that decides whether there is free compute for one more
180
185
  * task. Typically a {@link ConcurrencySystem}, though any {@link IConcurrencySystem} is accepted. All
@@ -278,7 +283,9 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
278
283
  *
279
284
  * Currently supports:
280
285
  * - [**Cloudflare** Bot Management](https://www.cloudflare.com/products/bot-management/)
286
+ * - [**AWS WAF** Challenge and CAPTCHA](https://docs.aws.amazon.com/waf/latest/developerguide/waf-captcha-and-challenge.html)
281
287
  * - [**Google Search** Rate Limiting](https://www.google.com/sorry/)
288
+ * - **Imperva Incapsula**
282
289
  */
283
290
  retryOnBlocked?: boolean;
284
291
  /**
@@ -654,8 +661,6 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
654
661
  protected addCrawlDepthRequestGenerator(requests: RequestsLike, newRequestDepth: number): AsyncGenerator<Source, void, undefined>;
655
662
  /**
656
663
  * Handles errors thrown by user provided requestHandler()
657
- *
658
- * @param request The request object, passed separately to circumvent potential dynamic logic in crawlingContext.request
659
664
  */
660
665
  private requestFunctionErrorHandler;
661
666
  private handleFailedRequestHandler;
@@ -2,29 +2,28 @@ import { mkdir, writeFile } from 'node:fs/promises';
2
2
  import { dirname } from 'node:path';
3
3
  import { bindMethodsToServiceLocator, Configuration, createStorageTransaction, Request, CriticalError, currentStorageTransaction, Dataset, EnqueueStrategy, EventManager, EventType, getObjectType, KeyValueStore, log, NonRetryableError, OwnedOrInjected, purgeDefaultStorages, RequestManagerTandem, RequestQueue, ServiceLocator, serviceLocator, SessionError, validators, withDirectStorageAccess, } from '@crawlee/core';
4
4
  import { BaseHttpClient, FetchHttpClient } from '@crawlee/http-client';
5
- import { isAsyncIterable, isIterable, parseArgument, ROTATE_PROXY_ERRORS, schemas } from '@crawlee/utils/internal';
5
+ import { filterUrl, isAsyncIterable, isIterable, matchesEnqueueStrategy, parseArgument, ROTATE_PROXY_ERRORS, schemas, } from '@crawlee/utils/internal';
6
6
  import { RobotsTxtFile } from '@crawlee/utils';
7
- import { getDomain } from 'tldts';
8
7
  import { z } from 'zod';
9
8
  import { LruCache } from '@apify/datastructures';
10
9
  import { addTimeoutToPromise, extendTimeout, storage as timeoutStorage, TimeoutError, tryCancel } from '@apify/timeout';
11
10
  import { cryptoRandomObjectId } from '@apify/utilities';
12
- import { ConcurrencySystem } from './autoscaling/concurrency_system.js';
13
- import { mergeCookies } from './cookie_utils.js';
14
- import { ContextPipeline } from './crawlers/context_pipeline.js';
15
- import { CrawlingRequest, RequestState } from './crawling_request.js';
11
+ import { ConcurrencySystem } from './autoscaling/concurrency-system.js';
12
+ import { mergeCookies } from './cookie-utils.js';
13
+ import { ContextPipeline } from './crawlers/context-pipeline.js';
14
+ import { CrawlingRequest, RequestState } from './crawling-request.js';
16
15
  import { ContextPipelineCleanupError, ContextPipelineInterruptedError, MissingSessionError, NavigationSkippedError, PersistentRateLimitError, ContextPipelineInitializationError, RequestHandlerError, RequestThrottledError, RetryRequestError, } from './errors.js';
17
16
  import { Statistics } from './crawlers/statistics.js';
18
- import { applyRequestTransform, buildEnqueueStrategyPatterns, constructUrlPatternObjects, createRequestOptions, createSkippedRequestArgs, filterRequestOptionsByPatterns, } from './enqueue_links/index.js';
17
+ import { applyRequestTransform, constructUrlPatternObjects, createRequestOptions, createSkippedRequestArgs, filterRequestOptionsByPatterns, } from './enqueue-links/index.js';
19
18
  import { parseRetryAfterHeader } from './http.js';
20
19
  import { CrawlerRun } from './crawler-run.js';
21
20
  import { extendTimeoutKey, navigationDeadlineKey, raceWithTimeout, suspendTimeoutKey, timeoutExpiredKey, } from './request-timeout.js';
22
21
  import { createSendRequest } from './send-request.js';
23
22
  import { Router, validateUserData } from './router.js';
24
- import { BLOCKED_STATUS_CODES } from './session_pool/consts.js';
25
- import { Session } from './session_pool/session.js';
26
- import { SessionPool } from './session_pool/session_pool.js';
27
- import { ThrottlingRequestManager } from './throttling_request_manager.js';
23
+ import { BLOCKED_STATUS_CODES } from './session-pool/consts.js';
24
+ import { Session } from './session-pool/session.js';
25
+ import { SessionPool } from './session-pool/session-pool.js';
26
+ import { ThrottlingRequestManager } from './throttling-request-manager.js';
28
27
  class LazyDefaultHttpClient extends BaseHttpClient {
29
28
  #delegatePromise;
30
29
  constructor(options) {
@@ -471,14 +470,14 @@ export class BasicCrawler {
471
470
  // Navigation, the navigation hooks and the request handler are timed individually, but the
472
471
  // phases between them are not, so a request could still get stuck indefinitely. This is the
473
472
  // catch-all for that - see `raceWithTimeout` for why it is a bare timer, not a timeout frame.
474
- await this.#withRequestTimeout(crawlingContext, this.#basicPipeline.chain(this.contextPipeline).call(crawlingContext, (ctx) => this.handleRequest(ctx, source, request), (error) => this.#handleRequestFailure(error, crawlingContext, source, request))));
473
+ await this.#withRequestTimeout(crawlingContext, this.#basicPipeline.chain(this.contextPipeline).call(crawlingContext, (ctx) => this.handleRequest(ctx, source), (error) => this.#handleRequestFailure(error, crawlingContext, source))));
475
474
  }
476
475
  catch (error) {
477
476
  // The internal timeout above fired, or `handleRequestFailure` threw a `SessionError` - both
478
477
  // take the normal error flow. Anything else is fatal.
479
478
  if (error instanceof SessionError || crawlingContext[timeoutExpiredKey]?.()) {
480
479
  const unwrappedError = this.#unwrapError(error);
481
- await this.requestFunctionErrorHandler(unwrappedError, crawlingContext, request, this.requestManager);
480
+ await this.requestFunctionErrorHandler(unwrappedError, crawlingContext, this.requestManager);
482
481
  if (!this.#errorAbsolvesSession(unwrappedError)) {
483
482
  crawlingContext.session?.markBad();
484
483
  }
@@ -511,7 +510,7 @@ export class BasicCrawler {
511
510
  this.log.info('The crawler has finished all the remaining ongoing requests and will shut down now.');
512
511
  return true;
513
512
  }
514
- // `maybeFinish()` calls this only once nothing is in flight (`autoscaled_pool.ts`) - the point
513
+ // `maybeFinish()` calls this only once nothing is in flight (`autoscaled-pool.ts`) - the point
515
514
  // where a crawl that cannot progress becomes distinguishable from one that is merely waiting,
516
515
  // and the only place where throwing does not abandon requests mid-processing.
517
516
  const state = await this.requestManager?.checkReadiness();
@@ -625,7 +624,7 @@ export class BasicCrawler {
625
624
  context[navigationDeadlineKey] += extraMillis;
626
625
  }
627
626
  // Extension failure must not fail the request that asked for more time.
628
- this.requestManager?.extendRequestProcessingTimeSecs?.(context.request, secs)?.catch((error) => {
627
+ this.requestManager?.extendRequestProcessingTime?.(context.request, secs)?.catch((error) => {
629
628
  this.log.debug('Extending the request processing time failed', { url: context.request.url, error });
630
629
  });
631
630
  },
@@ -686,7 +685,7 @@ export class BasicCrawler {
686
685
  let contextPipeline;
687
686
  if (extendContext !== undefined) {
688
687
  contextPipeline = ContextPipeline.create()
689
- .compose(async (context) => await extendContext(context))
688
+ .compose(extendContext)
690
689
  .chain(subclassPipeline);
691
690
  }
692
691
  else {
@@ -795,18 +794,6 @@ export class BasicCrawler {
795
794
  if (this.running) {
796
795
  throw new Error('This crawler instance is already running, you can add more requests to it via `crawler.addRequests()`.');
797
796
  }
798
- if (this.#hasFinishedBefore) {
799
- // A supplied statistics instance keeps whatever state it was handed - only wipe a default we built.
800
- await this.#statisticsDep.ifOwned(async (stats) => {
801
- stats.reset();
802
- await stats.resetStore();
803
- });
804
- // A session pool the crawler built follows the same blank-slate rule as its statistics.
805
- await this.#sessionPoolDep.ifOwned(async (pool) => {
806
- pool.reset();
807
- await pool.resetStore();
808
- });
809
- }
810
797
  // An owned governor is rebuilt for every run, so it always starts from a clean slate — stale
811
798
  // resource snapshots or a previous run's scaled desired concurrency would otherwise distort this
812
799
  // run's scaling. An injected one is long-lived and its lifecycle belongs to the caller.
@@ -819,6 +806,18 @@ export class BasicCrawler {
819
806
  });
820
807
  this.#run = run;
821
808
  try {
809
+ if (this.#hasFinishedBefore) {
810
+ // A supplied statistics instance keeps whatever state it was handed - only wipe a default we built.
811
+ await this.#statisticsDep.ifOwned(async (stats) => {
812
+ stats.reset();
813
+ await stats.resetStore();
814
+ });
815
+ // A session pool the crawler built follows the same blank-slate rule as its statistics.
816
+ await this.#sessionPoolDep.ifOwned(async (pool) => {
817
+ pool.reset();
818
+ await pool.resetStore();
819
+ });
820
+ }
822
821
  await purgeDefaultStorages({
823
822
  onlyPurgeOnce: true,
824
823
  storageBackend: serviceLocator.getStorageBackend(),
@@ -970,7 +969,7 @@ export class BasicCrawler {
970
969
  this.#requestManager = await this.#openOwnedRequestQueue();
971
970
  }
972
971
  // Apply the processing-time hint here (an async lifecycle point) rather than in the constructor,
973
- // now that `setExpectedRequestProcessingTimeSecs` is async. The hint is raise-only and idempotent,
972
+ // now that `setExpectedRequestProcessingTime` is async. The hint is raise-only and idempotent,
974
973
  // but guard so we do not re-issue it on every call.
975
974
  if (!this.#requestManagerTimeoutsApplied) {
976
975
  this.#requestManagerTimeoutsApplied = true;
@@ -1007,7 +1006,7 @@ export class BasicCrawler {
1007
1006
  // raise-only, so erring high here is safe.
1008
1007
  const maxRouteTimeoutSecs = this.requestHandler.getMaxTimeoutSecs?.() ?? 0;
1009
1008
  const handlerTimeoutSecs = Math.max(this.#requestHandlerTimeoutMillis / 1000, maxRouteTimeoutSecs);
1010
- await requestManager.setExpectedRequestProcessingTimeSecs?.(Math.max(handlerTimeoutSecs + 5, 60));
1009
+ await requestManager.setExpectedRequestProcessingTime?.(Math.max(handlerTimeoutSecs + 5, 60));
1011
1010
  }
1012
1011
  /**
1013
1012
  * Validates a request source's `userData` against the {@link RouteSchemas|Standard Schema} registered
@@ -1116,7 +1115,7 @@ export class BasicCrawler {
1116
1115
  const urlPatternObjects = options.include?.length ? constructUrlPatternObjects(options.include) : [];
1117
1116
  // The strategy always applies, even when `include` patterns are provided - the two are AND-ed together
1118
1117
  // (a URL must match an `include` pattern *and* satisfy the strategy). This mirrors crawlee-python.
1119
- const enqueueStrategyPatterns = options.baseUrl ? buildEnqueueStrategyPatterns(options.baseUrl, strategy) : [];
1118
+ const strategyBaseUrl = options.baseUrl ? new URL(options.baseUrl) : undefined;
1120
1119
  const isAllowedBasedOnRobotsTxtFile = this.#isAllowedBasedOnRobotsTxtFile.bind(this);
1121
1120
  const maxCrawlDepth = this.#maxCrawlDepth;
1122
1121
  const validateRequestUserData = this.#validateRequestUserData.bind(this);
@@ -1167,7 +1166,14 @@ export class BasicCrawler {
1167
1166
  // Filter by user patterns first (with exclude)...
1168
1167
  let filtered = filterRequestOptionsByPatterns([requestOptions], urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, strategy, onSkippedByFilter);
1169
1168
  // ...then filter by the enqueue strategy (making this an AND check)
1170
- filtered = filterRequestOptionsByPatterns(filtered, enqueueStrategyPatterns.length > 0 ? enqueueStrategyPatterns : undefined, [], strategy, onSkippedByFilter);
1169
+ if (strategyBaseUrl) {
1170
+ filtered = filtered.filter((opts) => {
1171
+ const { allowed } = filterUrl(opts.url, strategyBaseUrl, strategy);
1172
+ if (!allowed)
1173
+ onSkippedByFilter(opts);
1174
+ return allowed;
1175
+ });
1176
+ }
1171
1177
  if (filtered.length === 0) {
1172
1178
  continue;
1173
1179
  }
@@ -1185,7 +1191,7 @@ export class BasicCrawler {
1185
1191
  }
1186
1192
  const result = await this.requestManager.addRequestsBatched(filteredRequests(), {
1187
1193
  forefront: options.forefront,
1188
- waitForAllRequestsToBeAdded: options.waitForAllRequestsToBeAdded,
1194
+ waitForAllRequestsToBeAdded: options.waitForAllRequestsToBeAdded || requestLimit !== undefined,
1189
1195
  batchSize: options.batchSize,
1190
1196
  waitBetweenBatchesMillis: options.waitBetweenBatchesMillis,
1191
1197
  maxNewRequests: requestLimit,
@@ -1202,6 +1208,7 @@ export class BasicCrawler {
1202
1208
  });
1203
1209
  // Keep callback failures observable to callers that await this promise without emitting an unhandled rejection
1204
1210
  // when callers intentionally leave background additions running, matching `drainRequestBatches` behavior.
1211
+ // Background failures are already logged by `drainRequestBatches`.
1205
1212
  void waitForAllRequestsToBeAdded.catch(() => { });
1206
1213
  return { ...result, waitForAllRequestsToBeAdded };
1207
1214
  }
@@ -1455,7 +1462,15 @@ export class BasicCrawler {
1455
1462
  if (cachedRobotsTxtFile) {
1456
1463
  return cachedRobotsTxtFile;
1457
1464
  }
1458
- const robotsTxtFile = await RobotsTxtFile.find(url, { logger: this.log });
1465
+ // Fetch robots.txt the way pages are fetched: through the crawler's HTTP client, with a session's browser
1466
+ // fingerprint and proxy. Sites with bot protection answer a bare request with 4xx or 5xx, and that status
1467
+ // alone would then decide whether everything is allowed or disallowed.
1468
+ const robotsTxtFile = await RobotsTxtFile.find(url, {
1469
+ httpClient: this.httpClient,
1470
+ session: await this.sessionPool.getSession(),
1471
+ timeoutMillis: 30_000,
1472
+ logger: this.log,
1473
+ });
1459
1474
  this.#robotsTxtFileCache.add(origin, robotsTxtFile);
1460
1475
  return robotsTxtFile;
1461
1476
  }
@@ -1492,13 +1507,14 @@ export class BasicCrawler {
1492
1507
  }
1493
1508
  /** Runs the request handler and marks the request as handled. Failures are left to `handleRequestFailure`. */
1494
1509
  // oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
1495
- async handleRequest(crawlingContext, requestSource, request) {
1510
+ async handleRequest(crawlingContext, requestSource) {
1496
1511
  // An earlier phase we cannot cancel (e.g. a slow `extendContext`) may have run past the internal timeout,
1497
1512
  // which already failed the request in `runTaskFunction`. Bail before running the handler so it does not
1498
1513
  // execute (and re-report) on top of a request the crawler has already moved past.
1499
1514
  if (crawlingContext[timeoutExpiredKey]?.()) {
1500
1515
  return;
1501
1516
  }
1517
+ const { request } = crawlingContext;
1502
1518
  request.state = RequestState.REQUEST_HANDLER;
1503
1519
  await this.runRequestHandler(crawlingContext);
1504
1520
  // Commit *before* marking the request as handled, so a commit failure fails the request and
@@ -1516,7 +1532,8 @@ export class BasicCrawler {
1516
1532
  * or `handleRequest`. Runs inside the pipeline call, so the cleanups (e.g. closing the page) only fire after the
1517
1533
  * error handler has seen the context.
1518
1534
  */
1519
- async #handleRequestFailure(rawError, crawlingContext, requestSource, request) {
1535
+ async #handleRequestFailure(rawError, crawlingContext, requestSource) {
1536
+ const { request } = crawlingContext;
1520
1537
  // Roll back *before* any error handler runs - error handlers write to real storage precisely
1521
1538
  // because the transaction is already closed. A no-op when the commit in `handleRequest` succeeded.
1522
1539
  currentStorageTransaction()?.rollback();
@@ -1528,18 +1545,19 @@ export class BasicCrawler {
1528
1545
  ? await this.#trySkipRequest(unwrappedError, request, requestSource)
1529
1546
  : unwrappedError;
1530
1547
  if (err) {
1531
- await this.#runErrorHandler(err, crawlingContext, requestSource, request);
1548
+ await this.#runErrorHandler(err, crawlingContext, requestSource);
1532
1549
  }
1533
1550
  }
1534
1551
  finally {
1535
1552
  resumeRequestTimeout?.();
1536
1553
  }
1537
1554
  }
1538
- async #runErrorHandler(err, context, requestSource, request) {
1555
+ async #runErrorHandler(err, context, requestSource) {
1556
+ const { request } = context;
1539
1557
  const retryCountBefore = request.retryCount;
1540
1558
  try {
1541
1559
  request.state = RequestState.ERROR_HANDLER;
1542
- await addTimeoutToPromise(async () => this.requestFunctionErrorHandler(err, context, request, requestSource), this.internalTimeoutMillis, `Handling request failure of ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
1560
+ await addTimeoutToPromise(async () => this.requestFunctionErrorHandler(err, context, requestSource), this.internalTimeoutMillis, `Handling request failure of ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
1543
1561
  // The error handler may have set `SKIPPED`; TS keeps the narrowing from the assignment above.
1544
1562
  if (request.state !== RequestState.SKIPPED) {
1545
1563
  request.state = RequestState.DONE;
@@ -1648,11 +1666,10 @@ export class BasicCrawler {
1648
1666
  }
1649
1667
  /**
1650
1668
  * Handles errors thrown by user provided requestHandler()
1651
- *
1652
- * @param request The request object, passed separately to circumvent potential dynamic logic in crawlingContext.request
1653
1669
  */
1654
1670
  // oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
1655
- async requestFunctionErrorHandler(error, crawlingContext, request, source) {
1671
+ async requestFunctionErrorHandler(error, crawlingContext, source) {
1672
+ const { request } = crawlingContext;
1656
1673
  if (error instanceof RequestThrottledError) {
1657
1674
  // The domain told us to come back later, so the request was never really attempted. Put it back where
1658
1675
  // it was, without recording a failure - it costs neither a retry nor session reputation.
@@ -1695,6 +1712,7 @@ export class BasicCrawler {
1695
1712
  }
1696
1713
  if (!request.noRetry) {
1697
1714
  request.retryCount++;
1715
+ this.statistics.registerRetry?.(request.retryCount);
1698
1716
  const { url, retryCount, id } = request;
1699
1717
  // We don't want to see the stack trace in the logs by default, when we are going to retry the request.
1700
1718
  // Thus, we print the full stack trace only when CRAWLEE_VERBOSE_LOG environment variable is set to true.
@@ -1867,30 +1885,7 @@ export class BasicCrawler {
1867
1885
  if (!loadedUrl) {
1868
1886
  return true;
1869
1887
  }
1870
- const baseUrl = new URL(url);
1871
- const loadedBaseUrl = new URL(loadedUrl);
1872
- switch (strategy) {
1873
- case EnqueueStrategy.SameHostname: {
1874
- return baseUrl.hostname === loadedBaseUrl.hostname;
1875
- }
1876
- case EnqueueStrategy.SameDomain: {
1877
- const baseUrlHostname = getDomain(baseUrl.hostname, { mixedInputs: false });
1878
- if (baseUrlHostname) {
1879
- const loadedBaseUrlHostname = getDomain(loadedBaseUrl.hostname, { mixedInputs: false });
1880
- return baseUrlHostname === loadedBaseUrlHostname;
1881
- }
1882
- // Can happen for IPs, we just check like same origin
1883
- return baseUrl.origin === loadedBaseUrl.origin;
1884
- }
1885
- case EnqueueStrategy.SameOrigin: {
1886
- // Same as hostname, but also checks protocol
1887
- return baseUrl.origin === loadedBaseUrl.origin;
1888
- }
1889
- case EnqueueStrategy.All:
1890
- default: {
1891
- return baseUrl.protocol === 'http:' || baseUrl.protocol === 'https:';
1892
- }
1893
- }
1888
+ return matchesEnqueueStrategy(strategy, new URL(loadedUrl), new URL(url));
1894
1889
  }
1895
1890
  }
1896
1891
  /** The hostname of `url`, falling back to the whole string when it is not parseable - for log messages only. */