@crawlee/core 4.0.0-beta.24 → 4.0.0-beta.240

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (265) hide show
  1. package/README.md +15 -15
  2. package/byte-utils.d.ts +17 -0
  3. package/byte-utils.js +42 -0
  4. package/configuration.d.ts +77 -235
  5. package/configuration.js +157 -227
  6. package/debug.d.ts +36 -0
  7. package/debug.js +70 -0
  8. package/errors.d.ts +45 -18
  9. package/errors.js +58 -27
  10. package/events/event-manager.d.ts +78 -0
  11. package/events/{event_manager.js → event-manager.js} +12 -13
  12. package/events/index.d.ts +3 -3
  13. package/events/index.js +2 -3
  14. package/events/local-event-manager.d.ts +32 -0
  15. package/events/local-event-manager.js +101 -0
  16. package/events/system-info.d.ts +38 -0
  17. package/events/system-info.js +1 -0
  18. package/index.d.ts +9 -12
  19. package/index.js +10 -11
  20. package/internal.d.ts +8 -0
  21. package/internal.js +9 -0
  22. package/iterables.d.ts +79 -0
  23. package/iterables.js +134 -0
  24. package/log.d.ts +77 -1
  25. package/log.js +134 -1
  26. package/memory-storage/consts.d.ts +4 -0
  27. package/memory-storage/consts.js +4 -0
  28. package/memory-storage/index.d.ts +1 -0
  29. package/memory-storage/index.js +1 -0
  30. package/memory-storage/memory-storage.d.ts +43 -0
  31. package/memory-storage/memory-storage.js +158 -0
  32. package/memory-storage/resource-clients/common/base-client.d.ts +4 -0
  33. package/memory-storage/resource-clients/common/base-client.js +6 -0
  34. package/memory-storage/resource-clients/dataset.d.ts +35 -0
  35. package/memory-storage/resource-clients/dataset.js +105 -0
  36. package/memory-storage/resource-clients/key-value-store.d.ts +54 -0
  37. package/memory-storage/resource-clients/key-value-store.js +179 -0
  38. package/memory-storage/resource-clients/request-queue.d.ts +55 -0
  39. package/memory-storage/resource-clients/request-queue.js +413 -0
  40. package/memory-storage/utils.d.ts +16 -0
  41. package/memory-storage/utils.js +41 -0
  42. package/owned-or-injected.d.ts +58 -0
  43. package/owned-or-injected.js +98 -0
  44. package/package.json +20 -20
  45. package/proxy-configuration.d.ts +118 -0
  46. package/proxy-configuration.js +141 -0
  47. package/recoverable-state.d.ts +177 -0
  48. package/recoverable-state.js +268 -0
  49. package/request.d.ts +48 -93
  50. package/request.js +112 -230
  51. package/serialization.d.ts +0 -1
  52. package/serialization.js +18 -18
  53. package/service-locator.d.ts +168 -0
  54. package/service-locator.js +255 -0
  55. package/storages/batched-adds.d.ts +37 -0
  56. package/storages/batched-adds.js +73 -0
  57. package/storages/dataset.d.ts +105 -58
  58. package/storages/dataset.js +292 -150
  59. package/storages/index.d.ts +11 -12
  60. package/storages/index.js +7 -12
  61. package/storages/key-value-store-codec.d.ts +32 -0
  62. package/storages/key-value-store-codec.js +108 -0
  63. package/storages/{key_value_store.d.ts → key-value-store.d.ts} +169 -72
  64. package/storages/key-value-store.js +767 -0
  65. package/storages/request-dedup-cache.d.ts +20 -0
  66. package/storages/request-dedup-cache.js +48 -0
  67. package/storages/{request_list.d.ts → request-list.d.ts} +33 -173
  68. package/storages/{request_list.js → request-list.js} +276 -255
  69. package/storages/request-loader.d.ts +127 -0
  70. package/storages/request-loader.js +36 -0
  71. package/storages/request-manager-tandem.d.ts +82 -0
  72. package/storages/request-manager-tandem.js +200 -0
  73. package/storages/request-manager.d.ts +119 -0
  74. package/storages/request-manager.js +1 -0
  75. package/storages/{request_provider.d.ts → request-queue.d.ts} +128 -159
  76. package/storages/request-queue.js +870 -0
  77. package/storages/storage-instance-manager.d.ts +86 -0
  78. package/storages/storage-instance-manager.js +256 -0
  79. package/storages/storage-stats.d.ts +48 -0
  80. package/storages/storage-stats.js +29 -0
  81. package/storages/transaction.d.ts +270 -0
  82. package/storages/transaction.js +296 -0
  83. package/storages/utils.d.ts +59 -11
  84. package/storages/utils.js +76 -16
  85. package/system-info/cpu-info.d.ts +67 -0
  86. package/system-info/cpu-info.js +216 -0
  87. package/system-info/memory-info.d.ts +31 -0
  88. package/system-info/memory-info.js +115 -0
  89. package/system-info/ps-tree.d.ts +17 -0
  90. package/system-info/ps-tree.js +144 -0
  91. package/system-info/runtime.d.ts +14 -0
  92. package/system-info/runtime.js +80 -0
  93. package/typedefs.d.ts +0 -6
  94. package/typedefs.js +0 -1
  95. package/url.d.ts +9 -0
  96. package/url.js +11 -0
  97. package/validators.d.ts +23 -18
  98. package/validators.js +14 -18
  99. package/autoscaling/autoscaled_pool.d.ts +0 -300
  100. package/autoscaling/autoscaled_pool.d.ts.map +0 -1
  101. package/autoscaling/autoscaled_pool.js +0 -533
  102. package/autoscaling/autoscaled_pool.js.map +0 -1
  103. package/autoscaling/index.d.ts +0 -4
  104. package/autoscaling/index.d.ts.map +0 -1
  105. package/autoscaling/index.js +0 -4
  106. package/autoscaling/index.js.map +0 -1
  107. package/autoscaling/snapshotter.d.ts +0 -189
  108. package/autoscaling/snapshotter.d.ts.map +0 -1
  109. package/autoscaling/snapshotter.js +0 -285
  110. package/autoscaling/snapshotter.js.map +0 -1
  111. package/autoscaling/system_status.d.ts +0 -181
  112. package/autoscaling/system_status.d.ts.map +0 -1
  113. package/autoscaling/system_status.js +0 -169
  114. package/autoscaling/system_status.js.map +0 -1
  115. package/configuration.d.ts.map +0 -1
  116. package/configuration.js.map +0 -1
  117. package/cookie_utils.d.ts +0 -43
  118. package/cookie_utils.d.ts.map +0 -1
  119. package/cookie_utils.js +0 -111
  120. package/cookie_utils.js.map +0 -1
  121. package/crawlers/context_pipeline.d.ts +0 -61
  122. package/crawlers/context_pipeline.d.ts.map +0 -1
  123. package/crawlers/context_pipeline.js +0 -99
  124. package/crawlers/context_pipeline.js.map +0 -1
  125. package/crawlers/crawler_commons.d.ts +0 -191
  126. package/crawlers/crawler_commons.d.ts.map +0 -1
  127. package/crawlers/crawler_commons.js +0 -108
  128. package/crawlers/crawler_commons.js.map +0 -1
  129. package/crawlers/crawler_utils.d.ts +0 -10
  130. package/crawlers/crawler_utils.d.ts.map +0 -1
  131. package/crawlers/crawler_utils.js +0 -12
  132. package/crawlers/crawler_utils.js.map +0 -1
  133. package/crawlers/error_snapshotter.d.ts +0 -60
  134. package/crawlers/error_snapshotter.d.ts.map +0 -1
  135. package/crawlers/error_snapshotter.js +0 -118
  136. package/crawlers/error_snapshotter.js.map +0 -1
  137. package/crawlers/error_tracker.d.ts +0 -55
  138. package/crawlers/error_tracker.d.ts.map +0 -1
  139. package/crawlers/error_tracker.js +0 -309
  140. package/crawlers/error_tracker.js.map +0 -1
  141. package/crawlers/index.d.ts +0 -7
  142. package/crawlers/index.d.ts.map +0 -1
  143. package/crawlers/index.js +0 -7
  144. package/crawlers/index.js.map +0 -1
  145. package/crawlers/internals/types.d.ts +0 -8
  146. package/crawlers/internals/types.d.ts.map +0 -1
  147. package/crawlers/internals/types.js +0 -2
  148. package/crawlers/internals/types.js.map +0 -1
  149. package/crawlers/statistics.d.ts +0 -213
  150. package/crawlers/statistics.d.ts.map +0 -1
  151. package/crawlers/statistics.js +0 -349
  152. package/crawlers/statistics.js.map +0 -1
  153. package/enqueue_links/enqueue_links.d.ts +0 -249
  154. package/enqueue_links/enqueue_links.d.ts.map +0 -1
  155. package/enqueue_links/enqueue_links.js +0 -269
  156. package/enqueue_links/enqueue_links.js.map +0 -1
  157. package/enqueue_links/index.d.ts +0 -3
  158. package/enqueue_links/index.d.ts.map +0 -1
  159. package/enqueue_links/index.js +0 -3
  160. package/enqueue_links/index.js.map +0 -1
  161. package/enqueue_links/shared.d.ts +0 -73
  162. package/enqueue_links/shared.d.ts.map +0 -1
  163. package/enqueue_links/shared.js +0 -212
  164. package/enqueue_links/shared.js.map +0 -1
  165. package/errors.d.ts.map +0 -1
  166. package/errors.js.map +0 -1
  167. package/events/event_manager.d.ts +0 -52
  168. package/events/event_manager.d.ts.map +0 -1
  169. package/events/event_manager.js.map +0 -1
  170. package/events/index.d.ts.map +0 -1
  171. package/events/index.js.map +0 -1
  172. package/events/local_event_manager.d.ts +0 -27
  173. package/events/local_event_manager.d.ts.map +0 -1
  174. package/events/local_event_manager.js +0 -75
  175. package/events/local_event_manager.js.map +0 -1
  176. package/index.d.ts.map +0 -1
  177. package/index.js.map +0 -1
  178. package/log.d.ts.map +0 -1
  179. package/log.js.map +0 -1
  180. package/proxy_configuration.d.ts +0 -169
  181. package/proxy_configuration.d.ts.map +0 -1
  182. package/proxy_configuration.js +0 -247
  183. package/proxy_configuration.js.map +0 -1
  184. package/recoverable_state.d.ts +0 -123
  185. package/recoverable_state.d.ts.map +0 -1
  186. package/recoverable_state.js +0 -140
  187. package/recoverable_state.js.map +0 -1
  188. package/request.d.ts.map +0 -1
  189. package/request.js.map +0 -1
  190. package/router.d.ts +0 -134
  191. package/router.d.ts.map +0 -1
  192. package/router.js +0 -164
  193. package/router.js.map +0 -1
  194. package/serialization.d.ts.map +0 -1
  195. package/serialization.js.map +0 -1
  196. package/session_pool/consts.d.ts +0 -4
  197. package/session_pool/consts.d.ts.map +0 -1
  198. package/session_pool/consts.js +0 -4
  199. package/session_pool/consts.js.map +0 -1
  200. package/session_pool/errors.d.ts +0 -8
  201. package/session_pool/errors.d.ts.map +0 -1
  202. package/session_pool/errors.js +0 -12
  203. package/session_pool/errors.js.map +0 -1
  204. package/session_pool/events.d.ts +0 -3
  205. package/session_pool/events.d.ts.map +0 -1
  206. package/session_pool/events.js +0 -3
  207. package/session_pool/events.js.map +0 -1
  208. package/session_pool/index.d.ts +0 -6
  209. package/session_pool/index.d.ts.map +0 -1
  210. package/session_pool/index.js +0 -6
  211. package/session_pool/index.js.map +0 -1
  212. package/session_pool/session.d.ts +0 -186
  213. package/session_pool/session.d.ts.map +0 -1
  214. package/session_pool/session.js +0 -278
  215. package/session_pool/session.js.map +0 -1
  216. package/session_pool/session_pool.d.ts +0 -258
  217. package/session_pool/session_pool.d.ts.map +0 -1
  218. package/session_pool/session_pool.js +0 -387
  219. package/session_pool/session_pool.js.map +0 -1
  220. package/storages/access_checking.d.ts +0 -13
  221. package/storages/access_checking.d.ts.map +0 -1
  222. package/storages/access_checking.js +0 -14
  223. package/storages/access_checking.js.map +0 -1
  224. package/storages/dataset.d.ts.map +0 -1
  225. package/storages/dataset.js.map +0 -1
  226. package/storages/index.d.ts.map +0 -1
  227. package/storages/index.js.map +0 -1
  228. package/storages/key_value_store.d.ts.map +0 -1
  229. package/storages/key_value_store.js +0 -512
  230. package/storages/key_value_store.js.map +0 -1
  231. package/storages/request_list.d.ts.map +0 -1
  232. package/storages/request_list.js.map +0 -1
  233. package/storages/request_list_adapter.d.ts +0 -58
  234. package/storages/request_list_adapter.d.ts.map +0 -1
  235. package/storages/request_list_adapter.js +0 -81
  236. package/storages/request_list_adapter.js.map +0 -1
  237. package/storages/request_manager_tandem.d.ts +0 -68
  238. package/storages/request_manager_tandem.d.ts.map +0 -1
  239. package/storages/request_manager_tandem.js +0 -124
  240. package/storages/request_manager_tandem.js.map +0 -1
  241. package/storages/request_provider.d.ts.map +0 -1
  242. package/storages/request_provider.js +0 -591
  243. package/storages/request_provider.js.map +0 -1
  244. package/storages/request_queue.d.ts +0 -128
  245. package/storages/request_queue.d.ts.map +0 -1
  246. package/storages/request_queue.js +0 -320
  247. package/storages/request_queue.js.map +0 -1
  248. package/storages/request_queue_v2.d.ts +0 -87
  249. package/storages/request_queue_v2.d.ts.map +0 -1
  250. package/storages/request_queue_v2.js +0 -438
  251. package/storages/request_queue_v2.js.map +0 -1
  252. package/storages/sitemap_request_list.d.ts +0 -250
  253. package/storages/sitemap_request_list.d.ts.map +0 -1
  254. package/storages/sitemap_request_list.js +0 -431
  255. package/storages/sitemap_request_list.js.map +0 -1
  256. package/storages/storage_manager.d.ts +0 -62
  257. package/storages/storage_manager.d.ts.map +0 -1
  258. package/storages/storage_manager.js +0 -105
  259. package/storages/storage_manager.js.map +0 -1
  260. package/storages/utils.d.ts.map +0 -1
  261. package/storages/utils.js.map +0 -1
  262. package/typedefs.d.ts.map +0 -1
  263. package/typedefs.js.map +0 -1
  264. package/validators.d.ts.map +0 -1
  265. package/validators.js.map +0 -1
package/README.md CHANGED
@@ -1,23 +1,23 @@
1
1
  <h1 align="center">
2
2
  <a href="https://crawlee.dev">
3
3
  <picture>
4
- <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true">
5
- <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500">
4
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true" />
5
+ <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500" />
6
6
  </picture>
7
7
  </a>
8
- <br>
8
+ <br />
9
9
  <small>A web scraping and browser automation library</small>
10
10
  </h1>
11
11
 
12
- <p align=center>
13
- <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
12
+ <p align="center">
13
+ <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" width="250" height="55"/></a>
14
14
  </p>
15
15
 
16
- <p align=center>
17
- <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" style="max-width: 100%;"></a>
18
- <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" style="max-width: 100%;"></a>
19
- <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" style="max-width: 100%;"></a>
20
- <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" style="max-width: 100%;"></a>
16
+ <p align="center">
17
+ <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" /></a>
18
+ <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" /></a>
19
+ <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" /></a>
20
+ <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" /></a>
21
21
  </p>
22
22
 
23
23
  Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
@@ -34,7 +34,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
34
34
 
35
35
  We recommend visiting the [Introduction tutorial](https://crawlee.dev/js/docs/introduction) in Crawlee documentation for more information.
36
36
 
37
- > Crawlee requires **Node.js 16 or higher**.
37
+ > Crawlee requires **Node.js 22.13 or higher**.
38
38
 
39
39
  ### With Crawlee CLI
40
40
 
@@ -89,7 +89,7 @@ By default, Crawlee stores data to `./storage` in the current working directory.
89
89
  We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
90
90
 
91
91
  ```bash
92
- npm install crawlee@3.12.3-beta.13
92
+ npm install crawlee@next
93
93
  ```
94
94
 
95
95
  If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
@@ -98,9 +98,9 @@ If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need
98
98
  {
99
99
  "overrides": {
100
100
  "apify": {
101
- "@crawlee/core": "3.12.3-beta.13",
102
- "@crawlee/types": "3.12.3-beta.13",
103
- "@crawlee/utils": "3.12.3-beta.13"
101
+ "@crawlee/core": "$crawlee",
102
+ "@crawlee/types": "$crawlee",
103
+ "@crawlee/utils": "$crawlee"
104
104
  }
105
105
  }
106
106
  }
@@ -0,0 +1,17 @@
1
+ /**
2
+ * Checks if the given value is a Node.js Stream or a Web API ReadableStream.
3
+ * @ignore
4
+ */
5
+ export declare function isStream(value: unknown): value is NodeJS.ReadableStream | ReadableStream;
6
+ /**
7
+ * Checks if the given value is a Node.js Buffer, ArrayBuffer, or TypedArray.
8
+ * @ignore
9
+ */
10
+ export declare function isBuffer(value: unknown): value is Buffer | ArrayBuffer | ArrayBufferView;
11
+ /**
12
+ * Converts a byte-like value (Buffer, ArrayBuffer, or any typed-array / DataView) into a Buffer over
13
+ * the exact same bytes, honoring `byteOffset` / `byteLength` for views. Existing Buffers are returned
14
+ * as-is. Used by storage backends, which persist raw bytes regardless of the input's concrete shape.
15
+ * @ignore
16
+ */
17
+ export declare function toBuffer(value: Buffer | ArrayBuffer | ArrayBufferView): Buffer;
package/byte-utils.js ADDED
@@ -0,0 +1,42 @@
1
+ /**
2
+ * Checks if the given value is a Node.js Stream or a Web API ReadableStream.
3
+ * @ignore
4
+ */
5
+ export function isStream(value) {
6
+ if (typeof value !== 'object' || value === null) {
7
+ return false;
8
+ }
9
+ // A Node.js Readable is both pipeable and async-iterable; a Web ReadableStream exposes pipeTo.
10
+ // Requiring async-iterability for the `pipe` branch rejects plain `{ pipe }` ducks that would
11
+ // otherwise blow up later in the storage backends' drain loop with a cryptic TypeError.
12
+ const isNodeStream = typeof value.pipe === 'function' && typeof value[Symbol.asyncIterator] === 'function';
13
+ const isWebStream = typeof value.pipeTo === 'function';
14
+ return isNodeStream || isWebStream;
15
+ }
16
+ /**
17
+ * Checks if the given value is a Node.js Buffer, ArrayBuffer, or TypedArray.
18
+ * @ignore
19
+ */
20
+ export function isBuffer(value) {
21
+ return (value != null &&
22
+ typeof value === 'object' &&
23
+ (Buffer.isBuffer(value) ||
24
+ value instanceof ArrayBuffer ||
25
+ ArrayBuffer.isView(value) ||
26
+ value.constructor?.name === 'Buffer'));
27
+ }
28
+ /**
29
+ * Converts a byte-like value (Buffer, ArrayBuffer, or any typed-array / DataView) into a Buffer over
30
+ * the exact same bytes, honoring `byteOffset` / `byteLength` for views. Existing Buffers are returned
31
+ * as-is. Used by storage backends, which persist raw bytes regardless of the input's concrete shape.
32
+ * @ignore
33
+ */
34
+ export function toBuffer(value) {
35
+ if (Buffer.isBuffer(value)) {
36
+ return value;
37
+ }
38
+ if (value instanceof ArrayBuffer) {
39
+ return Buffer.from(value);
40
+ }
41
+ return Buffer.from(value.buffer, value.byteOffset, value.byteLength);
42
+ }
@@ -1,151 +1,60 @@
1
- import { AsyncLocalStorage } from 'node:async_hooks';
2
- import type { MemoryStorageOptions } from '@crawlee/memory-storage';
3
- import { MemoryStorage } from '@crawlee/memory-storage';
4
- import type { Dictionary, StorageClient } from '@crawlee/types';
5
- import { LogLevel } from '@apify/log';
6
- import { type EventManager } from './events/event_manager.js';
7
- import type { StorageManager } from './storages/storage_manager.js';
8
- import { type Constructor } from './typedefs.js';
9
- export interface ConfigurationOptions {
10
- /**
11
- * Defines storage client to be used.
12
- * @default {@link MemoryStorage}
13
- */
14
- storageClient?: StorageClient;
15
- /**
16
- * Defines the Event Manager to be used.
17
- * @default {@link EventManager}
18
- */
19
- eventManager?: EventManager;
20
- /**
21
- * Could be used to adjust the storage client behavior
22
- * e.g. {@link MemoryStorageOptions} could be used to adjust the {@link MemoryStorage} behavior.
23
- */
24
- storageClientOptions?: Dictionary;
25
- /**
26
- * Default dataset id.
27
- *
28
- * Alternative to `CRAWLEE_DEFAULT_DATASET_ID` environment variable.
29
- * @default 'default'
30
- */
31
- defaultDatasetId?: string;
32
- /**
33
- * Defines whether to purge the default storage folders before starting the crawler run.
34
- *
35
- * Alternative to `CRAWLEE_PURGE_ON_START` environment variable.
36
- * @default true
37
- */
38
- purgeOnStart?: boolean;
39
- /**
40
- * Default key-value store id.
41
- *
42
- * Alternative to `CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID` environment variable.
43
- * @default 'default'
44
- */
45
- defaultKeyValueStoreId?: string;
46
- /**
47
- * Default request queue id.
48
- *
49
- * Alternative to `CRAWLEE_DEFAULT_REQUEST_QUEUE_ID` environment variable.
50
- * @default 'default'
51
- */
52
- defaultRequestQueueId?: string;
53
- /**
54
- * Sets the ratio, defining the maximum CPU usage.
55
- * When the CPU usage is higher than the provided ratio, the CPU is considered overloaded.
56
- * @default 0.95
57
- */
58
- maxUsedCpuRatio?: number;
59
- /**
60
- * Sets the ratio, defining the amount of system memory that could be used by the {@link AutoscaledPool}.
61
- * When the memory usage is more than the provided ratio, the memory is considered overloaded.
62
- *
63
- * Alternative to `CRAWLEE_AVAILABLE_MEMORY_RATIO` environment variable.
64
- * @default 0.25
65
- */
66
- availableMemoryRatio?: number;
67
- /**
68
- * Sets the amount of system memory in megabytes to be used by the {@link AutoscaledPool}.
69
- * By default, the maximum memory is set to one quarter of total system memory.
70
- *
71
- * Alternative to `CRAWLEE_MEMORY_MBYTES` environment variable.
72
- */
73
- memoryMbytes?: number;
74
- /**
75
- * Defines the interval of emitting the `persistState` event.
76
- *
77
- * Alternative to `CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS` environment variable.
78
- * @default 60_000
79
- */
80
- persistStateIntervalMillis?: number;
81
- /**
82
- Defines the interval of emitting the `systemInfo` event.
83
- @default 1_000
84
- */
85
- systemInfoIntervalMillis?: number;
86
- /**
87
- * Defines the default input key, i.e. the key that is used to get the crawler input value
88
- * from the default {@link KeyValueStore} associated with the current crawler run.
89
- *
90
- * Alternative to `CRAWLEE_INPUT_KEY` environment variable.
91
- * @default 'INPUT'
92
- */
93
- inputKey?: string;
94
- /**
95
- * Defines whether web browsers launched by Crawlee will run in the headless mode.
96
- *
97
- * Alternative to `CRAWLEE_HEADLESS` environment variable.
98
- * @default true
99
- */
100
- headless?: boolean;
101
- /**
102
- * Defines whether to run X virtual framebuffer on the web browsers launched by Crawlee.
103
- *
104
- * Alternative to `CRAWLEE_XVFB` environment variable.
105
- * @default false
106
- */
107
- xvfb?: boolean;
108
- /**
109
- * Defines a path to Chrome executable.
110
- *
111
- * Alternative to `CRAWLEE_CHROME_EXECUTABLE_PATH` environment variable.
112
- */
113
- chromeExecutablePath?: string;
114
- /**
115
- * Defines a path to default browser executable.
116
- *
117
- * Alternative to `CRAWLEE_DEFAULT_BROWSER_PATH` environment variable.
118
- */
119
- defaultBrowserPath?: string;
120
- /**
121
- * Defines whether to disable browser sandbox by adding `--no-sandbox` flag to `launchOptions`.
122
- *
123
- * Alternative to `CRAWLEE_DISABLE_BROWSER_SANDBOX` environment variable.
124
- */
125
- disableBrowserSandbox?: boolean;
126
- /**
127
- * Sets the log level to the given value.
128
- *
129
- * Alternative to `CRAWLEE_LOG_LEVEL` environment variable.
130
- * @default 'INFO'
131
- */
132
- logLevel?: LogLevel | LogLevel[keyof LogLevel];
133
- /**
134
- * Defines whether the storage client used should persist the data it stores.
135
- *
136
- * Alternative to `CRAWLEE_PERSIST_STORAGE` environment variable.
137
- */
138
- persistStorage?: boolean;
139
- /**
140
- * Used in place of `isContainerized()` when collecting system metrics.
141
- *
142
- * Alternative to `CRAWLEE_CONTAINERIZED` environment variable.
143
- */
144
- containerized?: boolean;
1
+ import { z } from 'zod';
2
+ import { LogLevel } from './log.js';
3
+ export interface ConfigField<T extends z.ZodType = z.ZodType> {
4
+ schema: T;
5
+ envVar?: string | string[];
6
+ }
7
+ export declare function field<T extends z.ZodType>(schema: T, envVar?: string | string[]): ConfigField<T>;
8
+ /** Zod preprocessor treating `'0'` and `'false'` as falsy. */
9
+ export declare const coerceBoolean: z.ZodPreprocess<z.ZodBoolean, unknown>;
10
+ export declare const coerceNumber: z.ZodPreprocess<z.ZodNumber, unknown>;
11
+ export declare const crawleeConfigFields: {
12
+ /** @default true */
13
+ purgeOnStart: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
14
+ /** @default 0.95 */
15
+ maxUsedCpuRatio: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber, unknown>>>;
16
+ /** @default 0.25 */
17
+ availableMemoryRatio: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber, unknown>>>;
18
+ memoryMbytes: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodNumber, unknown>>>;
19
+ /** @default 60_000 */
20
+ persistStateIntervalMillis: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber, unknown>>>;
21
+ /**
22
+ * Internal safety-net timeout for a single request, in milliseconds. When unset the crawler derives it from
23
+ * the request handler timeout (twice it, and never below 5 minutes).
24
+ */
25
+ internalTimeoutMillis: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodNumber, unknown>>>;
26
+ /** @default 1_000 */
27
+ systemInfoIntervalMillis: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber, unknown>>>;
28
+ /** @default true */
29
+ headless: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
30
+ /** @default false */
31
+ xvfb: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
32
+ chromeExecutablePath: ConfigField<z.ZodOptional<z.ZodString>>;
33
+ defaultBrowserPath: ConfigField<z.ZodOptional<z.ZodString>>;
34
+ /** @default false */
35
+ disableBrowserSandbox: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
36
+ logLevel: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodEnum<typeof LogLevel>, unknown>>>;
37
+ /** @default true */
38
+ persistStorage: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
39
+ /** @default './storage' */
40
+ storageDir: ConfigField<z.ZodDefault<z.ZodString>>;
41
+ containerized: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
42
+ };
43
+ export type FieldsInput<F extends Record<string, ConfigField>> = {
44
+ [K in keyof F]?: z.output<F[K]['schema']>;
45
+ };
46
+ export type FieldsOutput<F extends Record<string, ConfigField>> = {
47
+ [K in keyof F]: z.output<F[K]['schema']>;
48
+ };
49
+ export type ConfigurationInput = FieldsInput<typeof crawleeConfigFields>;
50
+ export type ResolvedConfigValues = FieldsOutput<typeof crawleeConfigFields>;
51
+ /** @deprecated Use {@link ConfigurationInput} instead. */
52
+ export type ConfigurationOptions = ConfigurationInput;
53
+ export interface Configuration extends ResolvedConfigValues {
145
54
  }
146
55
  /**
147
56
  * `Configuration` is a value object holding Crawlee configuration. By default, there is a
148
- * global singleton instance of this class available via `Configuration.getGlobalConfig()`.
57
+ * global singleton instance of this class available via `Configuration.getGlobalConfiguration()`.
149
58
  * Places that depend on a configurable behaviour depend on this class, as they have the global
150
59
  * instance as the default value.
151
60
  *
@@ -154,14 +63,10 @@ export interface ConfigurationOptions {
154
63
  * import { BasicCrawler, Configuration } from 'crawlee';
155
64
  *
156
65
  * // Get the global configuration
157
- * const config = Configuration.getGlobalConfig();
158
- * // Set the 'persistStateIntervalMillis' option
159
- * // of global configuration to 10 seconds
160
- * config.set('persistStateIntervalMillis', 10_000);
161
- *
162
- * // No need to pass the configuration to the crawler,
163
- * // as it's using the global configuration by default
164
- * const crawler = new BasicCrawler();
66
+ * const config = Configuration.getGlobalConfiguration();
67
+ * // Access configuration values directly as properties
68
+ * console.log(config.headless);
69
+ * console.log(config.persistStateIntervalMillis);
165
70
  * ```
166
71
  *
167
72
  * *Using custom configuration:*
@@ -171,15 +76,14 @@ export interface ConfigurationOptions {
171
76
  * // Create a new configuration
172
77
  * const config = new Configuration({ persistStateIntervalMillis: 30_000 });
173
78
  * // Pass the configuration to the crawler
174
- * const crawler = new BasicCrawler({ ... }, config);
79
+ * const crawler = new BasicCrawler({ configuration: config });
175
80
  * ```
176
81
  *
177
- * The configuration provided via environment variables always takes precedence. We can also
178
- * define the `crawlee.json` file in the project root directory which will serve as a baseline,
179
- * so the options provided in constructor will override those. In other words, the precedence is:
82
+ * Configuration is immutable — values are set via the constructor and cannot be changed afterwards.
83
+ * The priority order for resolving values is (highest to lowest):
180
84
  *
181
85
  * ```text
182
- * crawlee.json < constructor options < environment variables
86
+ * constructor options > environment variables > crawlee.json > schema defaults
183
87
  * ```
184
88
  *
185
89
  * ## Supported Configuration Options
@@ -189,102 +93,40 @@ export interface ConfigurationOptions {
189
93
  * `memoryMbytes` | `CRAWLEE_MEMORY_MBYTES` | -
190
94
  * `logLevel` | `CRAWLEE_LOG_LEVEL` | -
191
95
  * `headless` | `CRAWLEE_HEADLESS` | `true`
192
- * `defaultDatasetId` | `CRAWLEE_DEFAULT_DATASET_ID` | `'default'`
193
- * `defaultKeyValueStoreId` | `CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID` | `'default'`
194
- * `defaultRequestQueueId` | `CRAWLEE_DEFAULT_REQUEST_QUEUE_ID` | `'default'`
195
96
  * `persistStateIntervalMillis` | `CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS` | `60_000`
97
+ * `internalTimeoutMillis` | `CRAWLEE_INTERNAL_TIMEOUT` | -
196
98
  * `purgeOnStart` | `CRAWLEE_PURGE_ON_START` | `true`
197
99
  * `persistStorage` | `CRAWLEE_PERSIST_STORAGE` | `true`
100
+ * `storageDir` | `CRAWLEE_STORAGE_DIR` | `'./storage'`
198
101
  *
199
102
  * ## Advanced Configuration Options
200
103
  *
201
104
  * Key | Environment Variable | Default Value
202
105
  * ---|---|---
203
- * `inputKey` | `CRAWLEE_INPUT_KEY` | `'INPUT'`
204
- * `xvfb` | `CRAWLEE_XVFB` | -
106
+ * `xvfb` | `CRAWLEE_XVFB` | `false`
205
107
  * `chromeExecutablePath` | `CRAWLEE_CHROME_EXECUTABLE_PATH` | -
206
108
  * `defaultBrowserPath` | `CRAWLEE_DEFAULT_BROWSER_PATH` | -
207
109
  * `disableBrowserSandbox` | `CRAWLEE_DISABLE_BROWSER_SANDBOX` | -
208
110
  * `availableMemoryRatio` | `CRAWLEE_AVAILABLE_MEMORY_RATIO` | `0.25`
209
- * `containerized | `CRAWLEE_CONTAINERIZED | -
111
+ * `containerized` | `CRAWLEE_CONTAINERIZED` | -
210
112
  */
211
113
  export declare class Configuration {
114
+ #private;
212
115
  /**
213
- * Maps environment variables to config keys (e.g. `CRAWLEE_MEMORY_MBYTES` to `memoryMbytes`)
116
+ * Field definitions for this configuration class.
117
+ * Subclasses override this to register additional fields.
214
118
  */
215
- protected static ENV_MAP: Dictionary;
216
- protected static BOOLEAN_VARS: string[];
217
- protected static INTEGER_VARS: string[];
218
- protected static COMMA_SEPARATED_LIST_VARS: string[];
219
- protected static DEFAULTS: Dictionary;
119
+ protected static fields: Record<string, ConfigField>;
220
120
  /**
221
- * Provides access to the current-instance-scoped Configuration without passing it around in parameters.
222
- * @internal
121
+ * Creates new `Configuration` instance with provided options.
122
+ * Constructor options take precedence over environment variables, which take precedence
123
+ * over crawlee.json values, which take precedence over schema defaults.
223
124
  */
224
- static storage: AsyncLocalStorage<Configuration>;
225
- protected options: Map<keyof ConfigurationOptions, ConfigurationOptions[keyof ConfigurationOptions]>;
226
- protected services: Map<string, unknown>;
227
- /** @internal */
228
- static globalConfig?: Configuration;
229
- // @ts-ignore optional peer dependency or compatibility with es2022
230
- readonly storageManagers: Map<Constructor, StorageManager<import("./storages/storage_manager.js").IStorage>>;
231
- /**
232
- * Creates new `Configuration` instance with provided options. Env vars will have precedence over those.
233
- */
234
- constructor(options?: ConfigurationOptions);
235
- /**
236
- * Returns configured value. First checks the environment variables, then provided configuration,
237
- * fallbacks to the `defaultValue` argument if provided, otherwise uses the default value as described
238
- * in the above section.
239
- */
240
- get<T extends keyof ConfigurationOptions, U extends ConfigurationOptions[T]>(key: T, defaultValue?: U): U;
241
- protected _castEnvValue(key: keyof ConfigurationOptions, value: number | string | boolean): string | number | boolean | string[];
242
- /**
243
- * Sets value for given option. Only affects this `Configuration` instance, the value will not be propagated down to the env var.
244
- * To reset a value, we can omit the `value` argument or pass `undefined` there.
245
- */
246
- set(key: keyof ConfigurationOptions, value?: any): void;
247
- /**
248
- * Sets value for given option. Only affects the global `Configuration` instance, the value will not be propagated down to the env var.
249
- * To reset a value, we can omit the `value` argument or pass `undefined` there.
250
- */
251
- static set(key: keyof ConfigurationOptions, value?: any): void;
252
- /**
253
- * Returns cached instance of {@link StorageClient} using options as defined in the environment variables or in
254
- * this {@link Configuration} instance. Only first call of this method will create the client, following calls will
255
- * return the same client instance.
256
- *
257
- * Caching works based on the `storageClientOptions`, so calling this method with different options will return
258
- * multiple instances, one for each variant of the options.
259
- * @internal
260
- */
261
- getStorageClient(): StorageClient;
262
- getEventManager(): EventManager;
263
- /**
264
- * Creates an instance of MemoryStorage using options as defined in the environment variables or in this `Configuration` instance.
265
- * @internal
266
- */
267
- createMemoryStorage(options?: MemoryStorageOptions): MemoryStorage;
268
- useStorageClient(client: StorageClient): void;
269
- static useStorageClient(client: StorageClient): void;
270
- useEventManager(events: EventManager): void;
125
+ constructor(options?: ConfigurationInput);
271
126
  /**
272
127
  * Returns the global configuration instance. It will respect the environment variables.
128
+ *
129
+ * Delegates to the global ServiceLocator, making it the single source of truth for service management.
273
130
  */
274
- static getGlobalConfig(): Configuration;
275
- /**
276
- * Gets default {@link StorageClient} instance.
277
- */
278
- static getStorageClient(): StorageClient;
279
- /**
280
- * Gets default {@link EventManager} instance.
281
- */
282
- static getEventManager(): EventManager;
283
- /**
284
- * Resets global configuration instance. The default instance holds configuration based on env vars,
285
- * if we want to change them, we need to first reset the global state. Used mainly for testing purposes.
286
- */
287
- static resetGlobalState(): void;
288
- protected buildOptions(options: ConfigurationOptions): void;
131
+ static getGlobalConfiguration(): Configuration;
289
132
  }
290
- //# sourceMappingURL=configuration.d.ts.map