@crawlee/core 4.0.0-beta.27 → 4.0.0-beta.271

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (265) hide show
  1. package/README.md +15 -15
  2. package/byte-utils.d.ts +17 -0
  3. package/byte-utils.js +42 -0
  4. package/configuration.d.ts +77 -235
  5. package/configuration.js +159 -227
  6. package/debug.d.ts +36 -0
  7. package/debug.js +70 -0
  8. package/errors.d.ts +45 -18
  9. package/errors.js +58 -27
  10. package/events/event-manager.d.ts +78 -0
  11. package/events/{event_manager.js → event-manager.js} +12 -13
  12. package/events/index.d.ts +3 -3
  13. package/events/index.js +2 -3
  14. package/events/local-event-manager.d.ts +32 -0
  15. package/events/local-event-manager.js +101 -0
  16. package/events/system-info.d.ts +38 -0
  17. package/events/system-info.js +1 -0
  18. package/index.d.ts +9 -12
  19. package/index.js +10 -11
  20. package/internal.d.ts +8 -0
  21. package/internal.js +9 -0
  22. package/iterables.d.ts +79 -0
  23. package/iterables.js +134 -0
  24. package/log.d.ts +77 -1
  25. package/log.js +134 -1
  26. package/memory-storage/consts.d.ts +4 -0
  27. package/memory-storage/consts.js +4 -0
  28. package/memory-storage/index.d.ts +1 -0
  29. package/memory-storage/index.js +1 -0
  30. package/memory-storage/memory-storage.d.ts +43 -0
  31. package/memory-storage/memory-storage.js +158 -0
  32. package/memory-storage/resource-clients/common/base-client.d.ts +4 -0
  33. package/memory-storage/resource-clients/common/base-client.js +6 -0
  34. package/memory-storage/resource-clients/dataset.d.ts +35 -0
  35. package/memory-storage/resource-clients/dataset.js +108 -0
  36. package/memory-storage/resource-clients/key-value-store.d.ts +54 -0
  37. package/memory-storage/resource-clients/key-value-store.js +179 -0
  38. package/memory-storage/resource-clients/request-queue.d.ts +55 -0
  39. package/memory-storage/resource-clients/request-queue.js +413 -0
  40. package/memory-storage/utils.d.ts +16 -0
  41. package/memory-storage/utils.js +41 -0
  42. package/owned-or-injected.d.ts +58 -0
  43. package/owned-or-injected.js +98 -0
  44. package/package.json +23 -23
  45. package/proxy-configuration.d.ts +118 -0
  46. package/proxy-configuration.js +141 -0
  47. package/recoverable-state.d.ts +177 -0
  48. package/recoverable-state.js +268 -0
  49. package/request.d.ts +48 -101
  50. package/request.js +139 -249
  51. package/serialization.d.ts +0 -1
  52. package/serialization.js +18 -18
  53. package/service-locator.d.ts +168 -0
  54. package/service-locator.js +255 -0
  55. package/storages/batched-adds.d.ts +39 -0
  56. package/storages/batched-adds.js +81 -0
  57. package/storages/dataset.d.ts +105 -58
  58. package/storages/dataset.js +292 -150
  59. package/storages/index.d.ts +11 -12
  60. package/storages/index.js +7 -12
  61. package/storages/key-value-store-codec.d.ts +32 -0
  62. package/storages/key-value-store-codec.js +108 -0
  63. package/storages/{key_value_store.d.ts → key-value-store.d.ts} +169 -72
  64. package/storages/key-value-store.js +767 -0
  65. package/storages/request-dedup-cache.d.ts +20 -0
  66. package/storages/request-dedup-cache.js +48 -0
  67. package/storages/{request_list.d.ts → request-list.d.ts} +33 -173
  68. package/storages/{request_list.js → request-list.js} +276 -255
  69. package/storages/request-loader.d.ts +127 -0
  70. package/storages/request-loader.js +34 -0
  71. package/storages/request-manager-tandem.d.ts +82 -0
  72. package/storages/request-manager-tandem.js +200 -0
  73. package/storages/request-manager.d.ts +119 -0
  74. package/storages/request-manager.js +1 -0
  75. package/storages/{request_provider.d.ts → request-queue.d.ts} +127 -159
  76. package/storages/request-queue.js +871 -0
  77. package/storages/storage-instance-manager.d.ts +86 -0
  78. package/storages/storage-instance-manager.js +256 -0
  79. package/storages/storage-stats.d.ts +48 -0
  80. package/storages/storage-stats.js +29 -0
  81. package/storages/transaction.d.ts +270 -0
  82. package/storages/transaction.js +296 -0
  83. package/storages/utils.d.ts +59 -11
  84. package/storages/utils.js +76 -16
  85. package/system-info/cpu-info.d.ts +67 -0
  86. package/system-info/cpu-info.js +216 -0
  87. package/system-info/memory-info.d.ts +31 -0
  88. package/system-info/memory-info.js +115 -0
  89. package/system-info/ps-tree.d.ts +17 -0
  90. package/system-info/ps-tree.js +144 -0
  91. package/system-info/runtime.d.ts +14 -0
  92. package/system-info/runtime.js +80 -0
  93. package/typedefs.d.ts +0 -6
  94. package/typedefs.js +0 -1
  95. package/url.d.ts +9 -0
  96. package/url.js +11 -0
  97. package/validators.d.ts +23 -18
  98. package/validators.js +14 -18
  99. package/autoscaling/autoscaled_pool.d.ts +0 -300
  100. package/autoscaling/autoscaled_pool.d.ts.map +0 -1
  101. package/autoscaling/autoscaled_pool.js +0 -533
  102. package/autoscaling/autoscaled_pool.js.map +0 -1
  103. package/autoscaling/index.d.ts +0 -4
  104. package/autoscaling/index.d.ts.map +0 -1
  105. package/autoscaling/index.js +0 -4
  106. package/autoscaling/index.js.map +0 -1
  107. package/autoscaling/snapshotter.d.ts +0 -189
  108. package/autoscaling/snapshotter.d.ts.map +0 -1
  109. package/autoscaling/snapshotter.js +0 -285
  110. package/autoscaling/snapshotter.js.map +0 -1
  111. package/autoscaling/system_status.d.ts +0 -181
  112. package/autoscaling/system_status.d.ts.map +0 -1
  113. package/autoscaling/system_status.js +0 -169
  114. package/autoscaling/system_status.js.map +0 -1
  115. package/configuration.d.ts.map +0 -1
  116. package/configuration.js.map +0 -1
  117. package/cookie_utils.d.ts +0 -43
  118. package/cookie_utils.d.ts.map +0 -1
  119. package/cookie_utils.js +0 -111
  120. package/cookie_utils.js.map +0 -1
  121. package/crawlers/context_pipeline.d.ts +0 -61
  122. package/crawlers/context_pipeline.d.ts.map +0 -1
  123. package/crawlers/context_pipeline.js +0 -99
  124. package/crawlers/context_pipeline.js.map +0 -1
  125. package/crawlers/crawler_commons.d.ts +0 -191
  126. package/crawlers/crawler_commons.d.ts.map +0 -1
  127. package/crawlers/crawler_commons.js +0 -108
  128. package/crawlers/crawler_commons.js.map +0 -1
  129. package/crawlers/crawler_utils.d.ts +0 -10
  130. package/crawlers/crawler_utils.d.ts.map +0 -1
  131. package/crawlers/crawler_utils.js +0 -12
  132. package/crawlers/crawler_utils.js.map +0 -1
  133. package/crawlers/error_snapshotter.d.ts +0 -60
  134. package/crawlers/error_snapshotter.d.ts.map +0 -1
  135. package/crawlers/error_snapshotter.js +0 -118
  136. package/crawlers/error_snapshotter.js.map +0 -1
  137. package/crawlers/error_tracker.d.ts +0 -55
  138. package/crawlers/error_tracker.d.ts.map +0 -1
  139. package/crawlers/error_tracker.js +0 -309
  140. package/crawlers/error_tracker.js.map +0 -1
  141. package/crawlers/index.d.ts +0 -7
  142. package/crawlers/index.d.ts.map +0 -1
  143. package/crawlers/index.js +0 -7
  144. package/crawlers/index.js.map +0 -1
  145. package/crawlers/internals/types.d.ts +0 -8
  146. package/crawlers/internals/types.d.ts.map +0 -1
  147. package/crawlers/internals/types.js +0 -2
  148. package/crawlers/internals/types.js.map +0 -1
  149. package/crawlers/statistics.d.ts +0 -213
  150. package/crawlers/statistics.d.ts.map +0 -1
  151. package/crawlers/statistics.js +0 -349
  152. package/crawlers/statistics.js.map +0 -1
  153. package/enqueue_links/enqueue_links.d.ts +0 -254
  154. package/enqueue_links/enqueue_links.d.ts.map +0 -1
  155. package/enqueue_links/enqueue_links.js +0 -268
  156. package/enqueue_links/enqueue_links.js.map +0 -1
  157. package/enqueue_links/index.d.ts +0 -3
  158. package/enqueue_links/index.d.ts.map +0 -1
  159. package/enqueue_links/index.js +0 -3
  160. package/enqueue_links/index.js.map +0 -1
  161. package/enqueue_links/shared.d.ts +0 -84
  162. package/enqueue_links/shared.d.ts.map +0 -1
  163. package/enqueue_links/shared.js +0 -219
  164. package/enqueue_links/shared.js.map +0 -1
  165. package/errors.d.ts.map +0 -1
  166. package/errors.js.map +0 -1
  167. package/events/event_manager.d.ts +0 -52
  168. package/events/event_manager.d.ts.map +0 -1
  169. package/events/event_manager.js.map +0 -1
  170. package/events/index.d.ts.map +0 -1
  171. package/events/index.js.map +0 -1
  172. package/events/local_event_manager.d.ts +0 -27
  173. package/events/local_event_manager.d.ts.map +0 -1
  174. package/events/local_event_manager.js +0 -75
  175. package/events/local_event_manager.js.map +0 -1
  176. package/index.d.ts.map +0 -1
  177. package/index.js.map +0 -1
  178. package/log.d.ts.map +0 -1
  179. package/log.js.map +0 -1
  180. package/proxy_configuration.d.ts +0 -169
  181. package/proxy_configuration.d.ts.map +0 -1
  182. package/proxy_configuration.js +0 -247
  183. package/proxy_configuration.js.map +0 -1
  184. package/recoverable_state.d.ts +0 -123
  185. package/recoverable_state.d.ts.map +0 -1
  186. package/recoverable_state.js +0 -140
  187. package/recoverable_state.js.map +0 -1
  188. package/request.d.ts.map +0 -1
  189. package/request.js.map +0 -1
  190. package/router.d.ts +0 -134
  191. package/router.d.ts.map +0 -1
  192. package/router.js +0 -164
  193. package/router.js.map +0 -1
  194. package/serialization.d.ts.map +0 -1
  195. package/serialization.js.map +0 -1
  196. package/session_pool/consts.d.ts +0 -4
  197. package/session_pool/consts.d.ts.map +0 -1
  198. package/session_pool/consts.js +0 -4
  199. package/session_pool/consts.js.map +0 -1
  200. package/session_pool/errors.d.ts +0 -8
  201. package/session_pool/errors.d.ts.map +0 -1
  202. package/session_pool/errors.js +0 -12
  203. package/session_pool/errors.js.map +0 -1
  204. package/session_pool/events.d.ts +0 -3
  205. package/session_pool/events.d.ts.map +0 -1
  206. package/session_pool/events.js +0 -3
  207. package/session_pool/events.js.map +0 -1
  208. package/session_pool/index.d.ts +0 -6
  209. package/session_pool/index.d.ts.map +0 -1
  210. package/session_pool/index.js +0 -6
  211. package/session_pool/index.js.map +0 -1
  212. package/session_pool/session.d.ts +0 -186
  213. package/session_pool/session.d.ts.map +0 -1
  214. package/session_pool/session.js +0 -278
  215. package/session_pool/session.js.map +0 -1
  216. package/session_pool/session_pool.d.ts +0 -258
  217. package/session_pool/session_pool.d.ts.map +0 -1
  218. package/session_pool/session_pool.js +0 -387
  219. package/session_pool/session_pool.js.map +0 -1
  220. package/storages/access_checking.d.ts +0 -13
  221. package/storages/access_checking.d.ts.map +0 -1
  222. package/storages/access_checking.js +0 -14
  223. package/storages/access_checking.js.map +0 -1
  224. package/storages/dataset.d.ts.map +0 -1
  225. package/storages/dataset.js.map +0 -1
  226. package/storages/index.d.ts.map +0 -1
  227. package/storages/index.js.map +0 -1
  228. package/storages/key_value_store.d.ts.map +0 -1
  229. package/storages/key_value_store.js +0 -512
  230. package/storages/key_value_store.js.map +0 -1
  231. package/storages/request_list.d.ts.map +0 -1
  232. package/storages/request_list.js.map +0 -1
  233. package/storages/request_list_adapter.d.ts +0 -58
  234. package/storages/request_list_adapter.d.ts.map +0 -1
  235. package/storages/request_list_adapter.js +0 -81
  236. package/storages/request_list_adapter.js.map +0 -1
  237. package/storages/request_manager_tandem.d.ts +0 -68
  238. package/storages/request_manager_tandem.d.ts.map +0 -1
  239. package/storages/request_manager_tandem.js +0 -124
  240. package/storages/request_manager_tandem.js.map +0 -1
  241. package/storages/request_provider.d.ts.map +0 -1
  242. package/storages/request_provider.js +0 -591
  243. package/storages/request_provider.js.map +0 -1
  244. package/storages/request_queue.d.ts +0 -128
  245. package/storages/request_queue.d.ts.map +0 -1
  246. package/storages/request_queue.js +0 -320
  247. package/storages/request_queue.js.map +0 -1
  248. package/storages/request_queue_v2.d.ts +0 -87
  249. package/storages/request_queue_v2.d.ts.map +0 -1
  250. package/storages/request_queue_v2.js +0 -438
  251. package/storages/request_queue_v2.js.map +0 -1
  252. package/storages/sitemap_request_list.d.ts +0 -250
  253. package/storages/sitemap_request_list.d.ts.map +0 -1
  254. package/storages/sitemap_request_list.js +0 -431
  255. package/storages/sitemap_request_list.js.map +0 -1
  256. package/storages/storage_manager.d.ts +0 -62
  257. package/storages/storage_manager.d.ts.map +0 -1
  258. package/storages/storage_manager.js +0 -105
  259. package/storages/storage_manager.js.map +0 -1
  260. package/storages/utils.d.ts.map +0 -1
  261. package/storages/utils.js.map +0 -1
  262. package/typedefs.d.ts.map +0 -1
  263. package/typedefs.js.map +0 -1
  264. package/validators.d.ts.map +0 -1
  265. package/validators.js.map +0 -1
package/configuration.js CHANGED
@@ -1,15 +1,73 @@
1
- import { AsyncLocalStorage } from 'node:async_hooks';
2
- import { EventEmitter } from 'node:events';
3
1
  import { readFileSync } from 'node:fs';
4
2
  import { join } from 'node:path';
5
- import { MemoryStorage } from '@crawlee/memory-storage';
6
- import { pathExistsSync } from 'fs-extra/esm';
7
- import log, { LogLevel } from '@apify/log';
8
- import { LocalEventManager } from './events/local_event_manager.js';
9
- import { entries } from './typedefs.js';
3
+ import JSON5 from 'json5';
4
+ import { z } from 'zod';
5
+ import { log, LogLevel } from './log.js';
6
+ import { serviceLocator } from './service-locator.js';
7
+ export function field(schema, envVar) {
8
+ return { schema, envVar };
9
+ }
10
+ // --- Zod preprocessors ---
11
+ /** Zod preprocessor treating `'0'` and `'false'` as falsy. */
12
+ export const coerceBoolean = z.preprocess((val) => {
13
+ if (typeof val === 'string') {
14
+ return !['0', 'false'].includes(val.toLowerCase());
15
+ }
16
+ return val;
17
+ }, z.boolean());
18
+ export const coerceNumber = z.preprocess((val) => {
19
+ if (typeof val === 'string')
20
+ return Number(val);
21
+ return val;
22
+ }, z.number());
23
+ /** Zod schema accepting both LogLevel enum values and string names (case-insensitive). */
24
+ const logLevelSchema = z.preprocess((val) => {
25
+ if (val == null)
26
+ return val;
27
+ const s = String(val);
28
+ if (Number.isFinite(+s))
29
+ return +s;
30
+ const key = s.toUpperCase();
31
+ if (key in LogLevel)
32
+ return LogLevel[key];
33
+ return val;
34
+ }, z.enum(LogLevel));
35
+ // --- Crawlee config field definitions ---
36
+ export const crawleeConfigFields = {
37
+ /** @default true */
38
+ purgeOnStart: field(coerceBoolean.default(true), 'CRAWLEE_PURGE_ON_START'),
39
+ /** @default 0.95 */
40
+ maxUsedCpuRatio: field(coerceNumber.default(0.95)),
41
+ /** @default 0.25 */
42
+ availableMemoryRatio: field(coerceNumber.default(0.25), 'CRAWLEE_AVAILABLE_MEMORY_RATIO'),
43
+ memoryMbytes: field(coerceNumber.optional(), 'CRAWLEE_MEMORY_MBYTES'),
44
+ /** @default 60_000 */
45
+ persistStateIntervalMillis: field(coerceNumber.default(60_000), 'CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS'),
46
+ /**
47
+ * Internal safety-net timeout for a single request, in milliseconds. When unset the crawler derives it from
48
+ * the request handler timeout (twice it, and never below 5 minutes).
49
+ */
50
+ internalTimeoutMillis: field(coerceNumber.optional(), 'CRAWLEE_INTERNAL_TIMEOUT'),
51
+ /** @default 1_000 */
52
+ systemInfoIntervalMillis: field(coerceNumber.default(1_000)),
53
+ /** @default true */
54
+ headless: field(coerceBoolean.default(true), 'CRAWLEE_HEADLESS'),
55
+ /** @default false */
56
+ xvfb: field(coerceBoolean.default(false), 'CRAWLEE_XVFB'),
57
+ chromeExecutablePath: field(z.string().optional(), 'CRAWLEE_CHROME_EXECUTABLE_PATH'),
58
+ defaultBrowserPath: field(z.string().optional(), 'CRAWLEE_DEFAULT_BROWSER_PATH'),
59
+ /** @default false */
60
+ disableBrowserSandbox: field(coerceBoolean.default(false), 'CRAWLEE_DISABLE_BROWSER_SANDBOX'),
61
+ logLevel: field(logLevelSchema.optional(), 'CRAWLEE_LOG_LEVEL'),
62
+ /** @default true */
63
+ persistStorage: field(coerceBoolean.default(true), 'CRAWLEE_PERSIST_STORAGE'),
64
+ /** @default './storage' */
65
+ storageDir: field(z.string().default('./storage'), 'CRAWLEE_STORAGE_DIR'),
66
+ containerized: field(coerceBoolean.optional(), 'CRAWLEE_CONTAINERIZED'),
67
+ };
10
68
  /**
11
69
  * `Configuration` is a value object holding Crawlee configuration. By default, there is a
12
- * global singleton instance of this class available via `Configuration.getGlobalConfig()`.
70
+ * global singleton instance of this class available via `Configuration.getGlobalConfiguration()`.
13
71
  * Places that depend on a configurable behaviour depend on this class, as they have the global
14
72
  * instance as the default value.
15
73
  *
@@ -18,14 +76,10 @@ import { entries } from './typedefs.js';
18
76
  * import { BasicCrawler, Configuration } from 'crawlee';
19
77
  *
20
78
  * // Get the global configuration
21
- * const config = Configuration.getGlobalConfig();
22
- * // Set the 'persistStateIntervalMillis' option
23
- * // of global configuration to 10 seconds
24
- * config.set('persistStateIntervalMillis', 10_000);
25
- *
26
- * // No need to pass the configuration to the crawler,
27
- * // as it's using the global configuration by default
28
- * const crawler = new BasicCrawler();
79
+ * const config = Configuration.getGlobalConfiguration();
80
+ * // Access configuration values directly as properties
81
+ * console.log(config.headless);
82
+ * console.log(config.persistStateIntervalMillis);
29
83
  * ```
30
84
  *
31
85
  * *Using custom configuration:*
@@ -35,15 +89,14 @@ import { entries } from './typedefs.js';
35
89
  * // Create a new configuration
36
90
  * const config = new Configuration({ persistStateIntervalMillis: 30_000 });
37
91
  * // Pass the configuration to the crawler
38
- * const crawler = new BasicCrawler({ ... }, config);
92
+ * const crawler = new BasicCrawler({ configuration: config });
39
93
  * ```
40
94
  *
41
- * The configuration provided via environment variables always takes precedence. We can also
42
- * define the `crawlee.json` file in the project root directory which will serve as a baseline,
43
- * so the options provided in constructor will override those. In other words, the precedence is:
95
+ * Configuration is immutable — values are set via the constructor and cannot be changed afterwards.
96
+ * The priority order for resolving values is (highest to lowest):
44
97
  *
45
98
  * ```text
46
- * crawlee.json < constructor options < environment variables
99
+ * constructor options > environment variables > crawlee.json > schema defaults
47
100
  * ```
48
101
  *
49
102
  * ## Supported Configuration Options
@@ -53,250 +106,129 @@ import { entries } from './typedefs.js';
53
106
  * `memoryMbytes` | `CRAWLEE_MEMORY_MBYTES` | -
54
107
  * `logLevel` | `CRAWLEE_LOG_LEVEL` | -
55
108
  * `headless` | `CRAWLEE_HEADLESS` | `true`
56
- * `defaultDatasetId` | `CRAWLEE_DEFAULT_DATASET_ID` | `'default'`
57
- * `defaultKeyValueStoreId` | `CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID` | `'default'`
58
- * `defaultRequestQueueId` | `CRAWLEE_DEFAULT_REQUEST_QUEUE_ID` | `'default'`
59
109
  * `persistStateIntervalMillis` | `CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS` | `60_000`
110
+ * `internalTimeoutMillis` | `CRAWLEE_INTERNAL_TIMEOUT` | -
60
111
  * `purgeOnStart` | `CRAWLEE_PURGE_ON_START` | `true`
61
112
  * `persistStorage` | `CRAWLEE_PERSIST_STORAGE` | `true`
113
+ * `storageDir` | `CRAWLEE_STORAGE_DIR` | `'./storage'`
62
114
  *
63
115
  * ## Advanced Configuration Options
64
116
  *
65
117
  * Key | Environment Variable | Default Value
66
118
  * ---|---|---
67
- * `inputKey` | `CRAWLEE_INPUT_KEY` | `'INPUT'`
68
- * `xvfb` | `CRAWLEE_XVFB` | -
119
+ * `xvfb` | `CRAWLEE_XVFB` | `false`
69
120
  * `chromeExecutablePath` | `CRAWLEE_CHROME_EXECUTABLE_PATH` | -
70
121
  * `defaultBrowserPath` | `CRAWLEE_DEFAULT_BROWSER_PATH` | -
71
122
  * `disableBrowserSandbox` | `CRAWLEE_DISABLE_BROWSER_SANDBOX` | -
72
123
  * `availableMemoryRatio` | `CRAWLEE_AVAILABLE_MEMORY_RATIO` | `0.25`
73
- * `containerized | `CRAWLEE_CONTAINERIZED | -
124
+ * `containerized` | `CRAWLEE_CONTAINERIZED` | -
74
125
  */
126
+ // eslint-disable-next-line @typescript-eslint/no-unsafe-declaration-merging
75
127
  export class Configuration {
76
128
  /**
77
- * Maps environment variables to config keys (e.g. `CRAWLEE_MEMORY_MBYTES` to `memoryMbytes`)
78
- */
79
- static ENV_MAP = {
80
- CRAWLEE_AVAILABLE_MEMORY_RATIO: 'availableMemoryRatio',
81
- CRAWLEE_PURGE_ON_START: 'purgeOnStart',
82
- CRAWLEE_MEMORY_MBYTES: 'memoryMbytes',
83
- CRAWLEE_DEFAULT_DATASET_ID: 'defaultDatasetId',
84
- CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID: 'defaultKeyValueStoreId',
85
- CRAWLEE_DEFAULT_REQUEST_QUEUE_ID: 'defaultRequestQueueId',
86
- CRAWLEE_INPUT_KEY: 'inputKey',
87
- CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS: 'persistStateIntervalMillis',
88
- CRAWLEE_HEADLESS: 'headless',
89
- CRAWLEE_XVFB: 'xvfb',
90
- CRAWLEE_CHROME_EXECUTABLE_PATH: 'chromeExecutablePath',
91
- CRAWLEE_DEFAULT_BROWSER_PATH: 'defaultBrowserPath',
92
- CRAWLEE_DISABLE_BROWSER_SANDBOX: 'disableBrowserSandbox',
93
- CRAWLEE_LOG_LEVEL: 'logLevel',
94
- CRAWLEE_PERSIST_STORAGE: 'persistStorage',
95
- CRAWLEE_CONTAINERIZED: 'containerized',
96
- };
97
- static BOOLEAN_VARS = [
98
- 'purgeOnStart',
99
- 'headless',
100
- 'xvfb',
101
- 'disableBrowserSandbox',
102
- 'persistStorage',
103
- 'containerized',
104
- ];
105
- static INTEGER_VARS = ['memoryMbytes', 'persistStateIntervalMillis', 'systemInfoIntervalMillis'];
106
- static COMMA_SEPARATED_LIST_VARS = [];
107
- static DEFAULTS = {
108
- defaultKeyValueStoreId: 'default',
109
- defaultDatasetId: 'default',
110
- defaultRequestQueueId: 'default',
111
- inputKey: 'INPUT',
112
- maxUsedCpuRatio: 0.95,
113
- availableMemoryRatio: 0.25,
114
- storageClientOptions: {},
115
- purgeOnStart: true,
116
- headless: true,
117
- persistStateIntervalMillis: 60_000,
118
- systemInfoIntervalMillis: 1_000,
119
- persistStorage: true,
120
- };
121
- /**
122
- * Provides access to the current-instance-scoped Configuration without passing it around in parameters.
123
- * @internal
129
+ * Field definitions for this configuration class.
130
+ * Subclasses override this to register additional fields.
124
131
  */
125
- static storage = new AsyncLocalStorage();
126
- options;
127
- services = new Map();
128
- /** @internal */
129
- static globalConfig;
130
- storageManagers = new Map();
132
+ static fields = crawleeConfigFields;
133
+ #resolvedValues;
131
134
  /**
132
- * Creates new `Configuration` instance with provided options. Env vars will have precedence over those.
135
+ * Creates new `Configuration` instance with provided options.
136
+ * Constructor options take precedence over environment variables, which take precedence
137
+ * over crawlee.json values, which take precedence over schema defaults.
133
138
  */
134
139
  constructor(options = {}) {
135
- this.buildOptions(options);
136
- // Increase the global limit for event emitter memory leak warnings.
137
- EventEmitter.defaultMaxListeners = 50;
138
- // set the log level to support CRAWLEE_ prefixed env var too
139
- const logLevel = this.get('logLevel');
140
- if (logLevel) {
141
- const level = Number.isFinite(+logLevel)
142
- ? +logLevel
143
- : LogLevel[String(logLevel).toUpperCase()];
144
- log.setLevel(level);
145
- }
146
- }
147
- /**
148
- * Returns configured value. First checks the environment variables, then provided configuration,
149
- * fallbacks to the `defaultValue` argument if provided, otherwise uses the default value as described
150
- * in the above section.
151
- */
152
- get(key, defaultValue) {
153
- // prefer env vars, always iterate through the whole map as there might be duplicate env vars for the same option
154
- let envValue;
155
- for (const [k, v] of entries(Configuration.ENV_MAP)) {
156
- if (key === v) {
157
- envValue = process.env[k];
158
- if (envValue) {
159
- break;
160
- }
161
- }
140
+ const fields = this.constructor.fields;
141
+ const fileOptions = Configuration.#loadFileOptions();
142
+ this.#resolvedValues = Configuration.#resolveAll(fields, options, fileOptions);
143
+ this.#registerAccessors();
144
+ // Set the log level
145
+ const logLevel = this.logLevel;
146
+ if (logLevel != null) {
147
+ log.setLevel(logLevel);
162
148
  }
163
- if (envValue != null) {
164
- return this._castEnvValue(key, envValue);
165
- }
166
- // check instance level options
167
- if (this.options.has(key)) {
168
- return this.options.get(key);
169
- }
170
- // fallback to defaults
171
- return (defaultValue ?? Configuration.DEFAULTS[key] ?? envValue);
172
- }
173
- _castEnvValue(key, value) {
174
- if (Configuration.INTEGER_VARS.includes(key)) {
175
- return +value;
176
- }
177
- if (Configuration.BOOLEAN_VARS.includes(key)) {
178
- // 0, false and empty string are considered falsy values
179
- return !['0', 'false', ''].includes(String(value).toLowerCase());
180
- }
181
- if (Configuration.COMMA_SEPARATED_LIST_VARS.includes(key)) {
182
- if (!value)
183
- return [];
184
- return String(value)
185
- .split(',')
186
- .map((v) => v.trim());
187
- }
188
- return value;
189
149
  }
190
150
  /**
191
- * Sets value for given option. Only affects this `Configuration` instance, the value will not be propagated down to the env var.
192
- * To reset a value, we can omit the `value` argument or pass `undefined` there.
193
- */
194
- set(key, value) {
195
- this.options.set(key, value);
196
- }
197
- /**
198
- * Sets value for given option. Only affects the global `Configuration` instance, the value will not be propagated down to the env var.
199
- * To reset a value, we can omit the `value` argument or pass `undefined` there.
200
- */
201
- static set(key, value) {
202
- this.getGlobalConfig().set(key, value);
203
- }
204
- /**
205
- * Returns cached instance of {@link StorageClient} using options as defined in the environment variables or in
206
- * this {@link Configuration} instance. Only first call of this method will create the client, following calls will
207
- * return the same client instance.
151
+ * Returns the global configuration instance. It will respect the environment variables.
208
152
  *
209
- * Caching works based on the `storageClientOptions`, so calling this method with different options will return
210
- * multiple instances, one for each variant of the options.
211
- * @internal
153
+ * Delegates to the global ServiceLocator, making it the single source of truth for service management.
212
154
  */
213
- getStorageClient() {
214
- if (this.options.has('storageClient')) {
215
- return this.options.get('storageClient');
216
- }
217
- const options = this.options.get('storageClientOptions');
218
- return this.createMemoryStorage(options);
219
- }
220
- getEventManager() {
221
- if (this.options.has('eventManager')) {
222
- return this.options.get('eventManager');
223
- }
224
- if (this.services.has('eventManager')) {
225
- return this.services.get('eventManager');
226
- }
227
- const eventManager = new LocalEventManager(this);
228
- this.services.set('eventManager', eventManager);
229
- return eventManager;
155
+ static getGlobalConfiguration() {
156
+ return serviceLocator.getConfiguration();
230
157
  }
231
158
  /**
232
- * Creates an instance of MemoryStorage using options as defined in the environment variables or in this `Configuration` instance.
233
- * @internal
159
+ * Resolves all field values once using the priority chain:
160
+ * constructor options > env vars > crawlee.json > schema defaults.
234
161
  */
235
- createMemoryStorage(options = {}) {
236
- const cacheKey = `MemoryStorage-${JSON.stringify(options)}`;
237
- if (this.services.has(cacheKey)) {
238
- return this.services.get(cacheKey);
162
+ static #resolveAll(fields, userOptions, fileOptions) {
163
+ const values = {};
164
+ for (const [key, fieldDef] of Object.entries(fields)) {
165
+ // 1. Constructor options (highest priority)
166
+ if (key in userOptions && userOptions[key] !== undefined) {
167
+ values[key] = fieldDef.schema.parse(userOptions[key]);
168
+ continue;
169
+ }
170
+ // 2. Environment variables
171
+ const envValue = Configuration.#readEnvVar(fieldDef);
172
+ if (envValue != null) {
173
+ values[key] = fieldDef.schema.parse(envValue);
174
+ continue;
175
+ }
176
+ // 3. crawlee.json file options
177
+ if (key in fileOptions && fileOptions[key] !== undefined) {
178
+ values[key] = fieldDef.schema.parse(fileOptions[key]);
179
+ continue;
180
+ }
181
+ // 4. Schema default (by parsing undefined through the schema)
182
+ const parsed = fieldDef.schema.safeParse(undefined);
183
+ values[key] = parsed.success ? parsed.data : undefined;
239
184
  }
240
- const storage = new MemoryStorage({
241
- persistStorage: this.get('persistStorage'),
242
- // Override persistStorage if user provides it via storageClientOptions
243
- ...options,
244
- });
245
- this.services.set(cacheKey, storage);
246
- return storage;
247
- }
248
- useStorageClient(client) {
249
- this.options.set('storageClient', client);
250
- }
251
- static useStorageClient(client) {
252
- this.getGlobalConfig().useStorageClient(client);
253
- }
254
- useEventManager(events) {
255
- this.options.set('eventManager', events);
185
+ return values;
256
186
  }
257
187
  /**
258
- * Returns the global configuration instance. It will respect the environment variables.
188
+ * Registers getters (and throwing setters) on the instance for each field.
259
189
  */
260
- static getGlobalConfig() {
261
- if (Configuration.storage.getStore()) {
262
- return Configuration.storage.getStore();
190
+ #registerAccessors() {
191
+ const fields = this.constructor.fields;
192
+ const descriptors = {};
193
+ for (const key of Object.keys(fields)) {
194
+ descriptors[key] = {
195
+ get: () => this.#resolvedValues[key],
196
+ set() {
197
+ throw new TypeError('Configuration is immutable. Pass options via the constructor instead.');
198
+ },
199
+ enumerable: true,
200
+ configurable: false,
201
+ };
263
202
  }
264
- Configuration.globalConfig ??= new Configuration();
265
- return Configuration.globalConfig;
266
- }
267
- /**
268
- * Gets default {@link StorageClient} instance.
269
- */
270
- static getStorageClient() {
271
- return this.getGlobalConfig().getStorageClient();
203
+ Object.defineProperties(this, descriptors);
272
204
  }
273
205
  /**
274
- * Gets default {@link EventManager} instance.
206
+ * Reads the first defined env var value for a field definition.
207
+ * Empty strings are treated as unset, falling through to crawlee.json or schema defaults.
208
+ * (Crawlee v3 coerced `''` to `false`/`0`/`''` per type — v4 drops that for consistency.)
275
209
  */
276
- static getEventManager() {
277
- return this.getGlobalConfig().getEventManager();
210
+ static #readEnvVar(fieldDef) {
211
+ if (!fieldDef.envVar)
212
+ return undefined;
213
+ const envVars = Array.isArray(fieldDef.envVar) ? fieldDef.envVar : [fieldDef.envVar];
214
+ for (const envVar of envVars) {
215
+ const value = process.env[envVar];
216
+ if (value != null && value !== '')
217
+ return value;
218
+ }
219
+ return undefined;
278
220
  }
279
221
  /**
280
- * Resets global configuration instance. The default instance holds configuration based on env vars,
281
- * if we want to change them, we need to first reset the global state. Used mainly for testing purposes.
222
+ * Loads config options from crawlee.json in the current working directory.
282
223
  */
283
- static resetGlobalState() {
284
- delete this.globalConfig;
285
- }
286
- buildOptions(options) {
287
- // try to load configuration from crawlee.json as the baseline
288
- const path = join(process.cwd(), 'crawlee.json');
289
- if (pathExistsSync(path)) {
290
- try {
291
- const file = readFileSync(path);
292
- const optionsFromFileConfig = JSON.parse(file.toString());
293
- Object.assign(options, optionsFromFileConfig);
294
- }
295
- catch {
296
- // ignore
297
- }
224
+ static #loadFileOptions() {
225
+ try {
226
+ const file = readFileSync(join(process.cwd(), 'crawlee.json'), 'utf8');
227
+ // JSON5 also handles a leading UTF-8 byte order mark, which some Windows tools write
228
+ return JSON5.parse(file);
229
+ }
230
+ catch {
231
+ return {};
298
232
  }
299
- this.options = new Map(entries(options));
300
233
  }
301
234
  }
302
- //# sourceMappingURL=configuration.js.map
package/debug.d.ts ADDED
@@ -0,0 +1,36 @@
1
+ import type { IncomingMessage } from 'node:http';
2
+ import type { Dictionary } from '@crawlee/types';
3
+ import type { Request } from './request.js';
4
+ interface BrowserResponseLike {
5
+ status(): number;
6
+ }
7
+ /**
8
+ * Creates a standardized debug info from request and response. This info is usually added to dataset under the hidden `#debug` field.
9
+ *
10
+ * @param request [Request](https://sdk.apify.com/docs/api/request) object.
11
+ * @param [response]
12
+ * Puppeteer [`Response`](https://pptr.dev/#?product=Puppeteer&version=v1.11.0&show=api-class-response)
13
+ * or NodeJS [`http.IncomingMessage`](https://nodejs.org/api/http.html#http_class_http_serverresponse).
14
+ * @param [additionalFields] Object containing additional fields to be added.
15
+ *
16
+ * @internal
17
+ */
18
+ export declare function createRequestDebugInfo(request: Request, response?: IncomingMessage | Partial<BrowserResponseLike>, additionalFields?: Dictionary): Dictionary;
19
+ /**
20
+ * Returns a human-readable label for an unknown value,
21
+ * suitable for embedding in error messages and log output.
22
+ *
23
+ * Returns `constructor.name` when available (e.g. `"Configuration"`, `"Number"`),
24
+ * otherwise falls back to `util.inspect` (e.g. for `null`, `undefined`).
25
+ *
26
+ * @internal
27
+ */
28
+ export declare function inspectValue(value: unknown): string;
29
+ /**
30
+ * Returns the type of a value as a lowercase string, with `Date`, `Buffer` and `RegExp` reported
31
+ * by their constructor name. Used for building validation error messages.
32
+ *
33
+ * @internal
34
+ */
35
+ export declare function getObjectType(value: unknown): string;
36
+ export {};
package/debug.js ADDED
@@ -0,0 +1,70 @@
1
+ import { inspect } from 'node:util';
2
+ import { parseArgument, schemas } from './validators.js';
3
+ /**
4
+ * Creates a standardized debug info from request and response. This info is usually added to dataset under the hidden `#debug` field.
5
+ *
6
+ * @param request [Request](https://sdk.apify.com/docs/api/request) object.
7
+ * @param [response]
8
+ * Puppeteer [`Response`](https://pptr.dev/#?product=Puppeteer&version=v1.11.0&show=api-class-response)
9
+ * or NodeJS [`http.IncomingMessage`](https://nodejs.org/api/http.html#http_class_http_serverresponse).
10
+ * @param [additionalFields] Object containing additional fields to be added.
11
+ *
12
+ * @internal
13
+ */
14
+ export function createRequestDebugInfo(request, response = {}, additionalFields = {}) {
15
+ parseArgument(request, schemas.anyObject);
16
+ parseArgument(response, schemas.anyObject);
17
+ parseArgument(additionalFields, schemas.anyObject);
18
+ return {
19
+ requestId: request.id,
20
+ url: request.url,
21
+ loadedUrl: request.loadedUrl,
22
+ method: request.method,
23
+ retryCount: request.retryCount,
24
+ errorMessages: request.errorMessages,
25
+ // Puppeteer response has .status() function and NodeJS response, statusCode property.
26
+ statusCode: 'status' in response && response.status instanceof Function
27
+ ? response.status()
28
+ : response.statusCode,
29
+ ...additionalFields,
30
+ };
31
+ }
32
+ /**
33
+ * Returns a human-readable label for an unknown value,
34
+ * suitable for embedding in error messages and log output.
35
+ *
36
+ * Returns `constructor.name` when available (e.g. `"Configuration"`, `"Number"`),
37
+ * otherwise falls back to `util.inspect` (e.g. for `null`, `undefined`).
38
+ *
39
+ * @internal
40
+ */
41
+ export function inspectValue(value) {
42
+ if (typeof value === 'object' && value !== null && value.constructor?.name) {
43
+ return value.constructor.name;
44
+ }
45
+ return inspect(value, {
46
+ depth: 0,
47
+ compact: true,
48
+ maxStringLength: 64,
49
+ breakLength: Infinity,
50
+ colors: false,
51
+ });
52
+ }
53
+ /**
54
+ * Returns the type of a value as a lowercase string, with `Date`, `Buffer` and `RegExp` reported
55
+ * by their constructor name. Used for building validation error messages.
56
+ *
57
+ * @internal
58
+ */
59
+ export function getObjectType(value) {
60
+ const simple = typeof value;
61
+ if (['string', 'number', 'boolean', 'bigint'].includes(simple)) {
62
+ return simple;
63
+ }
64
+ const objectType = Object.prototype.toString.call(value);
65
+ const type = /\[object (\w+)]/.exec(objectType)[1];
66
+ if (type === 'Uint8Array') {
67
+ return 'Buffer';
68
+ }
69
+ return ['Date', 'Buffer', 'RegExp'].includes(type) ? type : type.toLowerCase();
70
+ }
package/errors.d.ts CHANGED
@@ -10,36 +10,63 @@ export declare class NonRetryableError extends Error {
10
10
  export declare class CriticalError extends NonRetryableError {
11
11
  }
12
12
  /**
13
- * @ignore
13
+ * A schema validation issue, structurally compatible with `StandardSchemaV1.Issue`. Declared here so that
14
+ * error types do not have to depend on `@standard-schema/spec`.
14
15
  */
15
- export declare class MissingRouteError extends CriticalError {
16
+ export interface SchemaIssue {
17
+ readonly message: string;
18
+ readonly path?: readonly (PropertyKey | {
19
+ key: PropertyKey;
20
+ })[];
16
21
  }
17
22
  /**
18
- * Errors of `RetryRequestError` type will always be retried by the crawler.
23
+ * Thrown when a request's `userData` does not match the {@link RouteSchemas|Standard Schema} registered for its label.
19
24
  *
20
- * *This error overrides the `maxRequestRetries` option, i.e. the request can be retried indefinitely until it succeeds.*
25
+ * As the `userData` does not change between attempts, this error is non-retryable.
21
26
  */
22
- export declare class RetryRequestError extends Error {
23
- constructor(message?: string);
27
+ export declare class RequestValidationError extends NonRetryableError {
28
+ readonly label: string | symbol;
29
+ readonly issues: readonly SchemaIssue[];
30
+ constructor(label: string | symbol, issues: readonly SchemaIssue[]);
24
31
  }
25
32
  /**
26
- * Errors of `SessionError` type will trigger a session rotation.
33
+ * Thrown by {@link RecoverableState} when a persisted state record does not match its `stateSchema`.
27
34
  *
28
- * This error doesn't respect the `maxRequestRetries` option and has a separate limit of `maxSessionRotations`.
35
+ * Whether a corrupt record should abort the run or be discarded in favour of the defaults depends on what the
36
+ * state is for, so {@link RecoverableState.initialize} always throws and leaves the choice to the caller.
29
37
  */
30
- export declare class SessionError extends RetryRequestError {
31
- constructor(message?: string);
38
+ export declare class StateValidationError extends Error {
39
+ readonly persistStateKey: string;
40
+ readonly issues: readonly SchemaIssue[];
41
+ constructor(persistStateKey: string, issues: readonly SchemaIssue[]);
32
42
  }
33
- export declare class ContextPipelineInterruptedError extends Error {
43
+ /**
44
+ * Errors of `SessionError` type retire the session associated with the request and trigger a regular retry.
45
+ *
46
+ * The retry counts towards the `maxRequestRetries` limit, just like any other error.
47
+ */
48
+ export declare class SessionError extends Error {
34
49
  constructor(message?: string);
35
50
  }
36
- export declare class ContextPipelineInitializationError extends Error {
37
- constructor(error: unknown, options?: ErrorOptions);
51
+ /**
52
+ * A {@link SessionError} for a session that finished without being blocked, e.g. by reaching its
53
+ * `maxUsageCount` or `maxAgeSecs`. A plain `SessionError` means the session was blocked.
54
+ */
55
+ export declare class SessionRetiredError extends SessionError {
38
56
  }
39
- export declare class ContextPipelineCleanupError extends CriticalError {
40
- constructor(error: unknown, options?: ErrorOptions);
57
+ /**
58
+ * Wraps the failure of a callback registered with {@link StorageTransaction.afterCommit|`afterCommit`}
59
+ * that ran after the request's writes had already been committed.
60
+ *
61
+ * Non-retryable by nature: the writes are durable, so re-running the request handler would duplicate
62
+ * them. A callback that throws a `NonRetryableError` of its own is left alone.
63
+ */
64
+ export declare class AfterCommitError extends NonRetryableError {
65
+ constructor(cause: unknown);
41
66
  }
42
- export declare class RequestHandlerError extends Error {
43
- constructor(error: unknown, options?: ErrorOptions);
67
+ /**
68
+ * Thrown when attempting to set a different service instance after one has already been retrieved.
69
+ */
70
+ export declare class ServiceConflictError extends Error {
71
+ constructor(serviceName: string, newValue: unknown, existingValue: unknown);
44
72
  }
45
- //# sourceMappingURL=errors.d.ts.map