@crawlee/core 4.0.0-beta.8 → 4.0.0-beta.80

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (257) hide show
  1. package/README.md +17 -13
  2. package/autoscaling/autoscaled_pool.d.ts +5 -6
  3. package/autoscaling/autoscaled_pool.js +12 -11
  4. package/autoscaling/client_load_signal.d.ts +25 -0
  5. package/autoscaling/client_load_signal.js +36 -0
  6. package/autoscaling/cpu_load_signal.d.ts +28 -0
  7. package/autoscaling/cpu_load_signal.js +24 -0
  8. package/autoscaling/event_loop_load_signal.d.ts +23 -0
  9. package/autoscaling/event_loop_load_signal.js +35 -0
  10. package/autoscaling/index.d.ts +5 -1
  11. package/autoscaling/index.js +5 -1
  12. package/autoscaling/load_signal.d.ts +99 -0
  13. package/autoscaling/load_signal.js +135 -0
  14. package/autoscaling/memory_load_signal.d.ts +43 -0
  15. package/autoscaling/memory_load_signal.js +102 -0
  16. package/autoscaling/snapshotter.d.ts +35 -72
  17. package/autoscaling/snapshotter.js +78 -181
  18. package/autoscaling/system_status.d.ts +21 -36
  19. package/autoscaling/system_status.js +46 -84
  20. package/configuration.d.ts +88 -228
  21. package/configuration.js +162 -224
  22. package/cookie_utils.d.ts +4 -3
  23. package/cookie_utils.js +20 -13
  24. package/crawlers/context_pipeline.d.ts +70 -0
  25. package/crawlers/context_pipeline.js +122 -0
  26. package/crawlers/crawler_commons.d.ts +20 -30
  27. package/crawlers/crawler_commons.js +12 -21
  28. package/crawlers/crawler_utils.d.ts +2 -3
  29. package/crawlers/crawler_utils.js +1 -2
  30. package/crawlers/error_snapshotter.d.ts +3 -3
  31. package/crawlers/error_snapshotter.js +2 -3
  32. package/crawlers/error_tracker.d.ts +2 -2
  33. package/crawlers/error_tracker.js +0 -1
  34. package/crawlers/index.d.ts +1 -2
  35. package/crawlers/index.js +1 -2
  36. package/crawlers/internals/types.d.ts +7 -0
  37. package/crawlers/internals/types.js +1 -0
  38. package/crawlers/statistics.d.ts +22 -17
  39. package/crawlers/statistics.js +30 -26
  40. package/enqueue_links/enqueue_links.d.ts +44 -22
  41. package/enqueue_links/enqueue_links.js +51 -30
  42. package/enqueue_links/index.d.ts +0 -1
  43. package/enqueue_links/index.js +0 -1
  44. package/enqueue_links/shared.d.ts +25 -9
  45. package/enqueue_links/shared.js +69 -38
  46. package/errors.d.ts +53 -4
  47. package/errors.js +70 -5
  48. package/events/event_manager.d.ts +34 -8
  49. package/events/event_manager.js +8 -10
  50. package/events/index.d.ts +0 -1
  51. package/events/index.js +0 -1
  52. package/events/local_event_manager.d.ts +15 -5
  53. package/events/local_event_manager.js +37 -40
  54. package/index.d.ts +4 -3
  55. package/index.js +3 -2
  56. package/log.d.ts +82 -3
  57. package/log.js +102 -1
  58. package/memory-storage/consts.d.ts +4 -0
  59. package/memory-storage/consts.js +4 -0
  60. package/memory-storage/index.d.ts +1 -0
  61. package/memory-storage/index.js +1 -0
  62. package/memory-storage/memory-storage.d.ts +46 -0
  63. package/memory-storage/memory-storage.js +136 -0
  64. package/memory-storage/resource-clients/common/base-client.d.ts +4 -0
  65. package/memory-storage/resource-clients/common/base-client.js +6 -0
  66. package/memory-storage/resource-clients/dataset.d.ts +40 -0
  67. package/memory-storage/resource-clients/dataset.js +113 -0
  68. package/memory-storage/resource-clients/key-value-store.d.ts +63 -0
  69. package/memory-storage/resource-clients/key-value-store.js +203 -0
  70. package/memory-storage/resource-clients/request-queue.d.ts +96 -0
  71. package/memory-storage/resource-clients/request-queue.js +421 -0
  72. package/memory-storage/utils.d.ts +17 -0
  73. package/memory-storage/utils.js +42 -0
  74. package/package.json +12 -10
  75. package/proxy_configuration.d.ts +14 -149
  76. package/proxy_configuration.js +19 -168
  77. package/recoverable_state.d.ts +120 -0
  78. package/recoverable_state.js +141 -0
  79. package/request.d.ts +84 -13
  80. package/request.js +107 -28
  81. package/router.d.ts +117 -15
  82. package/router.js +124 -30
  83. package/serialization.d.ts +0 -1
  84. package/serialization.js +1 -2
  85. package/service_locator.d.ts +156 -0
  86. package/service_locator.js +238 -0
  87. package/session_pool/consts.d.ts +1 -2
  88. package/session_pool/consts.js +1 -2
  89. package/session_pool/errors.d.ts +0 -1
  90. package/session_pool/errors.js +0 -1
  91. package/session_pool/fingerprint.d.ts +9 -0
  92. package/session_pool/fingerprint.js +30 -0
  93. package/session_pool/index.d.ts +0 -2
  94. package/session_pool/index.js +0 -2
  95. package/session_pool/session.d.ts +35 -73
  96. package/session_pool/session.js +46 -99
  97. package/session_pool/session_pool.d.ts +71 -72
  98. package/session_pool/session_pool.js +111 -100
  99. package/storages/access_checking.d.ts +0 -1
  100. package/storages/access_checking.js +5 -2
  101. package/storages/dataset.d.ts +101 -51
  102. package/storages/dataset.js +168 -126
  103. package/storages/index.d.ts +8 -7
  104. package/storages/index.js +6 -7
  105. package/storages/key_value_store.d.ts +165 -36
  106. package/storages/key_value_store.js +269 -122
  107. package/storages/key_value_store_codec.d.ts +32 -0
  108. package/storages/key_value_store_codec.js +113 -0
  109. package/storages/request_dedup_cache.d.ts +23 -0
  110. package/storages/request_dedup_cache.js +48 -0
  111. package/storages/request_list.d.ts +38 -82
  112. package/storages/request_list.js +65 -41
  113. package/storages/request_loader.d.ts +96 -0
  114. package/storages/request_loader.js +1 -0
  115. package/storages/request_manager.d.ts +33 -0
  116. package/storages/request_manager.js +1 -0
  117. package/storages/request_manager_tandem.d.ts +106 -0
  118. package/storages/request_manager_tandem.js +197 -0
  119. package/storages/request_queue.d.ts +303 -46
  120. package/storages/request_queue.js +629 -214
  121. package/storages/{sitemap_request_list.d.ts → sitemap_request_loader.d.ts} +28 -24
  122. package/storages/{sitemap_request_list.js → sitemap_request_loader.js} +49 -46
  123. package/storages/storage_instance_manager.d.ts +91 -0
  124. package/storages/storage_instance_manager.js +258 -0
  125. package/storages/storage_stats.d.ts +48 -0
  126. package/storages/storage_stats.js +29 -0
  127. package/storages/utils.d.ts +51 -6
  128. package/storages/utils.js +60 -9
  129. package/system-info/cpu-info.d.ts +67 -0
  130. package/system-info/cpu-info.js +216 -0
  131. package/system-info/memory-info.d.ts +31 -0
  132. package/system-info/memory-info.js +115 -0
  133. package/system-info/ps-tree.d.ts +17 -0
  134. package/system-info/ps-tree.js +144 -0
  135. package/system-info/runtime.d.ts +14 -0
  136. package/system-info/runtime.js +80 -0
  137. package/typedefs.d.ts +1 -2
  138. package/typedefs.js +0 -1
  139. package/validators.d.ts +8 -1
  140. package/validators.js +8 -1
  141. package/autoscaling/autoscaled_pool.d.ts.map +0 -1
  142. package/autoscaling/autoscaled_pool.js.map +0 -1
  143. package/autoscaling/index.d.ts.map +0 -1
  144. package/autoscaling/index.js.map +0 -1
  145. package/autoscaling/snapshotter.d.ts.map +0 -1
  146. package/autoscaling/snapshotter.js.map +0 -1
  147. package/autoscaling/system_status.d.ts.map +0 -1
  148. package/autoscaling/system_status.js.map +0 -1
  149. package/configuration.d.ts.map +0 -1
  150. package/configuration.js.map +0 -1
  151. package/cookie_utils.d.ts.map +0 -1
  152. package/cookie_utils.js.map +0 -1
  153. package/crawlers/crawler_commons.d.ts.map +0 -1
  154. package/crawlers/crawler_commons.js.map +0 -1
  155. package/crawlers/crawler_extension.d.ts +0 -12
  156. package/crawlers/crawler_extension.d.ts.map +0 -1
  157. package/crawlers/crawler_extension.js +0 -14
  158. package/crawlers/crawler_extension.js.map +0 -1
  159. package/crawlers/crawler_utils.d.ts.map +0 -1
  160. package/crawlers/crawler_utils.js.map +0 -1
  161. package/crawlers/error_snapshotter.d.ts.map +0 -1
  162. package/crawlers/error_snapshotter.js.map +0 -1
  163. package/crawlers/error_tracker.d.ts.map +0 -1
  164. package/crawlers/error_tracker.js.map +0 -1
  165. package/crawlers/index.d.ts.map +0 -1
  166. package/crawlers/index.js.map +0 -1
  167. package/crawlers/statistics.d.ts.map +0 -1
  168. package/crawlers/statistics.js.map +0 -1
  169. package/enqueue_links/enqueue_links.d.ts.map +0 -1
  170. package/enqueue_links/enqueue_links.js.map +0 -1
  171. package/enqueue_links/index.d.ts.map +0 -1
  172. package/enqueue_links/index.js.map +0 -1
  173. package/enqueue_links/shared.d.ts.map +0 -1
  174. package/enqueue_links/shared.js.map +0 -1
  175. package/errors.d.ts.map +0 -1
  176. package/errors.js.map +0 -1
  177. package/events/event_manager.d.ts.map +0 -1
  178. package/events/event_manager.js.map +0 -1
  179. package/events/index.d.ts.map +0 -1
  180. package/events/index.js.map +0 -1
  181. package/events/local_event_manager.d.ts.map +0 -1
  182. package/events/local_event_manager.js.map +0 -1
  183. package/http_clients/base-http-client.d.ts +0 -134
  184. package/http_clients/base-http-client.d.ts.map +0 -1
  185. package/http_clients/base-http-client.js +0 -33
  186. package/http_clients/base-http-client.js.map +0 -1
  187. package/http_clients/form-data-like.d.ts +0 -67
  188. package/http_clients/form-data-like.d.ts.map +0 -1
  189. package/http_clients/form-data-like.js +0 -5
  190. package/http_clients/form-data-like.js.map +0 -1
  191. package/http_clients/got-scraping-http-client.d.ts +0 -15
  192. package/http_clients/got-scraping-http-client.d.ts.map +0 -1
  193. package/http_clients/got-scraping-http-client.js +0 -69
  194. package/http_clients/got-scraping-http-client.js.map +0 -1
  195. package/http_clients/index.d.ts +0 -3
  196. package/http_clients/index.d.ts.map +0 -1
  197. package/http_clients/index.js +0 -3
  198. package/http_clients/index.js.map +0 -1
  199. package/index.d.ts.map +0 -1
  200. package/index.js.map +0 -1
  201. package/log.d.ts.map +0 -1
  202. package/log.js.map +0 -1
  203. package/proxy_configuration.d.ts.map +0 -1
  204. package/proxy_configuration.js.map +0 -1
  205. package/request.d.ts.map +0 -1
  206. package/request.js.map +0 -1
  207. package/router.d.ts.map +0 -1
  208. package/router.js.map +0 -1
  209. package/serialization.d.ts.map +0 -1
  210. package/serialization.js.map +0 -1
  211. package/session_pool/consts.d.ts.map +0 -1
  212. package/session_pool/consts.js.map +0 -1
  213. package/session_pool/errors.d.ts.map +0 -1
  214. package/session_pool/errors.js.map +0 -1
  215. package/session_pool/events.d.ts +0 -3
  216. package/session_pool/events.d.ts.map +0 -1
  217. package/session_pool/events.js +0 -3
  218. package/session_pool/events.js.map +0 -1
  219. package/session_pool/index.d.ts.map +0 -1
  220. package/session_pool/index.js.map +0 -1
  221. package/session_pool/session.d.ts.map +0 -1
  222. package/session_pool/session.js.map +0 -1
  223. package/session_pool/session_pool.d.ts.map +0 -1
  224. package/session_pool/session_pool.js.map +0 -1
  225. package/storages/access_checking.d.ts.map +0 -1
  226. package/storages/access_checking.js.map +0 -1
  227. package/storages/dataset.d.ts.map +0 -1
  228. package/storages/dataset.js.map +0 -1
  229. package/storages/index.d.ts.map +0 -1
  230. package/storages/index.js.map +0 -1
  231. package/storages/key_value_store.d.ts.map +0 -1
  232. package/storages/key_value_store.js.map +0 -1
  233. package/storages/request_list.d.ts.map +0 -1
  234. package/storages/request_list.js.map +0 -1
  235. package/storages/request_provider.d.ts +0 -307
  236. package/storages/request_provider.d.ts.map +0 -1
  237. package/storages/request_provider.js +0 -555
  238. package/storages/request_provider.js.map +0 -1
  239. package/storages/request_queue.d.ts.map +0 -1
  240. package/storages/request_queue.js.map +0 -1
  241. package/storages/request_queue_v2.d.ts +0 -87
  242. package/storages/request_queue_v2.d.ts.map +0 -1
  243. package/storages/request_queue_v2.js +0 -438
  244. package/storages/request_queue_v2.js.map +0 -1
  245. package/storages/sitemap_request_list.d.ts.map +0 -1
  246. package/storages/sitemap_request_list.js.map +0 -1
  247. package/storages/storage_manager.d.ts +0 -58
  248. package/storages/storage_manager.d.ts.map +0 -1
  249. package/storages/storage_manager.js +0 -105
  250. package/storages/storage_manager.js.map +0 -1
  251. package/storages/utils.d.ts.map +0 -1
  252. package/storages/utils.js.map +0 -1
  253. package/tsconfig.build.tsbuildinfo +0 -1
  254. package/typedefs.d.ts.map +0 -1
  255. package/typedefs.js.map +0 -1
  256. package/validators.d.ts.map +0 -1
  257. package/validators.js.map +0 -1
package/errors.d.ts CHANGED
@@ -14,6 +14,26 @@ export declare class CriticalError extends NonRetryableError {
14
14
  */
15
15
  export declare class MissingRouteError extends CriticalError {
16
16
  }
17
+ /**
18
+ * Thrown when a request's `userData` does not match the {@link RouteSchemas|Standard Schema} registered for its label.
19
+ *
20
+ * As the `userData` does not change between attempts, this error is non-retryable.
21
+ */
22
+ export declare class RequestValidationError extends NonRetryableError {
23
+ readonly label: string | symbol;
24
+ readonly issues: readonly {
25
+ readonly message: string;
26
+ readonly path?: readonly (PropertyKey | {
27
+ key: PropertyKey;
28
+ })[];
29
+ }[];
30
+ constructor(label: string | symbol, issues: readonly {
31
+ readonly message: string;
32
+ readonly path?: readonly (PropertyKey | {
33
+ key: PropertyKey;
34
+ })[];
35
+ }[]);
36
+ }
17
37
  /**
18
38
  * Errors of `RetryRequestError` type will always be retried by the crawler.
19
39
  *
@@ -23,11 +43,40 @@ export declare class RetryRequestError extends Error {
23
43
  constructor(message?: string);
24
44
  }
25
45
  /**
26
- * Errors of `SessionError` type will trigger a session rotation.
46
+ * Errors of `SessionError` type retire the session associated with the request and trigger a regular retry.
27
47
  *
28
- * This error doesn't respect the `maxRequestRetries` option and has a separate limit of `maxSessionRotations`.
48
+ * The retry counts towards the `maxRequestRetries` limit, just like any other error.
29
49
  */
30
- export declare class SessionError extends RetryRequestError {
50
+ export declare class SessionError extends Error {
51
+ constructor(message?: string);
52
+ }
53
+ /**
54
+ * Thrown when a requested session is not found in the referenced SessionPool.
55
+ */
56
+ export declare class MissingSessionError extends Error {
57
+ constructor(sessionId?: string);
58
+ }
59
+ export declare class ContextPipelineInterruptedError extends Error {
31
60
  constructor(message?: string);
32
61
  }
33
- //# sourceMappingURL=errors.d.ts.map
62
+ export declare class ContextPipelineInitializationError extends Error {
63
+ constructor(error: unknown, options?: ErrorOptions);
64
+ }
65
+ export declare class ContextPipelineCleanupError extends CriticalError {
66
+ constructor(error: unknown, options?: ErrorOptions);
67
+ }
68
+ export declare class RequestHandlerError extends Error {
69
+ constructor(error: unknown, options?: ErrorOptions);
70
+ }
71
+ /**
72
+ * Thrown when attempting to set a different service instance after one has already been retrieved.
73
+ */
74
+ export declare class ServiceConflictError extends Error {
75
+ constructor(serviceName: string, newValue: unknown, existingValue: unknown);
76
+ }
77
+ /**
78
+ * Thrown by crawlers when `skipNavigation` is used on a request.
79
+ * Subclasses can catch this error to skip their own navigation-dependent logic.
80
+ */
81
+ export declare class NavigationSkippedError extends NonRetryableError {
82
+ }
package/errors.js CHANGED
@@ -1,3 +1,4 @@
1
+ import { inspectValue } from '@crawlee/utils';
1
2
  /**
2
3
  * Errors of `NonRetryableError` type will never be retried by the crawler.
3
4
  */
@@ -14,6 +15,28 @@ export class CriticalError extends NonRetryableError {
14
15
  */
15
16
  export class MissingRouteError extends CriticalError {
16
17
  }
18
+ /**
19
+ * Thrown when a request's `userData` does not match the {@link RouteSchemas|Standard Schema} registered for its label.
20
+ *
21
+ * As the `userData` does not change between attempts, this error is non-retryable.
22
+ */
23
+ export class RequestValidationError extends NonRetryableError {
24
+ label;
25
+ issues;
26
+ constructor(label, issues) {
27
+ const details = issues
28
+ .map((issue) => {
29
+ const path = (issue.path ?? [])
30
+ .map((segment) => (typeof segment === 'object' ? segment.key : segment))
31
+ .join('.');
32
+ return `- ${path ? `${path}: ` : ''}${issue.message}`;
33
+ })
34
+ .join('\n');
35
+ super(`Request userData for label '${String(label)}' failed schema validation:\n${details}`);
36
+ this.label = label;
37
+ this.issues = issues;
38
+ }
39
+ }
17
40
  /**
18
41
  * Errors of `RetryRequestError` type will always be retried by the crawler.
19
42
  *
@@ -25,13 +48,55 @@ export class RetryRequestError extends Error {
25
48
  }
26
49
  }
27
50
  /**
28
- * Errors of `SessionError` type will trigger a session rotation.
51
+ * Errors of `SessionError` type retire the session associated with the request and trigger a regular retry.
29
52
  *
30
- * This error doesn't respect the `maxRequestRetries` option and has a separate limit of `maxSessionRotations`.
53
+ * The retry counts towards the `maxRequestRetries` limit, just like any other error.
31
54
  */
32
- export class SessionError extends RetryRequestError {
55
+ export class SessionError extends Error {
33
56
  constructor(message) {
34
- super(`Detected a session error, rotating session... ${message ? `\n${message}` : ''}`);
57
+ super(`Detected a session error, retiring session... ${message ? `\n${message}` : ''}`);
35
58
  }
36
59
  }
37
- //# sourceMappingURL=errors.js.map
60
+ /**
61
+ * Thrown when a requested session is not found in the referenced SessionPool.
62
+ */
63
+ export class MissingSessionError extends Error {
64
+ constructor(sessionId) {
65
+ super(`The current SessionPool instance couldn't find a valid session${sessionId ? ` for the following id: ${sessionId}.` : '.'}`);
66
+ }
67
+ }
68
+ export class ContextPipelineInterruptedError extends Error {
69
+ constructor(message) {
70
+ super(`Request handling was interrupted during context initialization ${message ? ` - ${message}` : ''}`);
71
+ }
72
+ }
73
+ export class ContextPipelineInitializationError extends Error {
74
+ constructor(error, options) {
75
+ super(undefined, { cause: error, ...options });
76
+ }
77
+ }
78
+ export class ContextPipelineCleanupError extends CriticalError {
79
+ constructor(error, options) {
80
+ super(undefined, { cause: error, ...options });
81
+ }
82
+ }
83
+ export class RequestHandlerError extends Error {
84
+ constructor(error, options) {
85
+ super(undefined, { cause: error, ...options });
86
+ }
87
+ }
88
+ /**
89
+ * Thrown when attempting to set a different service instance after one has already been retrieved.
90
+ */
91
+ export class ServiceConflictError extends Error {
92
+ constructor(serviceName, newValue, existingValue) {
93
+ super(`Service ${serviceName} is already in use. ` +
94
+ `Existing value: ${inspectValue(existingValue)}, attempted new value: ${inspectValue(newValue)}.`);
95
+ }
96
+ }
97
+ /**
98
+ * Thrown by crawlers when `skipNavigation` is used on a request.
99
+ * Subclasses can catch this error to skip their own navigation-dependent logic.
100
+ */
101
+ export class NavigationSkippedError extends NonRetryableError {
102
+ }
@@ -1,28 +1,55 @@
1
1
  import { AsyncEventEmitter } from '@vladfrangu/async_event_emitter';
2
2
  import type { BetterIntervalID } from '@apify/utilities';
3
- import { Configuration } from '../configuration.js';
3
+ export interface EventManagerOptions {
4
+ /** Interval between emitted `persistState` events in milliseconds. */
5
+ persistStateIntervalMillis: number;
6
+ }
4
7
  export declare const enum EventType {
5
8
  PERSIST_STATE = "persistState",
6
9
  SYSTEM_INFO = "systemInfo",
7
10
  MIGRATING = "migrating",
8
11
  ABORTING = "aborting",
9
- EXIT = "exit"
12
+ EXIT = "exit",
13
+ STATUS_MESSAGE = "statusMessage"
14
+ }
15
+ export type EventTypeName = EventType | 'systemInfo' | 'persistState' | 'migrating' | 'aborting' | 'exit' | 'statusMessage';
16
+ /**
17
+ * Payload emitted with the {@link EventType.STATUS_MESSAGE|`statusMessage`} event.
18
+ *
19
+ * The crawler broadcasts these whenever it wants to report its progress (e.g. periodically, or on
20
+ * start/finish). Consumers such as the Apify SDK can listen for the event and propagate the message
21
+ * to the platform. This keeps the crawler decoupled from any specific status-reporting backend.
22
+ */
23
+ export interface EventStatusMessageData {
24
+ /**
25
+ * Identifies the crawler that emitted the message.
26
+ *
27
+ * Either the user-provided `id` from the crawler options, or a randomly generated one.
28
+ * Since a single event manager may be shared by multiple crawlers, consumers can use this
29
+ * to attribute the message to a specific crawler instance.
30
+ */
31
+ crawlerId: string;
32
+ /** The human-readable status message. */
33
+ message: string;
34
+ /** Whether this is the final status message of the run. */
35
+ isStatusMessageTerminal?: boolean;
36
+ /** The log level the message was logged with. */
37
+ level?: 'DEBUG' | 'INFO' | 'WARNING' | 'ERROR';
10
38
  }
11
- export type EventTypeName = EventType | 'systemInfo' | 'persistState' | 'migrating' | 'aborting' | 'exit';
12
39
  interface Intervals {
13
40
  persistState?: BetterIntervalID;
14
41
  systemInfo?: BetterIntervalID;
15
42
  }
16
43
  export declare abstract class EventManager {
17
- readonly config: Configuration;
18
44
  protected events: AsyncEventEmitter<{}>;
19
45
  protected initialized: boolean;
20
46
  protected intervals: Intervals;
21
47
  // @ts-ignore optional peer dependency or compatibility with es2022
22
- protected log: import("@apify/log").Log;
23
- constructor(config?: Configuration);
48
+ protected log: import("@crawlee/types").CrawleeLogger;
49
+ private persistStateIntervalMillis;
50
+ constructor(options: EventManagerOptions);
24
51
  /**
25
- * Initializes the event manager by creating the `persistState` event interval.
52
+ * Initializes the event manager by starting the `persistState` event interval.
26
53
  * This is automatically called at the beginning of `crawler.run()`.
27
54
  */
28
55
  init(): Promise<void>;
@@ -49,4 +76,3 @@ export declare abstract class EventManager {
49
76
  waitForAllListenersToComplete(): Promise<boolean>;
50
77
  }
51
78
  export {};
52
- //# sourceMappingURL=event_manager.d.ts.map
@@ -1,7 +1,6 @@
1
1
  import { AsyncEventEmitter } from '@vladfrangu/async_event_emitter';
2
- import log from '@apify/log';
3
2
  import { betterClearInterval, betterSetInterval } from '@apify/utilities';
4
- import { Configuration } from '../configuration.js';
3
+ import { serviceLocator } from '../service_locator.js';
5
4
  export var EventType;
6
5
  (function (EventType) {
7
6
  EventType["PERSIST_STATE"] = "persistState";
@@ -9,30 +8,30 @@ export var EventType;
9
8
  EventType["MIGRATING"] = "migrating";
10
9
  EventType["ABORTING"] = "aborting";
11
10
  EventType["EXIT"] = "exit";
11
+ EventType["STATUS_MESSAGE"] = "statusMessage";
12
12
  })(EventType || (EventType = {}));
13
13
  export class EventManager {
14
- config;
15
14
  events = new AsyncEventEmitter();
16
15
  initialized = false;
17
16
  intervals = {};
18
- log = log.child({ prefix: 'Events' });
19
- constructor(config = Configuration.getGlobalConfig()) {
20
- this.config = config;
17
+ log = serviceLocator.getLogger().child({ prefix: 'Events' });
18
+ persistStateIntervalMillis;
19
+ constructor(options) {
20
+ this.persistStateIntervalMillis = options.persistStateIntervalMillis;
21
21
  this.events.setMaxListeners(50);
22
22
  }
23
23
  /**
24
- * Initializes the event manager by creating the `persistState` event interval.
24
+ * Initializes the event manager by starting the `persistState` event interval.
25
25
  * This is automatically called at the beginning of `crawler.run()`.
26
26
  */
27
27
  async init() {
28
28
  if (this.initialized) {
29
29
  return;
30
30
  }
31
- const persistStateIntervalMillis = this.config.get('persistStateIntervalMillis');
32
31
  this.intervals.persistState = betterSetInterval((intervalCallback) => {
33
32
  this.emit("persistState" /* EventType.PERSIST_STATE */, { isMigrating: false });
34
33
  intervalCallback();
35
- }, persistStateIntervalMillis);
34
+ }, this.persistStateIntervalMillis);
36
35
  this.initialized = true;
37
36
  }
38
37
  /**
@@ -86,4 +85,3 @@ export class EventManager {
86
85
  return this.events.waitForAllListenersToComplete();
87
86
  }
88
87
  }
89
- //# sourceMappingURL=event_manager.js.map
package/events/index.d.ts CHANGED
@@ -1,3 +1,2 @@
1
1
  export * from './event_manager.js';
2
2
  export * from './local_event_manager.js';
3
- //# sourceMappingURL=index.d.ts.map
package/events/index.js CHANGED
@@ -1,3 +1,2 @@
1
1
  export * from './event_manager.js';
2
2
  export * from './local_event_manager.js';
3
- //# sourceMappingURL=index.js.map
@@ -1,8 +1,19 @@
1
- import { EventManager } from './event_manager.js';
1
+ import type { Configuration } from '../configuration.js';
2
+ import { EventManager, type EventManagerOptions } from './event_manager.js';
3
+ export interface LocalEventManagerOptions extends EventManagerOptions {
4
+ /** Interval between emitted `systemInfo` events in milliseconds. */
5
+ systemInfoIntervalMillis: number;
6
+ }
2
7
  export declare class LocalEventManager extends EventManager {
3
- private previousTicks;
8
+ private systemInfoIntervalMillis;
9
+ constructor(options: LocalEventManagerOptions);
10
+ /**
11
+ * Creates a new `LocalEventManager` based on the provided `Configuration`.
12
+ * Uses the global configuration from the service locator if none is provided.
13
+ */
14
+ static fromConfig(config?: Configuration): LocalEventManager;
4
15
  /**
5
- * Initializes the EventManager and sets up periodic `systemInfo` and `persistState` events.
16
+ * Initializes the EventManager and sets up periodic `systemInfo` events.
6
17
  * This is automatically called at the beginning of `crawler.run()`.
7
18
  */
8
19
  init(): Promise<void>;
@@ -18,12 +29,11 @@ export declare class LocalEventManager extends EventManager {
18
29
  * @internal
19
30
  */
20
31
  isContainerizedWrapper(): Promise<boolean>;
21
- private getCurrentCpuTicks;
22
32
  /**
23
33
  * Creates a SystemInfo object based on local metrics.
24
34
  */
25
35
  private createSystemInfo;
26
36
  private createCpuInfo;
27
37
  private createMemoryInfo;
38
+ private getMemoryInfo;
28
39
  }
29
- //# sourceMappingURL=local_event_manager.d.ts.map
@@ -1,12 +1,25 @@
1
- import os from 'node:os';
2
- import { getCurrentCpuTicksV2, getMemoryInfo, getMemoryInfoV2, isContainerized } from '@crawlee/utils';
3
- import log from '@apify/log';
4
1
  import { betterClearInterval, betterSetInterval } from '@apify/utilities';
2
+ import { serviceLocator } from '../service_locator.js';
5
3
  import { EventManager } from './event_manager.js';
6
4
  export class LocalEventManager extends EventManager {
7
- previousTicks = { idle: 0, total: 0 };
5
+ systemInfoIntervalMillis;
6
+ constructor(options) {
7
+ super(options);
8
+ this.systemInfoIntervalMillis = options.systemInfoIntervalMillis;
9
+ }
8
10
  /**
9
- * Initializes the EventManager and sets up periodic `systemInfo` and `persistState` events.
11
+ * Creates a new `LocalEventManager` based on the provided `Configuration`.
12
+ * Uses the global configuration from the service locator if none is provided.
13
+ */
14
+ static fromConfig(config) {
15
+ const resolvedConfig = config ?? serviceLocator.getConfiguration();
16
+ return new LocalEventManager({
17
+ persistStateIntervalMillis: resolvedConfig.persistStateIntervalMillis,
18
+ systemInfoIntervalMillis: resolvedConfig.systemInfoIntervalMillis,
19
+ });
20
+ }
21
+ /**
22
+ * Initializes the EventManager and sets up periodic `systemInfo` events.
10
23
  * This is automatically called at the beginning of `crawler.run()`.
11
24
  */
12
25
  async init() {
@@ -14,9 +27,8 @@ export class LocalEventManager extends EventManager {
14
27
  return;
15
28
  }
16
29
  await super.init();
17
- const systemInfoIntervalMillis = this.config.get('systemInfoIntervalMillis');
18
30
  this.emitSystemInfoEvent = this.emitSystemInfoEvent.bind(this);
19
- this.intervals.systemInfo = betterSetInterval(this.emitSystemInfoEvent.bind(this), systemInfoIntervalMillis);
31
+ this.intervals.systemInfo = betterSetInterval(this.emitSystemInfoEvent.bind(this), this.systemInfoIntervalMillis);
20
32
  }
21
33
  /**
22
34
  * @inheritDoc
@@ -33,7 +45,7 @@ export class LocalEventManager extends EventManager {
33
45
  */
34
46
  async emitSystemInfoEvent(intervalCallback) {
35
47
  const info = await this.createSystemInfo({
36
- maxUsedCpuRatio: this.config.get('maxUsedCpuRatio'),
48
+ maxUsedCpuRatio: serviceLocator.getConfiguration().maxUsedCpuRatio,
37
49
  });
38
50
  this.events.emit("systemInfo" /* EventType.SYSTEM_INFO */, info);
39
51
  intervalCallback();
@@ -42,17 +54,8 @@ export class LocalEventManager extends EventManager {
42
54
  * @internal
43
55
  */
44
56
  async isContainerizedWrapper() {
45
- return this.config.get('containerized', await isContainerized());
46
- }
47
- getCurrentCpuTicks() {
48
- const cpus = os.cpus();
49
- return cpus.reduce((acc, cpu) => {
50
- const cpuTimes = Object.values(cpu.times);
51
- return {
52
- idle: acc.idle + cpu.times.idle,
53
- total: acc.total + cpuTimes.reduce((sum, num) => sum + num),
54
- };
55
- }, { idle: 0, total: 0 });
57
+ const { isContainerized } = await import('../system-info/runtime.js');
58
+ return serviceLocator.getConfiguration().containerized ?? (await isContainerized());
56
59
  }
57
60
  /**
58
61
  * Creates a SystemInfo object based on local metrics.
@@ -65,18 +68,11 @@ export class LocalEventManager extends EventManager {
65
68
  };
66
69
  }
67
70
  async createCpuInfo(options) {
68
- if (this.config.get('systemInfoV2')) {
69
- const usedCpuRatio = await getCurrentCpuTicksV2(await this.isContainerizedWrapper());
70
- return {
71
- cpuCurrentUsage: usedCpuRatio * 100,
72
- isCpuOverloaded: usedCpuRatio > options.maxUsedCpuRatio,
73
- };
74
- }
75
- const ticks = this.getCurrentCpuTicks();
76
- const idleTicksDelta = ticks.idle - this.previousTicks.idle;
77
- const totalTicksDelta = ticks.total - this.previousTicks.total;
78
- const usedCpuRatio = totalTicksDelta ? 1 - idleTicksDelta / totalTicksDelta : 0;
79
- Object.assign(this.previousTicks, ticks);
71
+ const { getCurrentCpuTicksV2 } = await import('../system-info/cpu-info.js');
72
+ const usedCpuRatio = await getCurrentCpuTicksV2({
73
+ containerized: await this.isContainerizedWrapper(),
74
+ logger: serviceLocator.getLogger(),
75
+ });
80
76
  return {
81
77
  cpuCurrentUsage: usedCpuRatio * 100,
82
78
  isCpuOverloaded: usedCpuRatio > options.maxUsedCpuRatio,
@@ -84,21 +80,22 @@ export class LocalEventManager extends EventManager {
84
80
  }
85
81
  async createMemoryInfo() {
86
82
  try {
87
- if (this.config.get('systemInfoV2')) {
88
- const memInfo = await getMemoryInfoV2(await this.isContainerizedWrapper());
89
- return {
90
- memCurrentBytes: memInfo.mainProcessBytes + memInfo.childProcessesBytes,
91
- };
92
- }
93
- const memInfo = await getMemoryInfo();
83
+ const memInfo = await this.getMemoryInfo();
94
84
  return {
85
+ memTotalBytes: memInfo.totalBytes,
95
86
  memCurrentBytes: memInfo.mainProcessBytes + memInfo.childProcessesBytes,
96
87
  };
97
88
  }
98
89
  catch (err) {
99
- log.exception(err, 'Memory snapshot failed.');
90
+ this.log.exception(err, 'Memory snapshot failed.');
100
91
  return {};
101
92
  }
102
93
  }
94
+ async getMemoryInfo() {
95
+ const { getMemoryInfo } = await import('../system-info/memory-info.js');
96
+ return getMemoryInfo({
97
+ containerized: await this.isContainerizedWrapper(),
98
+ logger: serviceLocator.getLogger(),
99
+ });
100
+ }
103
101
  }
104
- //# sourceMappingURL=local_event_manager.js.map
package/index.d.ts CHANGED
@@ -1,10 +1,10 @@
1
1
  export * from './errors.js';
2
2
  export * from './autoscaling/index.js';
3
3
  export * from './configuration.js';
4
+ export * from './service_locator.js';
4
5
  export * from './crawlers/index.js';
5
6
  export * from './enqueue_links/index.js';
6
7
  export * from './events/index.js';
7
- export * from './http_clients/index.js';
8
8
  export * from './log.js';
9
9
  export * from './proxy_configuration.js';
10
10
  export * from './request.js';
@@ -12,8 +12,9 @@ export * from './router.js';
12
12
  export * from './serialization.js';
13
13
  export * from './session_pool/index.js';
14
14
  export * from './storages/index.js';
15
+ export * from './memory-storage/index.js';
15
16
  export * from './validators.js';
16
17
  export * from './cookie_utils.js';
18
+ export * from './recoverable_state.js';
17
19
  export { PseudoUrl } from '@apify/pseudo_url';
18
- export { Dictionary, Awaitable, Constructor, StorageClient, Cookie, QueueOperationInfo } from '@crawlee/types';
19
- //# sourceMappingURL=index.d.ts.map
20
+ export type { Dictionary, Awaitable, Constructor, StorageBackend, Cookie, QueueOperationInfo } from '@crawlee/types';
package/index.js CHANGED
@@ -1,10 +1,10 @@
1
1
  export * from './errors.js';
2
2
  export * from './autoscaling/index.js';
3
3
  export * from './configuration.js';
4
+ export * from './service_locator.js';
4
5
  export * from './crawlers/index.js';
5
6
  export * from './enqueue_links/index.js';
6
7
  export * from './events/index.js';
7
- export * from './http_clients/index.js';
8
8
  export * from './log.js';
9
9
  export * from './proxy_configuration.js';
10
10
  export * from './request.js';
@@ -12,7 +12,8 @@ export * from './router.js';
12
12
  export * from './serialization.js';
13
13
  export * from './session_pool/index.js';
14
14
  export * from './storages/index.js';
15
+ export * from './memory-storage/index.js';
15
16
  export * from './validators.js';
16
17
  export * from './cookie_utils.js';
18
+ export * from './recoverable_state.js';
17
19
  export { PseudoUrl } from '@apify/pseudo_url';
18
- //# sourceMappingURL=index.js.map
package/log.d.ts CHANGED
@@ -1,3 +1,82 @@
1
- import log, { Log, Logger, LoggerJson, LoggerOptions, LoggerText, LogLevel } from '@apify/log';
2
- export { log, Log, LoggerOptions, LogLevel, Logger, LoggerJson, LoggerText };
3
- //# sourceMappingURL=log.d.ts.map
1
+ import type { CrawleeLogger, CrawleeLoggerOptions } from '@crawlee/types';
2
+ import type { LoggerOptions } from '@apify/log';
3
+ import log, { Log, Logger, LoggerJson, LoggerText, LogLevel } from '@apify/log';
4
+ export type { CrawleeLogger, CrawleeLoggerOptions };
5
+ /**
6
+ * Abstract base class for custom Crawlee logger implementations.
7
+ *
8
+ * Subclasses must implement two methods:
9
+ * - {@link BaseCrawleeLogger.logWithLevel} — the core logging dispatch
10
+ * - {@link BaseCrawleeLogger.createChild} — how to create a child logger instance
11
+ *
12
+ * All other `CrawleeLogger` methods (`error`, `warning`, `info`, `debug`, etc.)
13
+ * are derived automatically. Level filtering is entirely the responsibility of the
14
+ * underlying library — `logWithLevel()` is called for every message.
15
+ *
16
+ * **Example — Winston adapter:**
17
+ * ```typescript
18
+ * const CRAWLEE_TO_WINSTON = { 1: 'error', 2: 'warn', 3: 'warn', 4: 'info', 5: 'debug', 6: 'debug' };
19
+ *
20
+ * class WinstonAdapter extends BaseCrawleeLogger {
21
+ * constructor(private logger: winston.Logger, options?: Partial<CrawleeLoggerOptions>) {
22
+ * super(options);
23
+ * }
24
+ *
25
+ * logWithLevel(level: number, message: string, data?: Record<string, unknown>): void {
26
+ * this.logger.log(CRAWLEE_TO_WINSTON[level] ?? 'info', message, data);
27
+ * }
28
+ *
29
+ * protected createChild(options: Partial<CrawleeLoggerOptions>): CrawleeLogger {
30
+ * return new WinstonAdapter(this.logger.child({ prefix: options.prefix }), { ...this.getOptions(), ...options });
31
+ * }
32
+ * }
33
+ * ```
34
+ */
35
+ export declare abstract class BaseCrawleeLogger implements CrawleeLogger {
36
+ private options;
37
+ private readonly warningsLogged;
38
+ constructor(options?: Partial<CrawleeLoggerOptions>);
39
+ /**
40
+ * Core logging method. Subclasses must implement this to dispatch log messages
41
+ * to the underlying logger (Winston, Pino, console, etc.).
42
+ *
43
+ * Level filtering is the responsibility of the underlying library — this method
44
+ * is called for every message regardless of the current level.
45
+ *
46
+ * @param level Crawlee log level (use {@link LogLevel} constants)
47
+ * @param message The log message
48
+ * @param data Optional structured data to attach to the log entry
49
+ */
50
+ abstract logWithLevel(level: number, message: string, data?: Record<string, unknown>): void;
51
+ /**
52
+ * Creates a child logger instance. Subclasses must implement this to define
53
+ * how child loggers are created for the underlying logger.
54
+ */
55
+ protected abstract createChild(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;
56
+ getOptions(): CrawleeLoggerOptions;
57
+ setOptions(options: Partial<CrawleeLoggerOptions>): void;
58
+ child(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;
59
+ error(message: string, data?: Record<string, unknown>): void;
60
+ exception(exception: Error, message: string, data?: Record<string, unknown>): void;
61
+ softFail(message: string, data?: Record<string, unknown>): void;
62
+ warning(message: string, data?: Record<string, unknown>): void;
63
+ warningOnce(message: string): void;
64
+ info(message: string, data?: Record<string, unknown>): void;
65
+ debug(message: string, data?: Record<string, unknown>): void;
66
+ perf(message: string, data?: Record<string, unknown>): void;
67
+ deprecated(message: string): void;
68
+ }
69
+ /**
70
+ * Adapter that wraps `@apify/log`'s {@link Log} instance to implement the {@link CrawleeLogger} interface.
71
+ *
72
+ * This is the default logger used by Crawlee when no custom logger is configured.
73
+ * Users who want to use a different logging library should implement {@link BaseCrawleeLogger} directly.
74
+ */
75
+ export declare class ApifyLogAdapter extends BaseCrawleeLogger {
76
+ private readonly apifyLog;
77
+ constructor(apifyLog: Log, options?: Partial<CrawleeLoggerOptions>);
78
+ logWithLevel(level: number, message: string, data?: Record<string, unknown>): void;
79
+ protected createChild(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;
80
+ }
81
+ export { log, Log, LogLevel, Logger, LoggerJson, LoggerText };
82
+ export type { LoggerOptions };