@crawlee/core 4.0.0-beta.70 → 4.0.0-beta.72

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (249) hide show
  1. package/autoscaling/autoscaled_pool.d.ts +2 -1
  2. package/autoscaling/autoscaled_pool.js +9 -2
  3. package/autoscaling/client_load_signal.d.ts +25 -0
  4. package/autoscaling/client_load_signal.js +36 -0
  5. package/autoscaling/cpu_load_signal.d.ts +28 -0
  6. package/autoscaling/cpu_load_signal.js +24 -0
  7. package/autoscaling/event_loop_load_signal.d.ts +23 -0
  8. package/autoscaling/event_loop_load_signal.js +35 -0
  9. package/autoscaling/index.d.ts +5 -1
  10. package/autoscaling/index.js +5 -1
  11. package/autoscaling/load_signal.d.ts +99 -0
  12. package/autoscaling/load_signal.js +135 -0
  13. package/autoscaling/memory_load_signal.d.ts +43 -0
  14. package/autoscaling/memory_load_signal.js +101 -0
  15. package/autoscaling/snapshotter.d.ts +36 -63
  16. package/autoscaling/snapshotter.js +83 -175
  17. package/autoscaling/system_status.d.ts +21 -33
  18. package/autoscaling/system_status.js +45 -82
  19. package/configuration.d.ts +0 -1
  20. package/configuration.js +0 -1
  21. package/cookie_utils.d.ts +0 -1
  22. package/cookie_utils.js +2 -1
  23. package/crawlers/context_pipeline.d.ts +0 -1
  24. package/crawlers/context_pipeline.js +0 -1
  25. package/crawlers/crawler_commons.d.ts +0 -1
  26. package/crawlers/crawler_commons.js +0 -1
  27. package/crawlers/crawler_utils.d.ts +0 -1
  28. package/crawlers/crawler_utils.js +0 -1
  29. package/crawlers/error_snapshotter.d.ts +0 -1
  30. package/crawlers/error_snapshotter.js +0 -1
  31. package/crawlers/error_tracker.d.ts +0 -1
  32. package/crawlers/error_tracker.js +0 -1
  33. package/crawlers/index.d.ts +0 -1
  34. package/crawlers/index.js +0 -1
  35. package/crawlers/internals/types.d.ts +0 -1
  36. package/crawlers/internals/types.js +0 -1
  37. package/crawlers/statistics.d.ts +6 -1
  38. package/crawlers/statistics.js +8 -1
  39. package/enqueue_links/enqueue_links.d.ts +8 -1
  40. package/enqueue_links/enqueue_links.js +9 -9
  41. package/enqueue_links/index.d.ts +0 -1
  42. package/enqueue_links/index.js +0 -1
  43. package/enqueue_links/shared.d.ts +0 -1
  44. package/enqueue_links/shared.js +0 -1
  45. package/errors.d.ts +20 -1
  46. package/errors.js +22 -1
  47. package/events/event_manager.d.ts +0 -1
  48. package/events/event_manager.js +0 -1
  49. package/events/index.d.ts +0 -1
  50. package/events/index.js +0 -1
  51. package/events/local_event_manager.d.ts +1 -1
  52. package/events/local_event_manager.js +9 -6
  53. package/index.d.ts +0 -1
  54. package/index.js +0 -1
  55. package/log.d.ts +0 -1
  56. package/log.js +0 -1
  57. package/memory-storage/consts.d.ts +0 -1
  58. package/memory-storage/consts.js +0 -1
  59. package/memory-storage/index.d.ts +0 -1
  60. package/memory-storage/index.js +0 -1
  61. package/memory-storage/memory-storage.d.ts +0 -1
  62. package/memory-storage/memory-storage.js +0 -1
  63. package/memory-storage/resource-clients/common/base-client.d.ts +0 -1
  64. package/memory-storage/resource-clients/common/base-client.js +0 -1
  65. package/memory-storage/resource-clients/dataset.d.ts +0 -1
  66. package/memory-storage/resource-clients/dataset.js +0 -1
  67. package/memory-storage/resource-clients/key-value-store.d.ts +0 -1
  68. package/memory-storage/resource-clients/key-value-store.js +0 -1
  69. package/memory-storage/resource-clients/request-queue.d.ts +0 -1
  70. package/memory-storage/resource-clients/request-queue.js +0 -1
  71. package/memory-storage/utils.d.ts +6 -1
  72. package/memory-storage/utils.js +15 -1
  73. package/package.json +5 -5
  74. package/proxy_configuration.d.ts +0 -1
  75. package/proxy_configuration.js +0 -1
  76. package/recoverable_state.d.ts +0 -1
  77. package/recoverable_state.js +0 -1
  78. package/request.d.ts +0 -1
  79. package/request.js +0 -1
  80. package/router.d.ts +117 -15
  81. package/router.js +124 -30
  82. package/serialization.d.ts +0 -1
  83. package/serialization.js +0 -1
  84. package/service_locator.d.ts +0 -1
  85. package/service_locator.js +0 -1
  86. package/session_pool/consts.d.ts +0 -1
  87. package/session_pool/consts.js +0 -1
  88. package/session_pool/errors.d.ts +0 -1
  89. package/session_pool/errors.js +0 -1
  90. package/session_pool/fingerprint.d.ts +0 -1
  91. package/session_pool/fingerprint.js +0 -1
  92. package/session_pool/index.d.ts +0 -1
  93. package/session_pool/index.js +0 -1
  94. package/session_pool/session.d.ts +0 -1
  95. package/session_pool/session.js +0 -1
  96. package/session_pool/session_pool.d.ts +0 -1
  97. package/session_pool/session_pool.js +0 -1
  98. package/storages/access_checking.d.ts +0 -1
  99. package/storages/access_checking.js +0 -1
  100. package/storages/dataset.d.ts +1 -2
  101. package/storages/dataset.js +0 -1
  102. package/storages/index.d.ts +0 -1
  103. package/storages/index.js +0 -1
  104. package/storages/key_value_store.d.ts +0 -1
  105. package/storages/key_value_store.js +0 -1
  106. package/storages/key_value_store_codec.d.ts +0 -1
  107. package/storages/key_value_store_codec.js +0 -1
  108. package/storages/request_dedup_cache.d.ts +23 -0
  109. package/storages/request_dedup_cache.js +48 -0
  110. package/storages/request_list.d.ts +6 -1
  111. package/storages/request_list.js +16 -3
  112. package/storages/request_loader.d.ts +0 -1
  113. package/storages/request_loader.js +0 -1
  114. package/storages/request_manager.d.ts +0 -1
  115. package/storages/request_manager.js +0 -1
  116. package/storages/request_manager_tandem.d.ts +0 -1
  117. package/storages/request_manager_tandem.js +0 -1
  118. package/storages/request_queue.d.ts +24 -1
  119. package/storages/request_queue.js +80 -27
  120. package/storages/sitemap_request_loader.d.ts +0 -1
  121. package/storages/sitemap_request_loader.js +0 -1
  122. package/storages/storage_instance_manager.d.ts +0 -1
  123. package/storages/storage_instance_manager.js +0 -1
  124. package/storages/storage_stats.d.ts +0 -1
  125. package/storages/storage_stats.js +0 -1
  126. package/storages/utils.d.ts +0 -1
  127. package/storages/utils.js +0 -1
  128. package/typedefs.d.ts +0 -1
  129. package/typedefs.js +0 -1
  130. package/validators.d.ts +0 -1
  131. package/validators.js +0 -1
  132. package/autoscaling/autoscaled_pool.d.ts.map +0 -1
  133. package/autoscaling/autoscaled_pool.js.map +0 -1
  134. package/autoscaling/index.d.ts.map +0 -1
  135. package/autoscaling/index.js.map +0 -1
  136. package/autoscaling/snapshotter.d.ts.map +0 -1
  137. package/autoscaling/snapshotter.js.map +0 -1
  138. package/autoscaling/system_status.d.ts.map +0 -1
  139. package/autoscaling/system_status.js.map +0 -1
  140. package/configuration.d.ts.map +0 -1
  141. package/configuration.js.map +0 -1
  142. package/cookie_utils.d.ts.map +0 -1
  143. package/cookie_utils.js.map +0 -1
  144. package/crawlers/context_pipeline.d.ts.map +0 -1
  145. package/crawlers/context_pipeline.js.map +0 -1
  146. package/crawlers/crawler_commons.d.ts.map +0 -1
  147. package/crawlers/crawler_commons.js.map +0 -1
  148. package/crawlers/crawler_utils.d.ts.map +0 -1
  149. package/crawlers/crawler_utils.js.map +0 -1
  150. package/crawlers/error_snapshotter.d.ts.map +0 -1
  151. package/crawlers/error_snapshotter.js.map +0 -1
  152. package/crawlers/error_tracker.d.ts.map +0 -1
  153. package/crawlers/error_tracker.js.map +0 -1
  154. package/crawlers/index.d.ts.map +0 -1
  155. package/crawlers/index.js.map +0 -1
  156. package/crawlers/internals/types.d.ts.map +0 -1
  157. package/crawlers/internals/types.js.map +0 -1
  158. package/crawlers/statistics.d.ts.map +0 -1
  159. package/crawlers/statistics.js.map +0 -1
  160. package/enqueue_links/enqueue_links.d.ts.map +0 -1
  161. package/enqueue_links/enqueue_links.js.map +0 -1
  162. package/enqueue_links/index.d.ts.map +0 -1
  163. package/enqueue_links/index.js.map +0 -1
  164. package/enqueue_links/shared.d.ts.map +0 -1
  165. package/enqueue_links/shared.js.map +0 -1
  166. package/errors.d.ts.map +0 -1
  167. package/errors.js.map +0 -1
  168. package/events/event_manager.d.ts.map +0 -1
  169. package/events/event_manager.js.map +0 -1
  170. package/events/index.d.ts.map +0 -1
  171. package/events/index.js.map +0 -1
  172. package/events/local_event_manager.d.ts.map +0 -1
  173. package/events/local_event_manager.js.map +0 -1
  174. package/index.d.ts.map +0 -1
  175. package/index.js.map +0 -1
  176. package/log.d.ts.map +0 -1
  177. package/log.js.map +0 -1
  178. package/memory-storage/consts.d.ts.map +0 -1
  179. package/memory-storage/consts.js.map +0 -1
  180. package/memory-storage/index.d.ts.map +0 -1
  181. package/memory-storage/index.js.map +0 -1
  182. package/memory-storage/memory-storage.d.ts.map +0 -1
  183. package/memory-storage/memory-storage.js.map +0 -1
  184. package/memory-storage/resource-clients/common/base-client.d.ts.map +0 -1
  185. package/memory-storage/resource-clients/common/base-client.js.map +0 -1
  186. package/memory-storage/resource-clients/dataset.d.ts.map +0 -1
  187. package/memory-storage/resource-clients/dataset.js.map +0 -1
  188. package/memory-storage/resource-clients/key-value-store.d.ts.map +0 -1
  189. package/memory-storage/resource-clients/key-value-store.js.map +0 -1
  190. package/memory-storage/resource-clients/request-queue.d.ts.map +0 -1
  191. package/memory-storage/resource-clients/request-queue.js.map +0 -1
  192. package/memory-storage/utils.d.ts.map +0 -1
  193. package/memory-storage/utils.js.map +0 -1
  194. package/proxy_configuration.d.ts.map +0 -1
  195. package/proxy_configuration.js.map +0 -1
  196. package/recoverable_state.d.ts.map +0 -1
  197. package/recoverable_state.js.map +0 -1
  198. package/request.d.ts.map +0 -1
  199. package/request.js.map +0 -1
  200. package/router.d.ts.map +0 -1
  201. package/router.js.map +0 -1
  202. package/serialization.d.ts.map +0 -1
  203. package/serialization.js.map +0 -1
  204. package/service_locator.d.ts.map +0 -1
  205. package/service_locator.js.map +0 -1
  206. package/session_pool/consts.d.ts.map +0 -1
  207. package/session_pool/consts.js.map +0 -1
  208. package/session_pool/errors.d.ts.map +0 -1
  209. package/session_pool/errors.js.map +0 -1
  210. package/session_pool/fingerprint.d.ts.map +0 -1
  211. package/session_pool/fingerprint.js.map +0 -1
  212. package/session_pool/index.d.ts.map +0 -1
  213. package/session_pool/index.js.map +0 -1
  214. package/session_pool/session.d.ts.map +0 -1
  215. package/session_pool/session.js.map +0 -1
  216. package/session_pool/session_pool.d.ts.map +0 -1
  217. package/session_pool/session_pool.js.map +0 -1
  218. package/storages/access_checking.d.ts.map +0 -1
  219. package/storages/access_checking.js.map +0 -1
  220. package/storages/dataset.d.ts.map +0 -1
  221. package/storages/dataset.js.map +0 -1
  222. package/storages/index.d.ts.map +0 -1
  223. package/storages/index.js.map +0 -1
  224. package/storages/key_value_store.d.ts.map +0 -1
  225. package/storages/key_value_store.js.map +0 -1
  226. package/storages/key_value_store_codec.d.ts.map +0 -1
  227. package/storages/key_value_store_codec.js.map +0 -1
  228. package/storages/request_list.d.ts.map +0 -1
  229. package/storages/request_list.js.map +0 -1
  230. package/storages/request_loader.d.ts.map +0 -1
  231. package/storages/request_loader.js.map +0 -1
  232. package/storages/request_manager.d.ts.map +0 -1
  233. package/storages/request_manager.js.map +0 -1
  234. package/storages/request_manager_tandem.d.ts.map +0 -1
  235. package/storages/request_manager_tandem.js.map +0 -1
  236. package/storages/request_queue.d.ts.map +0 -1
  237. package/storages/request_queue.js.map +0 -1
  238. package/storages/sitemap_request_loader.d.ts.map +0 -1
  239. package/storages/sitemap_request_loader.js.map +0 -1
  240. package/storages/storage_instance_manager.d.ts.map +0 -1
  241. package/storages/storage_instance_manager.js.map +0 -1
  242. package/storages/storage_stats.d.ts.map +0 -1
  243. package/storages/storage_stats.js.map +0 -1
  244. package/storages/utils.d.ts.map +0 -1
  245. package/storages/utils.js.map +0 -1
  246. package/typedefs.d.ts.map +0 -1
  247. package/typedefs.js.map +0 -1
  248. package/validators.d.ts.map +0 -1
  249. package/validators.js.map +0 -1
@@ -170,6 +170,8 @@ export declare class AutoscaledPool {
170
170
  private resolve;
171
171
  private reject;
172
172
  private snapshotter;
173
+ /** Additional SystemStatus loadSignals - tracked here for initialization and cleanup */
174
+ private loadSignals;
173
175
  private systemStatus;
174
176
  private autoscaleInterval;
175
177
  private maybeRunInterval;
@@ -295,4 +297,3 @@ export declare class AutoscaledPool {
295
297
  protected _incrementTasksDonePerSecond(intervalCallback: () => void): void;
296
298
  protected get _isOverMaxRequestLimit(): boolean;
297
299
  }
298
- //# sourceMappingURL=autoscaled_pool.d.ts.map
@@ -77,6 +77,8 @@ export class AutoscaledPool {
77
77
  resolve = null;
78
78
  reject = null;
79
79
  snapshotter;
80
+ /** Additional SystemStatus loadSignals - tracked here for initialization and cleanup */
81
+ loadSignals;
80
82
  systemStatus;
81
83
  autoscaleInterval;
82
84
  maybeRunInterval;
@@ -136,6 +138,7 @@ export class AutoscaledPool {
136
138
  log: this.log,
137
139
  });
138
140
  this.snapshotter = ssoCopy.snapshotter;
141
+ this.loadSignals = ssoCopy.loadSignals ?? [];
139
142
  this.systemStatus = new SystemStatus(ssoCopy);
140
143
  }
141
144
  /**
@@ -198,6 +201,7 @@ export class AutoscaledPool {
198
201
  this.reject = reject;
199
202
  });
200
203
  await this.snapshotter.start();
204
+ await Promise.all(this.loadSignals.map((s) => s.start()));
201
205
  // This interval checks the system status and updates the desired concurrency accordingly.
202
206
  this.autoscaleInterval = betterSetInterval(this._autoscale, this.autoscaleIntervalMillis);
203
207
  // This is here because if we scale down to let's say 1, then after each promise is finished
@@ -252,14 +256,17 @@ export class AutoscaledPool {
252
256
  this.isStopped = true;
253
257
  await new Promise((resolve, reject) => {
254
258
  let timeout;
259
+ let interval;
255
260
  if (timeoutSecs) {
256
261
  timeout = setTimeout(() => {
262
+ // Clean up the polling interval to prevent it from leaking on timeout.
263
+ clearInterval(interval);
257
264
  const err = new Error("The pool's running tasks did not finish" +
258
265
  `in ${timeoutSecs} secs after pool.pause() invocation.`);
259
266
  reject(err);
260
267
  }, timeoutSecs);
261
268
  }
262
- const interval = setInterval(() => {
269
+ interval = setInterval(() => {
263
270
  if (this._currentConcurrency <= 0) {
264
271
  // Clean up timeout and interval to prevent process hanging.
265
272
  if (timeout)
@@ -511,6 +518,7 @@ export class AutoscaledPool {
511
518
  betterClearInterval(this.tasksDonePerSecondInterval);
512
519
  if (this.snapshotter)
513
520
  await this.snapshotter.stop();
521
+ await Promise.all(this.loadSignals.map((s) => s.stop()));
514
522
  }
515
523
  _incrementTasksDonePerSecond(intervalCallback) {
516
524
  this._tasksPerMinute.unshift(0);
@@ -524,4 +532,3 @@ export class AutoscaledPool {
524
532
  return this._tasksPerMinute.reduce((acc, curr) => acc + curr, 0) >= this.maxTasksPerMinute;
525
533
  }
526
534
  }
527
- //# sourceMappingURL=autoscaled_pool.js.map
@@ -0,0 +1,25 @@
1
+ import type { StorageBackend } from '@crawlee/types';
2
+ import type { LoadSnapshot } from './load_signal.js';
3
+ import { SnapshotStore } from './load_signal.js';
4
+ export interface ClientSnapshot extends LoadSnapshot {
5
+ rateLimitErrorCount: number;
6
+ }
7
+ export interface ClientLoadSignalOptions {
8
+ client: StorageBackend;
9
+ clientSnapshotIntervalSecs?: number;
10
+ maxClientErrors?: number;
11
+ overloadedRatio?: number;
12
+ snapshotHistoryMillis?: number;
13
+ }
14
+ /**
15
+ * Periodically checks the storage client for rate-limit errors (HTTP 429)
16
+ * and reports overload when the error delta exceeds a threshold.
17
+ */
18
+ // @ts-ignore optional peer dependency or compatibility with es2022
19
+ export declare function createClientLoadSignal(options: ClientLoadSignalOptions): Omit<import("./load_signal.js").LoadSignal, "getSample"> & {
20
+ store: SnapshotStore<ClientSnapshot>;
21
+ handle: (cb: () => unknown) => void;
22
+ getSample(sampleDurationMillis?: number): ClientSnapshot[];
23
+ };
24
+ /** @internal Return type for backward compat in Snapshotter facade */
25
+ export type ClientLoadSignal = ReturnType<typeof createClientLoadSignal>;
@@ -0,0 +1,36 @@
1
+ import { SnapshotStore } from './load_signal.js';
2
+ const CLIENT_RATE_LIMIT_ERROR_RETRY_COUNT = 2;
3
+ /**
4
+ * Periodically checks the storage client for rate-limit errors (HTTP 429)
5
+ * and reports overload when the error delta exceeds a threshold.
6
+ */
7
+ export function createClientLoadSignal(options) {
8
+ const maxClientErrors = options.maxClientErrors ?? 3;
9
+ const signal = SnapshotStore.fromInterval({
10
+ name: 'clientInfo',
11
+ overloadedRatio: options.overloadedRatio ?? 0.3,
12
+ intervalMillis: (options.clientSnapshotIntervalSecs ?? 1) * 1000,
13
+ snapshotHistoryMillis: options.snapshotHistoryMillis,
14
+ handler(store, intervalCallback) {
15
+ const now = new Date();
16
+ const allErrorCounts = options.client.stats?.rateLimitErrors ?? [];
17
+ const currentErrCount = allErrorCounts[CLIENT_RATE_LIMIT_ERROR_RETRY_COUNT] || 0;
18
+ const snapshot = {
19
+ createdAt: now,
20
+ isOverloaded: false,
21
+ rateLimitErrorCount: currentErrCount,
22
+ };
23
+ const all = store.getAll();
24
+ const previousSnapshot = all[all.length - 1];
25
+ if (previousSnapshot) {
26
+ const { rateLimitErrorCount } = previousSnapshot;
27
+ const delta = currentErrCount - rateLimitErrorCount;
28
+ if (delta > maxClientErrors)
29
+ snapshot.isOverloaded = true;
30
+ }
31
+ store.push(snapshot, now);
32
+ intervalCallback();
33
+ },
34
+ });
35
+ return signal;
36
+ }
@@ -0,0 +1,28 @@
1
+ import type { Configuration } from '../configuration.js';
2
+ import type { LoadSnapshot } from './load_signal.js';
3
+ import { SnapshotStore } from './load_signal.js';
4
+ import type { SystemInfo } from './system_status.js';
5
+ export interface CpuSnapshot extends LoadSnapshot {
6
+ usedRatio: number;
7
+ ticks?: {
8
+ idle: number;
9
+ total: number;
10
+ };
11
+ }
12
+ export interface CpuLoadSignalOptions {
13
+ overloadedRatio?: number;
14
+ snapshotHistoryMillis?: number;
15
+ config: Configuration;
16
+ }
17
+ /**
18
+ * Tracks CPU usage via `SYSTEM_INFO` events and reports overload when
19
+ * the platform or local OS metrics indicate the CPU is overloaded.
20
+ */
21
+ // @ts-ignore optional peer dependency or compatibility with es2022
22
+ export declare function createCpuLoadSignal(options: CpuLoadSignalOptions): Omit<import("./load_signal.js").LoadSignal, "getSample"> & {
23
+ store: SnapshotStore<CpuSnapshot>;
24
+ handle: (payload: SystemInfo) => void;
25
+ getSample(sampleDurationMillis?: number): CpuSnapshot[];
26
+ };
27
+ /** @internal Return type for backward compat in Snapshotter facade */
28
+ export type CpuLoadSignal = ReturnType<typeof createCpuLoadSignal>;
@@ -0,0 +1,24 @@
1
+ import { serviceLocator } from '../service_locator.js';
2
+ import { SnapshotStore } from './load_signal.js';
3
+ /**
4
+ * Tracks CPU usage via `SYSTEM_INFO` events and reports overload when
5
+ * the platform or local OS metrics indicate the CPU is overloaded.
6
+ */
7
+ export function createCpuLoadSignal(options) {
8
+ return SnapshotStore.fromEvent({
9
+ name: 'cpuInfo',
10
+ overloadedRatio: options.overloadedRatio ?? 0.4,
11
+ events: serviceLocator.getEventManager(),
12
+ event: "systemInfo" /* EventType.SYSTEM_INFO */,
13
+ snapshotHistoryMillis: options.snapshotHistoryMillis,
14
+ handler(store, systemInfo) {
15
+ const { cpuCurrentUsage, isCpuOverloaded } = systemInfo;
16
+ const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
17
+ store.push({
18
+ createdAt,
19
+ isOverloaded: isCpuOverloaded,
20
+ usedRatio: Math.ceil(cpuCurrentUsage / 100),
21
+ }, createdAt);
22
+ },
23
+ });
24
+ }
@@ -0,0 +1,23 @@
1
+ import type { LoadSnapshot } from './load_signal.js';
2
+ import { SnapshotStore } from './load_signal.js';
3
+ export interface EventLoopSnapshot extends LoadSnapshot {
4
+ exceededMillis: number;
5
+ }
6
+ export interface EventLoopLoadSignalOptions {
7
+ eventLoopSnapshotIntervalSecs?: number;
8
+ maxBlockedMillis?: number;
9
+ overloadedRatio?: number;
10
+ snapshotHistoryMillis?: number;
11
+ }
12
+ /**
13
+ * Periodically measures event loop delay and reports overload when the
14
+ * delay exceeds a configured threshold.
15
+ */
16
+ // @ts-ignore optional peer dependency or compatibility with es2022
17
+ export declare function createEventLoopLoadSignal(options?: EventLoopLoadSignalOptions): Omit<import("./load_signal.js").LoadSignal, "getSample"> & {
18
+ store: SnapshotStore<EventLoopSnapshot>;
19
+ handle: (cb: () => unknown) => void;
20
+ getSample(sampleDurationMillis?: number): EventLoopSnapshot[];
21
+ };
22
+ /** @internal Return type for backward compat in Snapshotter facade */
23
+ export type EventLoopLoadSignal = ReturnType<typeof createEventLoopLoadSignal>;
@@ -0,0 +1,35 @@
1
+ import { SnapshotStore } from './load_signal.js';
2
+ /**
3
+ * Periodically measures event loop delay and reports overload when the
4
+ * delay exceeds a configured threshold.
5
+ */
6
+ export function createEventLoopLoadSignal(options = {}) {
7
+ const intervalMillis = (options.eventLoopSnapshotIntervalSecs ?? 0.5) * 1000;
8
+ const maxBlockedMillis = options.maxBlockedMillis ?? 50;
9
+ const signal = SnapshotStore.fromInterval({
10
+ name: 'eventLoopInfo',
11
+ overloadedRatio: options.overloadedRatio ?? 0.6,
12
+ intervalMillis,
13
+ snapshotHistoryMillis: options.snapshotHistoryMillis,
14
+ handler(store, intervalCallback) {
15
+ const now = new Date();
16
+ const snapshot = {
17
+ createdAt: now,
18
+ isOverloaded: false,
19
+ exceededMillis: 0,
20
+ };
21
+ const all = store.getAll();
22
+ const previousSnapshot = all[all.length - 1];
23
+ if (previousSnapshot) {
24
+ const { createdAt } = previousSnapshot;
25
+ const delta = now.getTime() - +createdAt - intervalMillis;
26
+ if (delta > maxBlockedMillis)
27
+ snapshot.isOverloaded = true;
28
+ snapshot.exceededMillis = Math.max(delta - maxBlockedMillis, 0);
29
+ }
30
+ store.push(snapshot, now);
31
+ intervalCallback();
32
+ },
33
+ });
34
+ return signal;
35
+ }
@@ -1,4 +1,8 @@
1
1
  export * from './autoscaled_pool.js';
2
+ export * from './client_load_signal.js';
3
+ export * from './cpu_load_signal.js';
4
+ export * from './event_loop_load_signal.js';
5
+ export * from './load_signal.js';
6
+ export * from './memory_load_signal.js';
2
7
  export * from './snapshotter.js';
3
8
  export * from './system_status.js';
4
- //# sourceMappingURL=index.d.ts.map
@@ -1,4 +1,8 @@
1
1
  export * from './autoscaled_pool.js';
2
+ export * from './client_load_signal.js';
3
+ export * from './cpu_load_signal.js';
4
+ export * from './event_loop_load_signal.js';
5
+ export * from './load_signal.js';
6
+ export * from './memory_load_signal.js';
2
7
  export * from './snapshotter.js';
3
8
  export * from './system_status.js';
4
- //# sourceMappingURL=index.js.map
@@ -0,0 +1,99 @@
1
+ import type { EventManager, EventTypeName } from '../events/event_manager.js';
2
+ import type { ClientInfo } from './system_status.js';
3
+ /**
4
+ * A snapshot of a resource's overload state at a point in time.
5
+ */
6
+ export interface LoadSnapshot {
7
+ createdAt: Date;
8
+ isOverloaded: boolean;
9
+ }
10
+ /**
11
+ * A signal that reports whether a particular resource is overloaded.
12
+ *
13
+ * `SystemStatus` aggregates multiple `LoadSignal` instances to determine
14
+ * overall system health. The built-in signals cover memory, CPU, event loop,
15
+ * and API client rate limits. You can implement this interface to add
16
+ * custom overload signals (e.g. navigation timeouts, proxy health).
17
+ */
18
+ export interface LoadSignal {
19
+ /** Human-readable name used in logging and `SystemInfo` keys. */
20
+ readonly name: string;
21
+ /**
22
+ * Maximum ratio of overloaded snapshots in a sample before the signal
23
+ * is considered overloaded. For example, `0.2` means the signal fires
24
+ * when more than 20% of the sample window is overloaded.
25
+ */
26
+ readonly overloadedRatio: number;
27
+ /** Start collecting snapshots. Called when the pool starts. */
28
+ start(): Promise<void>;
29
+ /** Stop collecting snapshots. Called when the pool shuts down. */
30
+ stop(): Promise<void>;
31
+ /**
32
+ * Return snapshots for a recent time window (used for "current" status).
33
+ * @param sampleDurationMillis How far back to look, in milliseconds.
34
+ */
35
+ getSample(sampleDurationMillis?: number): LoadSnapshot[];
36
+ }
37
+ /**
38
+ * A time-pruning, time-windowed store for `LoadSnapshot` values.
39
+ * Signals compose with this instead of inheriting from a base class.
40
+ */
41
+ export declare class SnapshotStore<T extends LoadSnapshot = LoadSnapshot> {
42
+ private snapshots;
43
+ private readonly historyMillis;
44
+ constructor(historyMillis?: number);
45
+ /**
46
+ * Add a snapshot and prune entries older than the history window.
47
+ */
48
+ push(snapshot: T, now?: Date): void;
49
+ /**
50
+ * Return all snapshots, or only those within the given time window.
51
+ */
52
+ getSample(sampleDurationMillis?: number): T[];
53
+ /**
54
+ * Direct access to the underlying array (for backward-compat getters).
55
+ */
56
+ getAll(): T[];
57
+ /**
58
+ * Create a `LoadSignal` that snapshots on a `betterSetInterval` tick.
59
+ *
60
+ * The `handler` receives the store (to read previous snapshots) and the
61
+ * interval callback (which it **must** call when done). It should call
62
+ * `store.push()` to record a snapshot.
63
+ */
64
+ static fromInterval<T extends LoadSnapshot>(options: {
65
+ name: string;
66
+ overloadedRatio: number;
67
+ intervalMillis: number;
68
+ snapshotHistoryMillis?: number;
69
+ handler: (store: SnapshotStore<T>, intervalCallback: () => unknown) => void;
70
+ }): Omit<LoadSignal, 'getSample'> & {
71
+ store: SnapshotStore<T>;
72
+ handle: (cb: () => unknown) => void;
73
+ getSample(sampleDurationMillis?: number): T[];
74
+ };
75
+ /**
76
+ * Create a `LoadSignal` that snapshots in response to an `EventManager` event.
77
+ *
78
+ * The `handler` receives the event payload and the store. It should call
79
+ * `store.push()` to record a snapshot.
80
+ */
81
+ static fromEvent<T extends LoadSnapshot, E>(options: {
82
+ name: string;
83
+ overloadedRatio: number;
84
+ events: EventManager;
85
+ event: EventTypeName;
86
+ snapshotHistoryMillis?: number;
87
+ handler: (store: SnapshotStore<T>, payload: E) => void;
88
+ }): Omit<LoadSignal, 'getSample'> & {
89
+ store: SnapshotStore<T>;
90
+ handle: (payload: E) => void;
91
+ getSample(sampleDurationMillis?: number): T[];
92
+ };
93
+ }
94
+ /**
95
+ * Evaluate whether a sample of `LoadSnapshot` values exceeds the given
96
+ * overloaded ratio, using a time-weighted average. This is the shared
97
+ * evaluation logic used by `SystemStatus` for all signal types.
98
+ */
99
+ export declare function evaluateLoadSignalSample(sample: LoadSnapshot[], overloadedRatio: number): ClientInfo;
@@ -0,0 +1,135 @@
1
+ import { weightedAvg } from '@crawlee/utils';
2
+ import { betterClearInterval, betterSetInterval } from '@apify/utilities';
3
+ /**
4
+ * A time-pruning, time-windowed store for `LoadSnapshot` values.
5
+ * Signals compose with this instead of inheriting from a base class.
6
+ */
7
+ export class SnapshotStore {
8
+ snapshots = [];
9
+ historyMillis;
10
+ constructor(historyMillis = 30_000) {
11
+ this.historyMillis = historyMillis;
12
+ }
13
+ /**
14
+ * Add a snapshot and prune entries older than the history window.
15
+ */
16
+ push(snapshot, now = snapshot.createdAt) {
17
+ // Inline pruning to avoid private-method transpilation issues
18
+ let oldCount = 0;
19
+ for (let i = 0; i < this.snapshots.length; i++) {
20
+ const { createdAt } = this.snapshots[i];
21
+ if (now.getTime() - new Date(createdAt).getTime() > this.historyMillis)
22
+ oldCount++;
23
+ else
24
+ break;
25
+ }
26
+ if (oldCount)
27
+ this.snapshots.splice(0, oldCount);
28
+ this.snapshots.push(snapshot);
29
+ }
30
+ /**
31
+ * Return all snapshots, or only those within the given time window.
32
+ */
33
+ getSample(sampleDurationMillis) {
34
+ if (!sampleDurationMillis)
35
+ return this.snapshots;
36
+ const sample = [];
37
+ let idx = this.snapshots.length;
38
+ if (!idx)
39
+ return sample;
40
+ const latestTime = this.snapshots[idx - 1].createdAt;
41
+ while (idx--) {
42
+ const snapshot = this.snapshots[idx];
43
+ if (+latestTime - +snapshot.createdAt <= sampleDurationMillis) {
44
+ sample.unshift(snapshot);
45
+ }
46
+ else {
47
+ break;
48
+ }
49
+ }
50
+ return sample;
51
+ }
52
+ /**
53
+ * Direct access to the underlying array (for backward-compat getters).
54
+ */
55
+ getAll() {
56
+ return this.snapshots;
57
+ }
58
+ /**
59
+ * Create a `LoadSignal` that snapshots on a `betterSetInterval` tick.
60
+ *
61
+ * The `handler` receives the store (to read previous snapshots) and the
62
+ * interval callback (which it **must** call when done). It should call
63
+ * `store.push()` to record a snapshot.
64
+ */
65
+ static fromInterval(options) {
66
+ const store = new SnapshotStore(options.snapshotHistoryMillis);
67
+ let interval = null;
68
+ const handle = (cb) => options.handler(store, cb);
69
+ return {
70
+ name: options.name,
71
+ overloadedRatio: options.overloadedRatio,
72
+ store,
73
+ handle,
74
+ getSample: (ms) => store.getSample(ms),
75
+ async start() {
76
+ interval = betterSetInterval(handle, options.intervalMillis);
77
+ },
78
+ async stop() {
79
+ betterClearInterval(interval);
80
+ },
81
+ };
82
+ }
83
+ /**
84
+ * Create a `LoadSignal` that snapshots in response to an `EventManager` event.
85
+ *
86
+ * The `handler` receives the event payload and the store. It should call
87
+ * `store.push()` to record a snapshot.
88
+ */
89
+ static fromEvent(options) {
90
+ const store = new SnapshotStore(options.snapshotHistoryMillis);
91
+ const handle = (payload) => options.handler(store, payload);
92
+ return {
93
+ name: options.name,
94
+ overloadedRatio: options.overloadedRatio,
95
+ store,
96
+ handle,
97
+ getSample: (ms) => store.getSample(ms),
98
+ async start() {
99
+ options.events.on(options.event, handle);
100
+ },
101
+ async stop() {
102
+ options.events.off(options.event, handle);
103
+ },
104
+ };
105
+ }
106
+ }
107
+ /**
108
+ * Evaluate whether a sample of `LoadSnapshot` values exceeds the given
109
+ * overloaded ratio, using a time-weighted average. This is the shared
110
+ * evaluation logic used by `SystemStatus` for all signal types.
111
+ */
112
+ export function evaluateLoadSignalSample(sample, overloadedRatio) {
113
+ if (sample.length === 0) {
114
+ return {
115
+ isOverloaded: false,
116
+ limitRatio: overloadedRatio,
117
+ actualRatio: 0,
118
+ };
119
+ }
120
+ const weights = [];
121
+ const values = [];
122
+ for (let i = 1; i < sample.length; i++) {
123
+ const previous = sample[i - 1];
124
+ const current = sample[i];
125
+ const weight = +current.createdAt - +previous.createdAt;
126
+ weights.push(weight || 1); // Prevent errors from 0ms long intervals (sync) between snapshots.
127
+ values.push(+current.isOverloaded);
128
+ }
129
+ const wAvg = sample.length === 1 ? +sample[0].isOverloaded : weightedAvg(values, weights);
130
+ return {
131
+ isOverloaded: wAvg > overloadedRatio,
132
+ limitRatio: overloadedRatio,
133
+ actualRatio: Math.round(wAvg * 1000) / 1000,
134
+ };
135
+ }
@@ -0,0 +1,43 @@
1
+ import type { Configuration } from '../configuration.js';
2
+ import type { CrawleeLogger } from '../log.js';
3
+ import type { LoadSignal, LoadSnapshot } from './load_signal.js';
4
+ import type { SystemInfo } from './system_status.js';
5
+ export interface MemorySnapshot extends LoadSnapshot {
6
+ usedBytes?: number;
7
+ }
8
+ export interface MemoryLoadSignalOptions {
9
+ maxUsedMemoryRatio?: number;
10
+ overloadedRatio?: number;
11
+ snapshotHistoryMillis?: number;
12
+ config: Configuration;
13
+ log?: CrawleeLogger;
14
+ }
15
+ /**
16
+ * Tracks memory usage via `SYSTEM_INFO` events and reports overload when
17
+ * the used-to-available memory ratio exceeds a threshold.
18
+ */
19
+ export declare class MemoryLoadSignal implements LoadSignal {
20
+ readonly name = "memInfo";
21
+ readonly overloadedRatio: number;
22
+ private readonly store;
23
+ private readonly config;
24
+ private readonly events;
25
+ private readonly log;
26
+ private readonly maxUsedMemoryRatio;
27
+ private maxMemoryRatio;
28
+ private maxMemoryBytes;
29
+ private lastLoggedCriticalMemoryOverloadAt;
30
+ constructor(options: MemoryLoadSignalOptions);
31
+ start(): Promise<void>;
32
+ stop(): Promise<void>;
33
+ getSample(sampleDurationMillis?: number): MemorySnapshot[];
34
+ /**
35
+ * Returns typed memory snapshots for backward compatibility with `Snapshotter`.
36
+ */
37
+ getMemorySnapshots(): MemorySnapshot[];
38
+ /** @internal */
39
+ _onSystemInfo(systemInfo: SystemInfo): void;
40
+ /** @internal */
41
+ _memoryOverloadWarning(systemInfo: SystemInfo, maxMemoryBytes?: number): void;
42
+ private _getTotalMemoryBytes;
43
+ }
@@ -0,0 +1,101 @@
1
+ import { getMemoryInfo, isContainerized } from '@crawlee/utils';
2
+ import { serviceLocator } from '../service_locator.js';
3
+ import { SnapshotStore } from './load_signal.js';
4
+ const RESERVE_MEMORY_RATIO = 0.5;
5
+ const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10_000;
6
+ /**
7
+ * Tracks memory usage via `SYSTEM_INFO` events and reports overload when
8
+ * the used-to-available memory ratio exceeds a threshold.
9
+ */
10
+ export class MemoryLoadSignal {
11
+ name = 'memInfo';
12
+ overloadedRatio;
13
+ store;
14
+ config;
15
+ events;
16
+ log;
17
+ maxUsedMemoryRatio;
18
+ maxMemoryRatio;
19
+ maxMemoryBytes;
20
+ lastLoggedCriticalMemoryOverloadAt = null;
21
+ constructor(options) {
22
+ this.store = new SnapshotStore(options.snapshotHistoryMillis);
23
+ this.config = options.config;
24
+ this.events = serviceLocator.getEventManager();
25
+ this.log = options.log ?? serviceLocator.getLogger().child({ prefix: 'MemoryLoadSignal' });
26
+ this.maxUsedMemoryRatio = options.maxUsedMemoryRatio ?? 0.9;
27
+ this.overloadedRatio = options.overloadedRatio ?? 0.2;
28
+ this._onSystemInfo = this._onSystemInfo.bind(this);
29
+ }
30
+ async start() {
31
+ const memoryMbytes = this.config.memoryMbytes ?? 0;
32
+ if (memoryMbytes > 0) {
33
+ this.maxMemoryBytes = memoryMbytes * 1024 * 1024;
34
+ }
35
+ else {
36
+ this.maxMemoryRatio = this.config.availableMemoryRatio;
37
+ if (!this.maxMemoryRatio) {
38
+ throw new Error('availableMemoryRatio is not set in configuration.');
39
+ }
40
+ else {
41
+ this.log.debug(`Setting max memory of this run to ${this.maxMemoryRatio * 100} % of available memory. ` +
42
+ 'Use the CRAWLEE_MEMORY_MBYTES or CRAWLEE_AVAILABLE_MEMORY_RATIO environment variable to override it.');
43
+ }
44
+ // Fallback memory measurement in case memTotalBytes is missing from SystemInfo.
45
+ this.maxMemoryBytes = await this._getTotalMemoryBytes();
46
+ }
47
+ this.events.on("systemInfo" /* EventType.SYSTEM_INFO */, this._onSystemInfo);
48
+ }
49
+ async stop() {
50
+ this.events.off("systemInfo" /* EventType.SYSTEM_INFO */, this._onSystemInfo);
51
+ }
52
+ getSample(sampleDurationMillis) {
53
+ return this.store.getSample(sampleDurationMillis);
54
+ }
55
+ /**
56
+ * Returns typed memory snapshots for backward compatibility with `Snapshotter`.
57
+ */
58
+ getMemorySnapshots() {
59
+ return this.store.getAll();
60
+ }
61
+ /** @internal */
62
+ _onSystemInfo(systemInfo) {
63
+ const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
64
+ const { memCurrentBytes, memTotalBytes } = systemInfo;
65
+ let maxMemoryBytes = this.maxMemoryBytes;
66
+ if (this.maxMemoryRatio !== undefined && this.maxMemoryRatio > 0) {
67
+ maxMemoryBytes = this.maxMemoryRatio * (memTotalBytes ?? this.maxMemoryBytes);
68
+ }
69
+ const snapshot = {
70
+ createdAt,
71
+ isOverloaded: memCurrentBytes / maxMemoryBytes > this.maxUsedMemoryRatio,
72
+ usedBytes: memCurrentBytes,
73
+ };
74
+ this.store.push(snapshot, createdAt);
75
+ this._memoryOverloadWarning(systemInfo, maxMemoryBytes);
76
+ }
77
+ /** @internal */
78
+ _memoryOverloadWarning(systemInfo, maxMemoryBytes) {
79
+ const effectiveMax = maxMemoryBytes ?? this.maxMemoryBytes;
80
+ const { memCurrentBytes } = systemInfo;
81
+ const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
82
+ if (this.lastLoggedCriticalMemoryOverloadAt &&
83
+ +createdAt < +this.lastLoggedCriticalMemoryOverloadAt + CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS)
84
+ return;
85
+ const maxDesiredMemoryBytes = this.maxUsedMemoryRatio * effectiveMax;
86
+ const reserveMemory = effectiveMax * (1 - this.maxUsedMemoryRatio) * RESERVE_MEMORY_RATIO;
87
+ const criticalOverloadBytes = maxDesiredMemoryBytes + reserveMemory;
88
+ const isCriticalOverload = memCurrentBytes > criticalOverloadBytes;
89
+ if (isCriticalOverload) {
90
+ const usedPercentage = Math.round((memCurrentBytes / effectiveMax) * 100);
91
+ const toMb = (bytes) => Math.round(bytes / 1024 ** 2);
92
+ this.log.warning('Memory is critically overloaded. ' +
93
+ `Using ${toMb(memCurrentBytes)} MB of ${toMb(effectiveMax)} MB (${usedPercentage}%). Consider increasing available memory.`);
94
+ this.lastLoggedCriticalMemoryOverloadAt = createdAt;
95
+ }
96
+ }
97
+ async _getTotalMemoryBytes() {
98
+ const containerized = this.config.containerized ?? (await isContainerized());
99
+ return (await getMemoryInfo({ containerized, logger: serviceLocator.getLogger() })).totalBytes;
100
+ }
101
+ }