@crawlee/core 4.0.0-beta.71 → 4.0.0-beta.73

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (249) hide show
  1. package/autoscaling/autoscaled_pool.d.ts +2 -1
  2. package/autoscaling/autoscaled_pool.js +9 -2
  3. package/autoscaling/client_load_signal.d.ts +25 -0
  4. package/autoscaling/client_load_signal.js +36 -0
  5. package/autoscaling/cpu_load_signal.d.ts +28 -0
  6. package/autoscaling/cpu_load_signal.js +24 -0
  7. package/autoscaling/event_loop_load_signal.d.ts +23 -0
  8. package/autoscaling/event_loop_load_signal.js +35 -0
  9. package/autoscaling/index.d.ts +5 -1
  10. package/autoscaling/index.js +5 -1
  11. package/autoscaling/load_signal.d.ts +99 -0
  12. package/autoscaling/load_signal.js +135 -0
  13. package/autoscaling/memory_load_signal.d.ts +43 -0
  14. package/autoscaling/memory_load_signal.js +101 -0
  15. package/autoscaling/snapshotter.d.ts +36 -63
  16. package/autoscaling/snapshotter.js +83 -175
  17. package/autoscaling/system_status.d.ts +21 -33
  18. package/autoscaling/system_status.js +45 -82
  19. package/configuration.d.ts +0 -1
  20. package/configuration.js +0 -1
  21. package/cookie_utils.d.ts +0 -1
  22. package/cookie_utils.js +2 -1
  23. package/crawlers/context_pipeline.d.ts +0 -1
  24. package/crawlers/context_pipeline.js +0 -1
  25. package/crawlers/crawler_commons.d.ts +0 -1
  26. package/crawlers/crawler_commons.js +0 -1
  27. package/crawlers/crawler_utils.d.ts +0 -1
  28. package/crawlers/crawler_utils.js +0 -1
  29. package/crawlers/error_snapshotter.d.ts +0 -1
  30. package/crawlers/error_snapshotter.js +0 -1
  31. package/crawlers/error_tracker.d.ts +0 -1
  32. package/crawlers/error_tracker.js +0 -1
  33. package/crawlers/index.d.ts +0 -1
  34. package/crawlers/index.js +0 -1
  35. package/crawlers/internals/types.d.ts +0 -1
  36. package/crawlers/internals/types.js +0 -1
  37. package/crawlers/statistics.d.ts +6 -1
  38. package/crawlers/statistics.js +8 -1
  39. package/enqueue_links/enqueue_links.d.ts +8 -1
  40. package/enqueue_links/enqueue_links.js +9 -9
  41. package/enqueue_links/index.d.ts +0 -1
  42. package/enqueue_links/index.js +0 -1
  43. package/enqueue_links/shared.d.ts +0 -1
  44. package/enqueue_links/shared.js +0 -1
  45. package/errors.d.ts +20 -1
  46. package/errors.js +22 -1
  47. package/events/event_manager.d.ts +0 -1
  48. package/events/event_manager.js +0 -1
  49. package/events/index.d.ts +0 -1
  50. package/events/index.js +0 -1
  51. package/events/local_event_manager.d.ts +1 -1
  52. package/events/local_event_manager.js +9 -6
  53. package/index.d.ts +0 -1
  54. package/index.js +0 -1
  55. package/log.d.ts +0 -1
  56. package/log.js +0 -1
  57. package/memory-storage/consts.d.ts +0 -1
  58. package/memory-storage/consts.js +0 -1
  59. package/memory-storage/index.d.ts +0 -1
  60. package/memory-storage/index.js +0 -1
  61. package/memory-storage/memory-storage.d.ts +0 -1
  62. package/memory-storage/memory-storage.js +0 -1
  63. package/memory-storage/resource-clients/common/base-client.d.ts +0 -1
  64. package/memory-storage/resource-clients/common/base-client.js +0 -1
  65. package/memory-storage/resource-clients/dataset.d.ts +0 -1
  66. package/memory-storage/resource-clients/dataset.js +0 -1
  67. package/memory-storage/resource-clients/key-value-store.d.ts +0 -1
  68. package/memory-storage/resource-clients/key-value-store.js +0 -1
  69. package/memory-storage/resource-clients/request-queue.d.ts +0 -1
  70. package/memory-storage/resource-clients/request-queue.js +0 -1
  71. package/memory-storage/utils.d.ts +6 -1
  72. package/memory-storage/utils.js +15 -1
  73. package/package.json +5 -5
  74. package/proxy_configuration.d.ts +0 -1
  75. package/proxy_configuration.js +0 -1
  76. package/recoverable_state.d.ts +0 -1
  77. package/recoverable_state.js +0 -1
  78. package/request.d.ts +0 -1
  79. package/request.js +0 -1
  80. package/router.d.ts +117 -15
  81. package/router.js +124 -30
  82. package/serialization.d.ts +0 -1
  83. package/serialization.js +0 -1
  84. package/service_locator.d.ts +0 -1
  85. package/service_locator.js +0 -1
  86. package/session_pool/consts.d.ts +0 -1
  87. package/session_pool/consts.js +0 -1
  88. package/session_pool/errors.d.ts +0 -1
  89. package/session_pool/errors.js +0 -1
  90. package/session_pool/fingerprint.d.ts +0 -1
  91. package/session_pool/fingerprint.js +0 -1
  92. package/session_pool/index.d.ts +0 -1
  93. package/session_pool/index.js +0 -1
  94. package/session_pool/session.d.ts +0 -1
  95. package/session_pool/session.js +0 -1
  96. package/session_pool/session_pool.d.ts +0 -1
  97. package/session_pool/session_pool.js +0 -1
  98. package/storages/access_checking.d.ts +0 -1
  99. package/storages/access_checking.js +0 -1
  100. package/storages/dataset.d.ts +1 -2
  101. package/storages/dataset.js +0 -1
  102. package/storages/index.d.ts +0 -1
  103. package/storages/index.js +0 -1
  104. package/storages/key_value_store.d.ts +0 -1
  105. package/storages/key_value_store.js +0 -1
  106. package/storages/key_value_store_codec.d.ts +0 -1
  107. package/storages/key_value_store_codec.js +0 -1
  108. package/storages/request_dedup_cache.d.ts +23 -0
  109. package/storages/request_dedup_cache.js +48 -0
  110. package/storages/request_list.d.ts +6 -1
  111. package/storages/request_list.js +17 -3
  112. package/storages/request_loader.d.ts +0 -1
  113. package/storages/request_loader.js +0 -1
  114. package/storages/request_manager.d.ts +0 -1
  115. package/storages/request_manager.js +0 -1
  116. package/storages/request_manager_tandem.d.ts +0 -1
  117. package/storages/request_manager_tandem.js +0 -1
  118. package/storages/request_queue.d.ts +24 -1
  119. package/storages/request_queue.js +80 -27
  120. package/storages/sitemap_request_loader.d.ts +0 -1
  121. package/storages/sitemap_request_loader.js +0 -1
  122. package/storages/storage_instance_manager.d.ts +0 -1
  123. package/storages/storage_instance_manager.js +0 -1
  124. package/storages/storage_stats.d.ts +0 -1
  125. package/storages/storage_stats.js +0 -1
  126. package/storages/utils.d.ts +0 -1
  127. package/storages/utils.js +0 -1
  128. package/typedefs.d.ts +0 -1
  129. package/typedefs.js +0 -1
  130. package/validators.d.ts +0 -1
  131. package/validators.js +0 -1
  132. package/autoscaling/autoscaled_pool.d.ts.map +0 -1
  133. package/autoscaling/autoscaled_pool.js.map +0 -1
  134. package/autoscaling/index.d.ts.map +0 -1
  135. package/autoscaling/index.js.map +0 -1
  136. package/autoscaling/snapshotter.d.ts.map +0 -1
  137. package/autoscaling/snapshotter.js.map +0 -1
  138. package/autoscaling/system_status.d.ts.map +0 -1
  139. package/autoscaling/system_status.js.map +0 -1
  140. package/configuration.d.ts.map +0 -1
  141. package/configuration.js.map +0 -1
  142. package/cookie_utils.d.ts.map +0 -1
  143. package/cookie_utils.js.map +0 -1
  144. package/crawlers/context_pipeline.d.ts.map +0 -1
  145. package/crawlers/context_pipeline.js.map +0 -1
  146. package/crawlers/crawler_commons.d.ts.map +0 -1
  147. package/crawlers/crawler_commons.js.map +0 -1
  148. package/crawlers/crawler_utils.d.ts.map +0 -1
  149. package/crawlers/crawler_utils.js.map +0 -1
  150. package/crawlers/error_snapshotter.d.ts.map +0 -1
  151. package/crawlers/error_snapshotter.js.map +0 -1
  152. package/crawlers/error_tracker.d.ts.map +0 -1
  153. package/crawlers/error_tracker.js.map +0 -1
  154. package/crawlers/index.d.ts.map +0 -1
  155. package/crawlers/index.js.map +0 -1
  156. package/crawlers/internals/types.d.ts.map +0 -1
  157. package/crawlers/internals/types.js.map +0 -1
  158. package/crawlers/statistics.d.ts.map +0 -1
  159. package/crawlers/statistics.js.map +0 -1
  160. package/enqueue_links/enqueue_links.d.ts.map +0 -1
  161. package/enqueue_links/enqueue_links.js.map +0 -1
  162. package/enqueue_links/index.d.ts.map +0 -1
  163. package/enqueue_links/index.js.map +0 -1
  164. package/enqueue_links/shared.d.ts.map +0 -1
  165. package/enqueue_links/shared.js.map +0 -1
  166. package/errors.d.ts.map +0 -1
  167. package/errors.js.map +0 -1
  168. package/events/event_manager.d.ts.map +0 -1
  169. package/events/event_manager.js.map +0 -1
  170. package/events/index.d.ts.map +0 -1
  171. package/events/index.js.map +0 -1
  172. package/events/local_event_manager.d.ts.map +0 -1
  173. package/events/local_event_manager.js.map +0 -1
  174. package/index.d.ts.map +0 -1
  175. package/index.js.map +0 -1
  176. package/log.d.ts.map +0 -1
  177. package/log.js.map +0 -1
  178. package/memory-storage/consts.d.ts.map +0 -1
  179. package/memory-storage/consts.js.map +0 -1
  180. package/memory-storage/index.d.ts.map +0 -1
  181. package/memory-storage/index.js.map +0 -1
  182. package/memory-storage/memory-storage.d.ts.map +0 -1
  183. package/memory-storage/memory-storage.js.map +0 -1
  184. package/memory-storage/resource-clients/common/base-client.d.ts.map +0 -1
  185. package/memory-storage/resource-clients/common/base-client.js.map +0 -1
  186. package/memory-storage/resource-clients/dataset.d.ts.map +0 -1
  187. package/memory-storage/resource-clients/dataset.js.map +0 -1
  188. package/memory-storage/resource-clients/key-value-store.d.ts.map +0 -1
  189. package/memory-storage/resource-clients/key-value-store.js.map +0 -1
  190. package/memory-storage/resource-clients/request-queue.d.ts.map +0 -1
  191. package/memory-storage/resource-clients/request-queue.js.map +0 -1
  192. package/memory-storage/utils.d.ts.map +0 -1
  193. package/memory-storage/utils.js.map +0 -1
  194. package/proxy_configuration.d.ts.map +0 -1
  195. package/proxy_configuration.js.map +0 -1
  196. package/recoverable_state.d.ts.map +0 -1
  197. package/recoverable_state.js.map +0 -1
  198. package/request.d.ts.map +0 -1
  199. package/request.js.map +0 -1
  200. package/router.d.ts.map +0 -1
  201. package/router.js.map +0 -1
  202. package/serialization.d.ts.map +0 -1
  203. package/serialization.js.map +0 -1
  204. package/service_locator.d.ts.map +0 -1
  205. package/service_locator.js.map +0 -1
  206. package/session_pool/consts.d.ts.map +0 -1
  207. package/session_pool/consts.js.map +0 -1
  208. package/session_pool/errors.d.ts.map +0 -1
  209. package/session_pool/errors.js.map +0 -1
  210. package/session_pool/fingerprint.d.ts.map +0 -1
  211. package/session_pool/fingerprint.js.map +0 -1
  212. package/session_pool/index.d.ts.map +0 -1
  213. package/session_pool/index.js.map +0 -1
  214. package/session_pool/session.d.ts.map +0 -1
  215. package/session_pool/session.js.map +0 -1
  216. package/session_pool/session_pool.d.ts.map +0 -1
  217. package/session_pool/session_pool.js.map +0 -1
  218. package/storages/access_checking.d.ts.map +0 -1
  219. package/storages/access_checking.js.map +0 -1
  220. package/storages/dataset.d.ts.map +0 -1
  221. package/storages/dataset.js.map +0 -1
  222. package/storages/index.d.ts.map +0 -1
  223. package/storages/index.js.map +0 -1
  224. package/storages/key_value_store.d.ts.map +0 -1
  225. package/storages/key_value_store.js.map +0 -1
  226. package/storages/key_value_store_codec.d.ts.map +0 -1
  227. package/storages/key_value_store_codec.js.map +0 -1
  228. package/storages/request_list.d.ts.map +0 -1
  229. package/storages/request_list.js.map +0 -1
  230. package/storages/request_loader.d.ts.map +0 -1
  231. package/storages/request_loader.js.map +0 -1
  232. package/storages/request_manager.d.ts.map +0 -1
  233. package/storages/request_manager.js.map +0 -1
  234. package/storages/request_manager_tandem.d.ts.map +0 -1
  235. package/storages/request_manager_tandem.js.map +0 -1
  236. package/storages/request_queue.d.ts.map +0 -1
  237. package/storages/request_queue.js.map +0 -1
  238. package/storages/sitemap_request_loader.d.ts.map +0 -1
  239. package/storages/sitemap_request_loader.js.map +0 -1
  240. package/storages/storage_instance_manager.d.ts.map +0 -1
  241. package/storages/storage_instance_manager.js.map +0 -1
  242. package/storages/storage_stats.d.ts.map +0 -1
  243. package/storages/storage_stats.js.map +0 -1
  244. package/storages/utils.d.ts.map +0 -1
  245. package/storages/utils.js.map +0 -1
  246. package/typedefs.d.ts.map +0 -1
  247. package/typedefs.js.map +0 -1
  248. package/validators.d.ts.map +0 -1
  249. package/validators.js.map +0 -1
@@ -1,5 +1,11 @@
1
- import type { BetterIntervalID } from '@apify/utilities';
1
+ import type { StorageBackend } from '@crawlee/types';
2
+ import type { Configuration } from '../configuration.js';
2
3
  import type { CrawleeLogger } from '../log.js';
4
+ import type { ClientSnapshot } from './client_load_signal.js';
5
+ import type { CpuSnapshot } from './cpu_load_signal.js';
6
+ import type { EventLoopSnapshot } from './event_loop_load_signal.js';
7
+ import type { LoadSignal } from './load_signal.js';
8
+ import type { MemorySnapshot } from './memory_load_signal.js';
3
9
  import type { SystemInfo } from './system_status.js';
4
10
  export interface SnapshotterOptions {
5
11
  /**
@@ -28,41 +34,21 @@ export interface SnapshotterOptions {
28
34
  /**
29
35
  * Defines the maximum number of new rate limit errors within
30
36
  * the given interval.
31
- * @default 1
37
+ * @default 3
32
38
  */
33
39
  maxClientErrors?: number;
34
40
  /**
35
41
  * Sets the interval in seconds for which a history of resource snapshots
36
42
  * will be kept. Increasing this to very high numbers will affect performance.
37
- * @default 60
43
+ * @default 30
38
44
  */
39
45
  snapshotHistorySecs?: number;
40
46
  /** @internal */
41
47
  log?: CrawleeLogger;
42
- }
43
- interface MemorySnapshot {
44
- createdAt: Date;
45
- isOverloaded: boolean;
46
- usedBytes?: number;
47
- }
48
- interface CpuSnapshot {
49
- createdAt: Date;
50
- isOverloaded: boolean;
51
- usedRatio: number;
52
- ticks?: {
53
- idle: number;
54
- total: number;
55
- };
56
- }
57
- interface EventLoopSnapshot {
58
- createdAt: Date;
59
- isOverloaded: boolean;
60
- exceededMillis: number;
61
- }
62
- interface ClientSnapshot {
63
- createdAt: Date;
64
- isOverloaded: boolean;
65
- rateLimitErrorCount: number;
48
+ /** @internal */
49
+ client?: StorageBackend;
50
+ /** @internal */
51
+ config?: Configuration;
66
52
  }
67
53
  /**
68
54
  * Creates snapshots of system resources at given intervals and marks the resource
@@ -87,24 +73,26 @@ interface ClientSnapshot {
87
73
  *
88
74
  * Client becomes overloaded when rate limit errors (429 - Too Many Requests),
89
75
  * typically received from the request queue, exceed the set limit within the set interval.
76
+ *
90
77
  * @category Scaling
91
78
  */
92
79
  export declare class Snapshotter {
93
80
  log: CrawleeLogger;
94
- eventLoopSnapshotIntervalMillis: number;
95
- clientSnapshotIntervalMillis: number;
96
- snapshotHistoryMillis: number;
97
- maxBlockedMillis: number;
98
- maxUsedMemoryRatio: number;
99
- maxClientErrors: number;
100
- maxMemoryBytes: number;
101
- cpuSnapshots: CpuSnapshot[];
102
- eventLoopSnapshots: EventLoopSnapshot[];
103
- memorySnapshots: MemorySnapshot[];
104
- clientSnapshots: ClientSnapshot[];
105
- eventLoopInterval: BetterIntervalID;
106
- clientInterval: BetterIntervalID;
107
- lastLoggedCriticalMemoryOverloadAt: Date | null;
81
+ client: StorageBackend;
82
+ config: Configuration;
83
+ private readonly memorySignal;
84
+ private readonly eventLoopSignal;
85
+ private readonly cpuSignal;
86
+ private readonly clientSignal;
87
+ /**
88
+ * Returns the four built-in signals as an array, so `SystemStatus` can
89
+ * iterate them alongside any custom `LoadSignal` instances.
90
+ */
91
+ getLoadSignals(): LoadSignal[];
92
+ get cpuSnapshots(): CpuSnapshot[];
93
+ get eventLoopSnapshots(): EventLoopSnapshot[];
94
+ get memorySnapshots(): MemorySnapshot[];
95
+ get clientSnapshots(): ClientSnapshot[];
108
96
  /**
109
97
  * @param [options] All `Snapshotter` configuration options.
110
98
  */
@@ -138,42 +126,27 @@ export declare class Snapshotter {
138
126
  */
139
127
  getClientSample(sampleDurationMillis?: number): ClientSnapshot[];
140
128
  /**
141
- * Finds the latest snapshots by sampleDurationMillis in the provided array.
142
- */
143
- protected _getSample<T extends {
144
- createdAt: Date;
145
- }>(snapshots: T[], sampleDurationMillis?: number): T[];
146
- /**
147
- * Creates a snapshot of current memory usage
148
- * using the Apify platform `systemInfo` event.
129
+ * @deprecated Kept for backward compatibility.
149
130
  */
150
131
  protected _snapshotMemory(systemInfo: SystemInfo): void;
151
132
  /**
152
- * Checks for critical memory overload and logs it to the console.
133
+ * @deprecated Kept for backward compatibility.
153
134
  */
154
135
  protected _memoryOverloadWarning(systemInfo: SystemInfo): void;
155
136
  /**
156
- * Creates a snapshot of current event loop delay.
137
+ * @deprecated Kept for backward compatibility.
157
138
  */
158
139
  protected _snapshotEventLoop(intervalCallback: () => unknown): void;
159
140
  /**
160
- * Creates a snapshot of current CPU usage using the Apify platform `systemInfo` event.
141
+ * @deprecated Kept for backward compatibility.
161
142
  */
162
143
  protected _snapshotCpu(systemInfo: SystemInfo): void;
163
144
  /**
164
- * Creates a snapshot of current API state by checking for
165
- * rate limit errors. Only errors produced by a 2nd retry
166
- * of the API call are considered for snapshotting since
167
- * earlier errors may just be caused by a random spike in
168
- * number of requests and do not necessarily signify API
169
- * overloading.
145
+ * @deprecated Kept for backward compatibility.
170
146
  */
171
147
  protected _snapshotClient(intervalCallback: () => unknown): void;
172
148
  /**
173
- * Removes snapshots that are older than the snapshotHistorySecs option
174
- * from the array (destructively - in place).
149
+ * @deprecated Pruning is now handled by individual signals.
175
150
  */
176
- protected _pruneSnapshots(snapshots: MemorySnapshot[] | CpuSnapshot[] | EventLoopSnapshot[] | ClientSnapshot[], now: Date): void;
151
+ protected _pruneSnapshots(_snapshots: any[], _now: Date): void;
177
152
  }
178
- export {};
179
- //# sourceMappingURL=snapshotter.d.ts.map
@@ -1,10 +1,9 @@
1
- import { getMemoryInfo, isContainerized } from '@crawlee/utils';
2
1
  import ow from 'ow';
3
- import { betterClearInterval, betterSetInterval } from '@apify/utilities';
4
2
  import { serviceLocator } from '../service_locator.js';
5
- const RESERVE_MEMORY_RATIO = 0.5;
6
- const CLIENT_RATE_LIMIT_ERROR_RETRY_COUNT = 2;
7
- const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10000;
3
+ import { createClientLoadSignal } from './client_load_signal.js';
4
+ import { createCpuLoadSignal } from './cpu_load_signal.js';
5
+ import { createEventLoopLoadSignal } from './event_loop_load_signal.js';
6
+ import { MemoryLoadSignal } from './memory_load_signal.js';
8
7
  /**
9
8
  * Creates snapshots of system resources at given intervals and marks the resource
10
9
  * as either overloaded or not during the last interval. Keeps a history of the snapshots.
@@ -28,24 +27,37 @@ const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10000;
28
27
  *
29
28
  * Client becomes overloaded when rate limit errors (429 - Too Many Requests),
30
29
  * typically received from the request queue, exceed the set limit within the set interval.
30
+ *
31
31
  * @category Scaling
32
32
  */
33
33
  export class Snapshotter {
34
34
  log;
35
- eventLoopSnapshotIntervalMillis;
36
- clientSnapshotIntervalMillis;
37
- snapshotHistoryMillis;
38
- maxBlockedMillis;
39
- maxUsedMemoryRatio;
40
- maxClientErrors;
41
- maxMemoryBytes;
42
- cpuSnapshots = [];
43
- eventLoopSnapshots = [];
44
- memorySnapshots = [];
45
- clientSnapshots = [];
46
- eventLoopInterval = null;
47
- clientInterval = null;
48
- lastLoggedCriticalMemoryOverloadAt = null;
35
+ client;
36
+ config;
37
+ memorySignal;
38
+ eventLoopSignal;
39
+ cpuSignal;
40
+ clientSignal;
41
+ /**
42
+ * Returns the four built-in signals as an array, so `SystemStatus` can
43
+ * iterate them alongside any custom `LoadSignal` instances.
44
+ */
45
+ getLoadSignals() {
46
+ return [this.memorySignal, this.eventLoopSignal, this.cpuSignal, this.clientSignal];
47
+ }
48
+ // Legacy public properties kept for backward compat (tests read these directly)
49
+ get cpuSnapshots() {
50
+ return this.cpuSignal.store.getAll();
51
+ }
52
+ get eventLoopSnapshots() {
53
+ return this.eventLoopSignal.store.getAll();
54
+ }
55
+ get memorySnapshots() {
56
+ return this.memorySignal.getMemorySnapshots();
57
+ }
58
+ get clientSnapshots() {
59
+ return this.clientSignal.store.getAll();
60
+ }
49
61
  /**
50
62
  * @param [options] All `Snapshotter` configuration options.
51
63
  */
@@ -58,54 +70,53 @@ export class Snapshotter {
58
70
  maxUsedMemoryRatio: ow.optional.number,
59
71
  maxClientErrors: ow.optional.number,
60
72
  log: ow.optional.object,
73
+ client: ow.optional.object,
74
+ config: ow.optional.object,
61
75
  }));
62
- const { eventLoopSnapshotIntervalSecs = 0.5, clientSnapshotIntervalSecs = 1, snapshotHistorySecs = 30, maxBlockedMillis = 50, maxUsedMemoryRatio = 0.9, maxClientErrors = 3, log = serviceLocator.getLogger(), } = options;
76
+ const { eventLoopSnapshotIntervalSecs = 0.5, clientSnapshotIntervalSecs = 1, snapshotHistorySecs = 30, maxBlockedMillis = 50, maxUsedMemoryRatio = 0.9, maxClientErrors = 3, log = serviceLocator.getLogger(), config = serviceLocator.getConfiguration(), client = serviceLocator.getStorageBackend(), } = options;
63
77
  this.log = log.child({ prefix: 'Snapshotter' });
64
- this.eventLoopSnapshotIntervalMillis = eventLoopSnapshotIntervalSecs * 1000;
65
- this.clientSnapshotIntervalMillis = clientSnapshotIntervalSecs * 1000;
66
- this.snapshotHistoryMillis = snapshotHistorySecs * 1000;
67
- this.maxBlockedMillis = maxBlockedMillis;
68
- this.maxUsedMemoryRatio = maxUsedMemoryRatio;
69
- this.maxClientErrors = maxClientErrors;
70
- // We need to pre-bind those functions to be able to successfully remove listeners.
71
- this._snapshotCpu = this._snapshotCpu.bind(this);
72
- this._snapshotMemory = this._snapshotMemory.bind(this);
78
+ this.client = client;
79
+ this.config = config;
80
+ const snapshotHistoryMillis = snapshotHistorySecs * 1000;
81
+ this.memorySignal = new MemoryLoadSignal({
82
+ maxUsedMemoryRatio,
83
+ snapshotHistoryMillis,
84
+ config: this.config,
85
+ log: this.log,
86
+ });
87
+ this.eventLoopSignal = createEventLoopLoadSignal({
88
+ eventLoopSnapshotIntervalSecs,
89
+ maxBlockedMillis,
90
+ snapshotHistoryMillis,
91
+ });
92
+ this.cpuSignal = createCpuLoadSignal({
93
+ snapshotHistoryMillis,
94
+ config: this.config,
95
+ });
96
+ this.clientSignal = createClientLoadSignal({
97
+ client: this.client,
98
+ clientSnapshotIntervalSecs,
99
+ maxClientErrors,
100
+ snapshotHistoryMillis,
101
+ });
73
102
  }
74
103
  /**
75
104
  * Starts capturing snapshots at configured intervals.
76
105
  */
77
106
  async start() {
78
- const memoryMbytes = serviceLocator.getConfiguration().memoryMbytes ?? 0;
79
- if (memoryMbytes > 0) {
80
- this.maxMemoryBytes = memoryMbytes * 1024 * 1024;
81
- }
82
- else {
83
- const containerized = serviceLocator.getConfiguration().containerized ?? (await isContainerized());
84
- const memInfo = await getMemoryInfo({
85
- containerized,
86
- logger: serviceLocator.getLogger(),
87
- });
88
- const totalBytes = memInfo.totalBytes;
89
- this.maxMemoryBytes = Math.ceil(totalBytes * serviceLocator.getConfiguration().availableMemoryRatio);
90
- this.log.debug(`Setting max memory of this run to ${Math.round(this.maxMemoryBytes / 1024 / 1024)} MB. ` +
91
- 'Use the CRAWLEE_MEMORY_MBYTES or CRAWLEE_AVAILABLE_MEMORY_RATIO environment variable to override it.');
92
- }
93
- // Start snapshotting.
94
- this.eventLoopInterval = betterSetInterval(this._snapshotEventLoop.bind(this), this.eventLoopSnapshotIntervalMillis);
95
- this.clientInterval = betterSetInterval(this._snapshotClient.bind(this), this.clientSnapshotIntervalMillis);
96
- const events = serviceLocator.getEventManager();
97
- events.on("systemInfo" /* EventType.SYSTEM_INFO */, this._snapshotCpu);
98
- events.on("systemInfo" /* EventType.SYSTEM_INFO */, this._snapshotMemory);
107
+ await this.memorySignal.start();
108
+ await this.eventLoopSignal.start();
109
+ await this.cpuSignal.start();
110
+ await this.clientSignal.start();
99
111
  }
100
112
  /**
101
113
  * Stops all resource capturing.
102
114
  */
103
115
  async stop() {
104
- betterClearInterval(this.eventLoopInterval);
105
- betterClearInterval(this.clientInterval);
106
- const events = serviceLocator.getEventManager();
107
- events.off("systemInfo" /* EventType.SYSTEM_INFO */, this._snapshotCpu);
108
- events.off("systemInfo" /* EventType.SYSTEM_INFO */, this._snapshotMemory);
116
+ await this.memorySignal.stop();
117
+ await this.eventLoopSignal.stop();
118
+ await this.cpuSignal.stop();
119
+ await this.clientSignal.stop();
109
120
  // Allow microtask queue to unwind before stop returns.
110
121
  await new Promise((resolve) => {
111
122
  setImmediate(resolve);
@@ -116,166 +127,63 @@ export class Snapshotter {
116
127
  * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
117
128
  */
118
129
  getMemorySample(sampleDurationMillis) {
119
- return this._getSample(this.memorySnapshots, sampleDurationMillis);
130
+ return this.memorySignal.getSample(sampleDurationMillis);
120
131
  }
121
132
  /**
122
133
  * Returns a sample of latest event loop snapshots, with the size of the sample defined
123
134
  * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
124
135
  */
125
136
  getEventLoopSample(sampleDurationMillis) {
126
- return this._getSample(this.eventLoopSnapshots, sampleDurationMillis);
137
+ return this.eventLoopSignal.getSample(sampleDurationMillis);
127
138
  }
128
139
  /**
129
140
  * Returns a sample of latest CPU snapshots, with the size of the sample defined
130
141
  * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
131
142
  */
132
143
  getCpuSample(sampleDurationMillis) {
133
- return this._getSample(this.cpuSnapshots, sampleDurationMillis);
144
+ return this.cpuSignal.getSample(sampleDurationMillis);
134
145
  }
135
146
  /**
136
147
  * Returns a sample of latest Client snapshots, with the size of the sample defined
137
148
  * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
138
149
  */
139
150
  getClientSample(sampleDurationMillis) {
140
- return this._getSample(this.clientSnapshots, sampleDurationMillis);
151
+ return this.clientSignal.getSample(sampleDurationMillis);
141
152
  }
142
153
  /**
143
- * Finds the latest snapshots by sampleDurationMillis in the provided array.
144
- */
145
- _getSample(snapshots, sampleDurationMillis) {
146
- if (!sampleDurationMillis)
147
- return snapshots;
148
- const sample = [];
149
- let idx = snapshots.length;
150
- if (!idx)
151
- return sample;
152
- const latestTime = snapshots[idx - 1].createdAt;
153
- while (idx--) {
154
- const snapshot = snapshots[idx];
155
- if (+latestTime - +snapshot.createdAt <= sampleDurationMillis) {
156
- sample.unshift(snapshot);
157
- }
158
- else {
159
- break;
160
- }
161
- }
162
- return sample;
163
- }
164
- /**
165
- * Creates a snapshot of current memory usage
166
- * using the Apify platform `systemInfo` event.
154
+ * @deprecated Kept for backward compatibility.
167
155
  */
168
156
  _snapshotMemory(systemInfo) {
169
- const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
170
- this._pruneSnapshots(this.memorySnapshots, createdAt);
171
- const { memCurrentBytes } = systemInfo;
172
- const snapshot = {
173
- createdAt,
174
- isOverloaded: memCurrentBytes / this.maxMemoryBytes > this.maxUsedMemoryRatio,
175
- usedBytes: memCurrentBytes,
176
- };
177
- this.memorySnapshots.push(snapshot);
178
- this._memoryOverloadWarning(systemInfo);
157
+ this.memorySignal._onSystemInfo(systemInfo);
179
158
  }
180
159
  /**
181
- * Checks for critical memory overload and logs it to the console.
160
+ * @deprecated Kept for backward compatibility.
182
161
  */
183
162
  _memoryOverloadWarning(systemInfo) {
184
- const { memCurrentBytes } = systemInfo;
185
- const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
186
- if (this.lastLoggedCriticalMemoryOverloadAt &&
187
- +createdAt < +this.lastLoggedCriticalMemoryOverloadAt + CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS)
188
- return;
189
- const maxDesiredMemoryBytes = this.maxUsedMemoryRatio * this.maxMemoryBytes;
190
- const reserveMemory = this.maxMemoryBytes * (1 - this.maxUsedMemoryRatio) * RESERVE_MEMORY_RATIO;
191
- const criticalOverloadBytes = maxDesiredMemoryBytes + reserveMemory;
192
- const isCriticalOverload = memCurrentBytes > criticalOverloadBytes;
193
- if (isCriticalOverload) {
194
- const usedPercentage = Math.round((memCurrentBytes / this.maxMemoryBytes) * 100);
195
- const toMb = (bytes) => Math.round(bytes / 1024 ** 2);
196
- this.log.warning('Memory is critically overloaded. ' +
197
- `Using ${toMb(memCurrentBytes)} MB of ${toMb(this.maxMemoryBytes)} MB (${usedPercentage}%). Consider increasing available memory.`);
198
- this.lastLoggedCriticalMemoryOverloadAt = createdAt;
199
- }
163
+ this.memorySignal._memoryOverloadWarning(systemInfo);
200
164
  }
201
165
  /**
202
- * Creates a snapshot of current event loop delay.
166
+ * @deprecated Kept for backward compatibility.
203
167
  */
204
168
  _snapshotEventLoop(intervalCallback) {
205
- const now = new Date();
206
- this._pruneSnapshots(this.eventLoopSnapshots, now);
207
- const snapshot = {
208
- createdAt: now,
209
- isOverloaded: false,
210
- exceededMillis: 0,
211
- };
212
- const previousSnapshot = this.eventLoopSnapshots[this.eventLoopSnapshots.length - 1];
213
- if (previousSnapshot) {
214
- const { createdAt } = previousSnapshot;
215
- const delta = now.getTime() - +createdAt - this.eventLoopSnapshotIntervalMillis;
216
- if (delta > this.maxBlockedMillis)
217
- snapshot.isOverloaded = true;
218
- snapshot.exceededMillis = Math.max(delta - this.maxBlockedMillis, 0);
219
- }
220
- this.eventLoopSnapshots.push(snapshot);
221
- intervalCallback();
169
+ this.eventLoopSignal.handle(intervalCallback);
222
170
  }
223
171
  /**
224
- * Creates a snapshot of current CPU usage using the Apify platform `systemInfo` event.
172
+ * @deprecated Kept for backward compatibility.
225
173
  */
226
174
  _snapshotCpu(systemInfo) {
227
- const { cpuCurrentUsage, isCpuOverloaded } = systemInfo;
228
- const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
229
- this._pruneSnapshots(this.cpuSnapshots, createdAt);
230
- this.cpuSnapshots.push({
231
- createdAt,
232
- isOverloaded: isCpuOverloaded,
233
- usedRatio: Math.ceil(cpuCurrentUsage / 100),
234
- });
175
+ this.cpuSignal.handle(systemInfo);
235
176
  }
236
177
  /**
237
- * Creates a snapshot of current API state by checking for
238
- * rate limit errors. Only errors produced by a 2nd retry
239
- * of the API call are considered for snapshotting since
240
- * earlier errors may just be caused by a random spike in
241
- * number of requests and do not necessarily signify API
242
- * overloading.
178
+ * @deprecated Kept for backward compatibility.
243
179
  */
244
180
  _snapshotClient(intervalCallback) {
245
- const now = new Date();
246
- this._pruneSnapshots(this.clientSnapshots, now);
247
- const allErrorCounts = serviceLocator.getStorageBackend().stats?.rateLimitErrors ?? []; // storage backend might not support this
248
- const currentErrCount = allErrorCounts[CLIENT_RATE_LIMIT_ERROR_RETRY_COUNT] || 0;
249
- // Handle empty snapshots array
250
- const snapshot = {
251
- createdAt: now,
252
- isOverloaded: false,
253
- rateLimitErrorCount: currentErrCount,
254
- };
255
- const previousSnapshot = this.clientSnapshots[this.clientSnapshots.length - 1];
256
- if (previousSnapshot) {
257
- const { rateLimitErrorCount } = previousSnapshot;
258
- const delta = currentErrCount - rateLimitErrorCount;
259
- if (delta > this.maxClientErrors)
260
- snapshot.isOverloaded = true;
261
- }
262
- this.clientSnapshots.push(snapshot);
263
- intervalCallback();
181
+ this.clientSignal.handle(intervalCallback);
264
182
  }
265
183
  /**
266
- * Removes snapshots that are older than the snapshotHistorySecs option
267
- * from the array (destructively - in place).
184
+ * @deprecated Pruning is now handled by individual signals.
268
185
  */
269
- _pruneSnapshots(snapshots, now) {
270
- let oldCount = 0;
271
- for (let i = 0; i < snapshots.length; i++) {
272
- const { createdAt } = snapshots[i];
273
- if (now.getTime() - new Date(createdAt).getTime() > this.snapshotHistoryMillis)
274
- oldCount++;
275
- else
276
- break;
277
- }
278
- snapshots.splice(0, oldCount);
186
+ _pruneSnapshots(_snapshots, _now) {
187
+ // no-op signals prune themselves
279
188
  }
280
189
  }
281
- //# sourceMappingURL=snapshotter.js.map
@@ -1,3 +1,4 @@
1
+ import type { LoadSignal } from './load_signal.js';
1
2
  import { Snapshotter } from './snapshotter.js';
2
3
  /**
3
4
  * Represents the current status of the system.
@@ -9,6 +10,7 @@ export interface SystemInfo {
9
10
  eventLoopInfo: ClientInfo;
10
11
  cpuInfo: ClientInfo;
11
12
  clientInfo: ClientInfo;
13
+ memTotalBytes?: number;
12
14
  memCurrentBytes?: number;
13
15
  /**
14
16
  * Platform only property
@@ -25,6 +27,11 @@ export interface SystemInfo {
25
27
  * @internal
26
28
  */
27
29
  createdAt?: Date;
30
+ /**
31
+ * Status of additional load signals beyond the built-in four.
32
+ * Keys are `LoadSignal.name` values, values are overload info.
33
+ */
34
+ loadSignalInfo?: Record<string, ClientInfo>;
28
35
  }
29
36
  export interface SystemStatusOptions {
30
37
  /**
@@ -60,6 +67,13 @@ export interface SystemStatusOptions {
60
67
  * The `Snapshotter` instance to be queried for `SystemStatus`.
61
68
  */
62
69
  snapshotter?: Snapshotter;
70
+ /**
71
+ * Additional load signals to include in the system status evaluation.
72
+ * These are evaluated alongside the built-in memory, CPU, event loop,
73
+ * and client signals. If any signal reports overload, the system is
74
+ * considered overloaded.
75
+ */
76
+ loadSignals?: LoadSignal[];
63
77
  }
64
78
  export interface ClientInfo {
65
79
  isOverloaded: boolean;
@@ -102,11 +116,14 @@ export interface FinalStatistics {
102
116
  */
103
117
  export declare class SystemStatus {
104
118
  private readonly currentHistoryMillis;
105
- private readonly maxMemoryOverloadedRatio;
106
- private readonly maxEventLoopOverloadedRatio;
107
- private readonly maxCpuOverloadedRatio;
108
- private readonly maxClientOverloadedRatio;
109
119
  private readonly snapshotter;
120
+ private readonly signals;
121
+ /**
122
+ * Per-signal ratio overrides. The built-in four get their overrides from
123
+ * the legacy `max*OverloadedRatio` options; custom signals use their own
124
+ * `overloadedRatio`.
125
+ */
126
+ private ratioOverrides;
110
127
  constructor(options?: SystemStatusOptions);
111
128
  /**
112
129
  * Returns an {@link SystemInfo} object with the following structure:
@@ -146,33 +163,4 @@ export declare class SystemStatus {
146
163
  * Returns a system status object.
147
164
  */
148
165
  protected _isSystemIdle(sampleDurationMillis?: number): SystemInfo;
149
- /**
150
- * Returns an object with an isOverloaded property set to true
151
- * if the memory has been overloaded in the last sampleDurationMillis.
152
- */
153
- protected _isMemoryOverloaded(sampleDurationMillis?: number): ClientInfo;
154
- /**
155
- * Returns an object with an isOverloaded property set to true
156
- * if the event loop has been overloaded in the last sampleDurationMillis.
157
- */
158
- protected _isEventLoopOverloaded(sampleDurationMillis?: number): ClientInfo;
159
- /**
160
- * Returns an object with an isOverloaded property set to true
161
- * if the CPU has been overloaded in the last sampleDurationMillis.
162
- */
163
- protected _isCpuOverloaded(sampleDurationMillis?: number): ClientInfo;
164
- /**
165
- * Returns an object with an isOverloaded property set to true
166
- * if the client has been overloaded in the last sampleDurationMillis.
167
- */
168
- protected _isClientOverloaded(sampleDurationMillis?: number): ClientInfo;
169
- /**
170
- * Returns an object with sample information and an isOverloaded property
171
- * set to true if at least the ratio of snapshots in the sample are overloaded.
172
- */
173
- protected _isSampleOverloaded<T extends {
174
- createdAt: Date;
175
- isOverloaded: boolean;
176
- }>(sample: T[], ratio: number): ClientInfo;
177
166
  }
178
- //# sourceMappingURL=system_status.d.ts.map