@crawlee/core 4.0.0-rc.0 → 4.0.0-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/README.md +1 -1
  2. package/configuration.d.ts +15 -46
  3. package/configuration.js +8 -20
  4. package/errors.d.ts +12 -53
  5. package/errors.js +13 -66
  6. package/events/index.d.ts +1 -0
  7. package/events/local_event_manager.d.ts +0 -7
  8. package/events/local_event_manager.js +8 -8
  9. package/events/system_info.d.ts +38 -0
  10. package/index.d.ts +2 -8
  11. package/index.js +4 -8
  12. package/internal.d.ts +8 -0
  13. package/internal.js +9 -0
  14. package/log.d.ts +10 -11
  15. package/log.js +53 -25
  16. package/memory-storage/memory-storage.d.ts +12 -7
  17. package/memory-storage/memory-storage.js +45 -17
  18. package/memory-storage/resource-clients/dataset.d.ts +0 -5
  19. package/memory-storage/resource-clients/dataset.js +17 -20
  20. package/memory-storage/resource-clients/key-value-store.d.ts +0 -9
  21. package/memory-storage/resource-clients/key-value-store.js +11 -33
  22. package/memory-storage/resource-clients/request-queue.d.ts +0 -22
  23. package/memory-storage/resource-clients/request-queue.js +57 -53
  24. package/package.json +16 -18
  25. package/proxy_configuration.d.ts +20 -23
  26. package/proxy_configuration.js +18 -12
  27. package/recoverable_state.d.ts +28 -6
  28. package/recoverable_state.js +51 -14
  29. package/request.d.ts +18 -104
  30. package/request.js +41 -220
  31. package/serialization.js +3 -3
  32. package/service_locator.d.ts +3 -0
  33. package/service_locator.js +2 -0
  34. package/storages/dataset.d.ts +9 -15
  35. package/storages/dataset.js +22 -14
  36. package/storages/index.d.ts +3 -4
  37. package/storages/index.js +1 -4
  38. package/storages/key_value_store.d.ts +12 -46
  39. package/storages/key_value_store.js +31 -47
  40. package/storages/key_value_store_codec.js +6 -11
  41. package/storages/request_dedup_cache.d.ts +0 -2
  42. package/storages/request_dedup_cache.js +8 -8
  43. package/storages/request_list.d.ts +6 -82
  44. package/storages/request_list.js +175 -179
  45. package/storages/request_loader.d.ts +48 -22
  46. package/storages/request_loader.js +36 -1
  47. package/storages/request_manager.d.ts +86 -0
  48. package/storages/request_manager_tandem.d.ts +13 -28
  49. package/storages/request_manager_tandem.js +46 -43
  50. package/storages/request_queue.d.ts +19 -49
  51. package/storages/request_queue.js +92 -88
  52. package/storages/storage_instance_manager.d.ts +1 -2
  53. package/storages/storage_instance_manager.js +4 -4
  54. package/storages/transaction.d.ts +27 -9
  55. package/storages/transaction.js +56 -11
  56. package/storages/utils.d.ts +2 -2
  57. package/validators.d.ts +3 -2
  58. package/validators.js +3 -2
  59. package/autoscaling/autoscaled_pool.d.ts +0 -195
  60. package/autoscaling/autoscaled_pool.js +0 -386
  61. package/autoscaling/concurrency_system.d.ts +0 -268
  62. package/autoscaling/concurrency_system.js +0 -362
  63. package/autoscaling/cpu_load_signal.d.ts +0 -43
  64. package/autoscaling/cpu_load_signal.js +0 -47
  65. package/autoscaling/event_loop_load_signal.d.ts +0 -51
  66. package/autoscaling/event_loop_load_signal.js +0 -60
  67. package/autoscaling/index.d.ts +0 -9
  68. package/autoscaling/index.js +0 -9
  69. package/autoscaling/load_signal.d.ts +0 -100
  70. package/autoscaling/load_signal.js +0 -105
  71. package/autoscaling/memory_load_signal.d.ts +0 -47
  72. package/autoscaling/memory_load_signal.js +0 -106
  73. package/autoscaling/snapshotter.d.ts +0 -84
  74. package/autoscaling/snapshotter.js +0 -67
  75. package/autoscaling/storage_backend_load_signal.d.ts +0 -56
  76. package/autoscaling/storage_backend_load_signal.js +0 -73
  77. package/autoscaling/system_status.d.ts +0 -159
  78. package/autoscaling/system_status.js +0 -139
  79. package/autoscaling/weighted_avg.d.ts +0 -5
  80. package/autoscaling/weighted_avg.js +0 -14
  81. package/cookie_utils.d.ts +0 -44
  82. package/cookie_utils.js +0 -122
  83. package/crawlers/context_pipeline.d.ts +0 -70
  84. package/crawlers/context_pipeline.js +0 -122
  85. package/crawlers/crawler_commons.d.ts +0 -159
  86. package/crawlers/error_snapshotter.d.ts +0 -57
  87. package/crawlers/error_snapshotter.js +0 -117
  88. package/crawlers/error_tracker.d.ts +0 -54
  89. package/crawlers/error_tracker.js +0 -308
  90. package/crawlers/index.d.ts +0 -5
  91. package/crawlers/index.js +0 -4
  92. package/crawlers/internals/types.d.ts +0 -7
  93. package/crawlers/internals/types.js +0 -1
  94. package/crawlers/statistics.d.ts +0 -328
  95. package/crawlers/statistics.js +0 -536
  96. package/enqueue_links/enqueue_links.d.ts +0 -156
  97. package/enqueue_links/enqueue_links.js +0 -78
  98. package/enqueue_links/index.d.ts +0 -2
  99. package/enqueue_links/index.js +0 -2
  100. package/enqueue_links/shared.d.ts +0 -93
  101. package/enqueue_links/shared.js +0 -239
  102. package/http.d.ts +0 -9
  103. package/http.js +0 -28
  104. package/router.d.ts +0 -306
  105. package/router.js +0 -309
  106. package/session_pool/consts.d.ts +0 -3
  107. package/session_pool/consts.js +0 -3
  108. package/session_pool/errors.d.ts +0 -7
  109. package/session_pool/errors.js +0 -11
  110. package/session_pool/fingerprint.d.ts +0 -9
  111. package/session_pool/fingerprint.js +0 -30
  112. package/session_pool/index.d.ts +0 -4
  113. package/session_pool/index.js +0 -4
  114. package/session_pool/session.d.ts +0 -150
  115. package/session_pool/session.js +0 -220
  116. package/session_pool/session_pool.d.ts +0 -240
  117. package/session_pool/session_pool.js +0 -394
  118. package/storages/sitemap_request_loader.d.ts +0 -201
  119. package/storages/sitemap_request_loader.js +0 -438
  120. package/storages/throttling_request_manager.d.ts +0 -239
  121. package/storages/throttling_request_manager.js +0 -646
  122. /package/{crawlers/crawler_commons.js → events/system_info.js} +0 -0
@@ -1,362 +0,0 @@
1
- import { z } from 'zod';
2
- import { betterClearInterval, betterSetInterval } from '@apify/utilities';
3
- import { serviceLocator } from '../service_locator.js';
4
- import { parseArgument, schemas, validators } from '../validators.js';
5
- import { Snapshotter } from './snapshotter.js';
6
- import { SystemStatus } from './system_status.js';
7
- const concurrencySchema = z.number().int().gte(1).optional();
8
- // `schemas.anyObject` passes values through by reference, so the load signal instances inside
9
- // `loadSignals` and class instances like loggers keep their prototype.
10
- const concurrencySystemOptionsSchema = z.strictObject({
11
- maxConcurrency: z.number().int().gte(1).default(200),
12
- minConcurrency: z.number().int().gte(1).default(1),
13
- desiredConcurrency: z.number().int().gte(1).optional(),
14
- desiredConcurrencyRatio: z.number().gt(0).lt(1).default(0.9),
15
- scaleUpStepRatio: z.number().gt(0).lt(1).default(0.05),
16
- scaleDownStepRatio: z.number().gt(0).lt(1).default(0.05),
17
- loggingIntervalSecs: schemas.anyNumber
18
- .refine((value) => value > 0, 'Expected a number greater than 0')
19
- .nullish()
20
- .default(60),
21
- autoscaleIntervalSecs: schemas.anyNumber
22
- .refine((value) => value > 0, 'Expected a number greater than 0')
23
- .default(10),
24
- loadSignals: schemas.anyObject.default(() => ({})),
25
- snapshotHistorySecs: schemas.anyNumber.refine((value) => value > 0, 'Expected a number greater than 0').optional(),
26
- currentHistorySecs: schemas.anyNumber.refine((value) => value > 0, 'Expected a number greater than 0').optional(),
27
- log: validators.logger.default(() => serviceLocator.getLogger()),
28
- maxTasksPerMinute: z
29
- .union([z.number().int().gte(1), z.literal(Number.POSITIVE_INFINITY)])
30
- .default(Number.POSITIVE_INFINITY),
31
- });
32
- /**
33
- * The shareable "governor" behind an {@link AutoscaledPool}: it decides whether there is free compute for one more
34
- * task by combining live system load (via an internal {@link Snapshotter}) with a concurrency budget it autoscales
35
- * over time.
36
- *
37
- * Sharing one instance between several pools (and therefore several crawlers) caps their *combined* compute, instead
38
- * of letting each scale independently and oversubscribe the machine.
39
- *
40
- * Whoever builds the instance owns its lifecycle: call {@link ConcurrencySystem.start|`start()`} before any
41
- * borrowing pool runs and {@link ConcurrencySystem.stop|`stop()`} once they are all done (crawlers do this for the
42
- * default system they build, never for an injected one). Both calls are idempotent, and the first `stop()` tears the
43
- * system down for every borrower.
44
- * @category Scaling
45
- */
46
- export class ConcurrencySystem {
47
- log;
48
- desiredConcurrencyRatio;
49
- scaleUpStepRatio;
50
- scaleDownStepRatio;
51
- #loggingIntervalMillis;
52
- #autoscaleIntervalMillis;
53
- maxTasksPerMinute;
54
- #minConcurrency;
55
- #maxConcurrency;
56
- #desiredConcurrency;
57
- // kept as TS-private _-prefixed: autoscaled_pool tests write this backing field directly
58
- _currentConcurrency = 0;
59
- #lastLoggingTime;
60
- #tasksPerMinute = Array.from({ length: 60 }, () => 0);
61
- snapshotter;
62
- #loadSignals;
63
- systemStatus;
64
- #autoscaleInterval;
65
- #tasksDonePerSecondInterval;
66
- /** Whether the snapshotter and autoscaling intervals are currently running. */
67
- #running = false;
68
- /** The in-flight (or completed) startup, memoized so concurrent `start()` calls await one boot. */
69
- #startPromise;
70
- /** Set once per session, so a pool outliving `stop()` is reported once rather than every half second. */
71
- #warnedAboutQueryWhileStopped = false;
72
- constructor(options = {}) {
73
- const { maxConcurrency, minConcurrency, desiredConcurrency, desiredConcurrencyRatio, scaleUpStepRatio, scaleDownStepRatio, loggingIntervalSecs, autoscaleIntervalSecs, loadSignals, snapshotHistorySecs, currentHistorySecs, log, maxTasksPerMinute, } = parseArgument(options, concurrencySystemOptionsSchema, 'ConcurrencySystemOptions');
74
- this.log = log.child({ prefix: 'ConcurrencySystem' });
75
- this.desiredConcurrencyRatio = desiredConcurrencyRatio;
76
- this.scaleUpStepRatio = scaleUpStepRatio;
77
- this.scaleDownStepRatio = scaleDownStepRatio;
78
- this.#loggingIntervalMillis = (loggingIntervalSecs ?? 0) * 1000;
79
- this.#autoscaleIntervalMillis = autoscaleIntervalSecs * 1000;
80
- this.maxTasksPerMinute = maxTasksPerMinute;
81
- this.#minConcurrency = minConcurrency;
82
- this.#maxConcurrency = maxConcurrency;
83
- this.#desiredConcurrency = desiredConcurrency ?? minConcurrency;
84
- this.clampDesiredConcurrency();
85
- this.autoscale = this.autoscale.bind(this);
86
- this.incrementTasksDonePerSecond = this.incrementTasksDonePerSecond.bind(this);
87
- // The built-in signals are collected by the snapshotter; custom ones are simply evaluated alongside them.
88
- const { custom: customLoadSignals = [], ...builtinSignalOptions } = loadSignals;
89
- this.snapshotter = new Snapshotter(builtinSignalOptions);
90
- this.#loadSignals = customLoadSignals;
91
- this.systemStatus = new SystemStatus({
92
- snapshotter: this.snapshotter,
93
- loadSignals: customLoadSignals,
94
- currentHistorySecs,
95
- // Both windows are requested from the signals explicitly, so a signal's own retention can neither widen
96
- // nor (given the start context below) narrow what it contributes.
97
- historySecs: snapshotHistorySecs,
98
- });
99
- }
100
- /**
101
- * Gets the minimum number of tasks running in parallel.
102
- */
103
- get minConcurrency() {
104
- return this.#minConcurrency;
105
- }
106
- /**
107
- * Sets the minimum number of tasks running in parallel.
108
- *
109
- * *WARNING:* If you set this value too high with respect to the available system memory and CPU, your code might run extremely slow or crash.
110
- * If you're not sure, just keep the default value and the concurrency will scale up automatically.
111
- */
112
- set minConcurrency(value) {
113
- parseArgument(value, concurrencySchema);
114
- this.#minConcurrency = value;
115
- this.clampDesiredConcurrency();
116
- }
117
- /**
118
- * Gets the maximum number of tasks running in parallel.
119
- */
120
- get maxConcurrency() {
121
- return this.#maxConcurrency;
122
- }
123
- /**
124
- * Sets the maximum number of tasks running in parallel. Lowering it below the current
125
- * {@link ConcurrencySystem.desiredConcurrency|`desiredConcurrency`} pulls that down to the new ceiling too, so
126
- * the change takes effect immediately (in-flight tasks are never cancelled — the budget simply drains to the new
127
- * limit as they settle).
128
- */
129
- set maxConcurrency(value) {
130
- parseArgument(value, concurrencySchema);
131
- this.#maxConcurrency = value;
132
- this.clampDesiredConcurrency();
133
- }
134
- /**
135
- * Gets the desired concurrency for the system,
136
- * which is an estimated number of parallel tasks that the system can currently support.
137
- */
138
- get desiredConcurrency() {
139
- return this.#desiredConcurrency;
140
- }
141
- /**
142
- * Sets the desired concurrency for the system, i.e. the number of tasks that should be running
143
- * in parallel if there's large enough supply of tasks.
144
- */
145
- set desiredConcurrency(value) {
146
- parseArgument(value, concurrencySchema);
147
- this.#desiredConcurrency = value;
148
- this.clampDesiredConcurrency();
149
- }
150
- /**
151
- * Re-establishes `minConcurrency <= desiredConcurrency <= maxConcurrency` after any of the three is retuned.
152
- * Dispatch gates on the desired value alone, so one stranded above `maxConcurrency` would make the ceiling
153
- * meaningless. A contradictory pair (`minConcurrency > maxConcurrency`) resolves in favour of the maximum, since
154
- * that is the limit callers set in order to protect something.
155
- */
156
- clampDesiredConcurrency() {
157
- const atLeastMin = Math.max(this.#desiredConcurrency, this.#minConcurrency);
158
- this.#desiredConcurrency = Math.min(atLeastMin, this.#maxConcurrency);
159
- }
160
- get currentConcurrency() {
161
- return this._currentConcurrency;
162
- }
163
- /** Whether the system is currently monitoring load and autoscaling the budget. */
164
- get isRunning() {
165
- return this.#running;
166
- }
167
- /**
168
- * Boots the underlying snapshotter and the autoscaling interval. Idempotent, so a shared system isn't restarted
169
- * when handed to another consumer; concurrent callers await one startup. Rejects, leaving nothing running, if a
170
- * signal fails to start.
171
- */
172
- async start() {
173
- // Unwound and dropped again on failure, so a later `start()` retries instead of resolving instantly against
174
- // a system that is down.
175
- this.#startPromise ??= this.boot().catch(async (error) => {
176
- this.#startPromise = undefined;
177
- await this.shutDown();
178
- throw error;
179
- });
180
- await this.#startPromise;
181
- }
182
- async boot() {
183
- // Per-session measurement state, reset so a restarted system isn't judged on the previous session. The
184
- // per-minute window matters most: its ageing interval is cleared while we are down, so starts from before an
185
- // arbitrarily long stop would otherwise still count against "this minute" and trip the cap immediately.
186
- this.#tasksPerMinute = Array.from({ length: 60 }, () => 0);
187
- this.#lastLoggingTime = undefined;
188
- this.#warnedAboutQueryWhileStopped = false;
189
- // Signals are told how much history to keep when they start: exactly the longest window they will be sampled
190
- // over, so nobody has to guess a retention value that matches this system's configuration.
191
- const startContext = { maxSampleWindowMillis: this.systemStatus.maxSampleWindowMillis };
192
- await this.snapshotter.start(startContext);
193
- await Promise.all(this.#loadSignals.map(async (s) => s.start(startContext)));
194
- this.#autoscaleInterval = betterSetInterval(this.autoscale, this.#autoscaleIntervalMillis);
195
- if (this.maxTasksPerMinute !== Infinity) {
196
- this.#tasksDonePerSecondInterval = betterSetInterval(this.incrementTasksDonePerSecond, 1000);
197
- }
198
- // Last, so `isRunning` never claims a system whose signals aren't collecting yet.
199
- this.#running = true;
200
- }
201
- /**
202
- * Stops the snapshotter and intervals. Idempotent and safe to call even if the system was never started.
203
- */
204
- async stop() {
205
- if (this.#startPromise === undefined) {
206
- return;
207
- }
208
- // Waited out rather than interrupted, or the intervals a starting signal is about to register outlive us.
209
- await this.#startPromise.catch(() => { });
210
- this.#startPromise = undefined;
211
- this.#running = false;
212
- await this.shutDown();
213
- }
214
- async shutDown() {
215
- if (this.#autoscaleInterval)
216
- betterClearInterval(this.#autoscaleInterval);
217
- if (this.#tasksDonePerSecondInterval)
218
- betterClearInterval(this.#tasksDonePerSecondInterval);
219
- await this.snapshotter.stop();
220
- await Promise.all(this.#loadSignals.map(async (s) => s.stop()));
221
- }
222
- /**
223
- * Reports, once per session, that capacity is being queried on a system that isn't running — a mistake nothing
224
- * else catches, since {@link AutoscaledPool.run|`pool.run()`} only checks
225
- * {@link ConcurrencySystem.isRunning|`isRunning`} on the way in. Both the overload verdict and
226
- * `desiredConcurrency` are frozen at that point, so the borrowing pool would otherwise just quietly mis-scale.
227
- */
228
- warnIfNotRunning() {
229
- if (this.#running || this.#warnedAboutQueryWhileStopped) {
230
- return;
231
- }
232
- this.#warnedAboutQueryWhileStopped = true;
233
- this.log.warning('Capacity is being queried on a ConcurrencySystem that is not running, so system load is no longer being ' +
234
- 'monitored and the concurrency will no longer be adjusted. Whoever creates a ConcurrencySystem owns ' +
235
- 'its lifecycle: call `await concurrencySystem.stop()` only once every pool and crawler borrowing it ' +
236
- 'has finished.');
237
- }
238
- /**
239
- * May **one more** task start right now? Returns `false` when the shared budget is spent (desired concurrency
240
- * reached) or when the machine is overloaded past `minConcurrency`.
241
- *
242
- * One budget for the whole machine, so the asking consumer is ignored — and therefore optional here, unlike in the
243
- * interface, letting the answer be queried directly.
244
- */
245
- hasCapacityForTask(_consumer) {
246
- this.warnIfNotRunning();
247
- if (this._currentConcurrency >= this.#desiredConcurrency) {
248
- this.log.perf('Task will not run. Desired concurrency achieved.');
249
- return false;
250
- }
251
- const currentStatus = this.systemStatus.getCurrentStatus();
252
- const { isSystemIdle } = currentStatus;
253
- if (!isSystemIdle && this._currentConcurrency >= this.#minConcurrency) {
254
- this.log.perf('Task will not be run. System is overloaded.', currentStatus);
255
- return false;
256
- }
257
- return true;
258
- }
259
- /** Whether the per-minute task cap has been reached. */
260
- get isOverMaxRequestLimit() {
261
- if (this.maxTasksPerMinute === Infinity) {
262
- return false;
263
- }
264
- return this.#tasksPerMinute.reduce((acc, curr) => acc + curr, 0) >= this.maxTasksPerMinute;
265
- }
266
- /**
267
- * Atomically books a task against the shared budget: re-checks
268
- * {@link ConcurrencySystem.hasCapacityForTask|`hasCapacityForTask()`} plus the per-minute task cap and
269
- * increments the current concurrency in one synchronous step, returning `false` (without booking) when there is no
270
- * room. Call right before the task actually runs.
271
- *
272
- * The cap is enforced here rather than in the pre-check so that an empty queue never blocks the pool for a whole
273
- * extra minute.
274
- */
275
- tryRegisterTaskStart(consumer) {
276
- if (!this.hasCapacityForTask(consumer)) {
277
- return false;
278
- }
279
- if (this.isOverMaxRequestLimit) {
280
- this.log.perf('Task will not run. Maximum tasks per minute reached.');
281
- return false;
282
- }
283
- this._currentConcurrency++;
284
- this.#tasksPerMinute[0]++;
285
- return true;
286
- }
287
- /** Returns a slot to the shared budget, whoever booked it. */
288
- registerTaskEnd(_consumer) {
289
- this._currentConcurrency--;
290
- }
291
- /**
292
- * What the system currently makes of the machine: the per-signal overload verdicts, evaluated over the
293
- * task-gating window, exactly as {@link ConcurrencySystem.hasCapacityForTask|`hasCapacityForTask()`} sees them.
294
- * The one public window into load monitoring — useful for answering *why* a crawl is not scaling up.
295
- */
296
- getCurrentStatus() {
297
- return this.systemStatus.getCurrentStatus();
298
- }
299
- /**
300
- * Evaluates the historical system status and scales the shared desired concurrency up or down accordingly. Driven
301
- * by the autoscaling interval started in {@link ConcurrencySystem.start|`start()`}.
302
- */
303
- autoscale(intervalCallback) {
304
- if (this.isOverMaxRequestLimit)
305
- return intervalCallback();
306
- const systemStatus = this.systemStatus.getHistoricalStatus();
307
- const { isSystemIdle } = systemStatus;
308
- const weAreNotAtMax = this.#desiredConcurrency < this.#maxConcurrency;
309
- const minCurrentConcurrency = Math.floor(this.#desiredConcurrency * this.desiredConcurrencyRatio);
310
- const weAreReachingDesiredConcurrency = this._currentConcurrency >= minCurrentConcurrency;
311
- if (isSystemIdle && weAreNotAtMax && weAreReachingDesiredConcurrency)
312
- this.scaleUp(systemStatus);
313
- const isSystemOverloaded = !isSystemIdle;
314
- const weAreNotAtMin = this.#desiredConcurrency > this.#minConcurrency;
315
- if (isSystemOverloaded && weAreNotAtMin)
316
- this.scaleDown(systemStatus);
317
- if (this.#loggingIntervalMillis > 0) {
318
- const now = Date.now();
319
- if (this.#lastLoggingTime == null) {
320
- this.#lastLoggingTime = now;
321
- }
322
- else if (now > this.#lastLoggingTime + this.#loggingIntervalMillis) {
323
- this.#lastLoggingTime = now;
324
- this.log.info('state', {
325
- currentConcurrency: this._currentConcurrency,
326
- desiredConcurrency: this.#desiredConcurrency,
327
- systemStatus,
328
- });
329
- }
330
- }
331
- return intervalCallback();
332
- }
333
- /**
334
- * Scales the system up by increasing the desired concurrency by the scaleUpStepRatio.
335
- */
336
- scaleUp(systemStatus) {
337
- const step = Math.ceil(this.#desiredConcurrency * this.scaleUpStepRatio);
338
- this.#desiredConcurrency = Math.min(this.#maxConcurrency, this.#desiredConcurrency + step);
339
- this.log.debug('scaling up', {
340
- oldConcurrency: this.#desiredConcurrency - step,
341
- newConcurrency: this.#desiredConcurrency,
342
- systemStatus,
343
- });
344
- }
345
- /**
346
- * Scales the system down by decreasing the desired concurrency by the scaleDownStepRatio.
347
- */
348
- scaleDown(systemStatus) {
349
- const step = Math.ceil(this.#desiredConcurrency * this.scaleDownStepRatio);
350
- this.#desiredConcurrency = Math.max(this.#minConcurrency, this.#desiredConcurrency - step);
351
- this.log.debug('scaling down', {
352
- oldConcurrency: this.#desiredConcurrency + step,
353
- newConcurrency: this.#desiredConcurrency,
354
- systemStatus,
355
- });
356
- }
357
- incrementTasksDonePerSecond(intervalCallback) {
358
- this.#tasksPerMinute.unshift(0);
359
- this.#tasksPerMinute.pop();
360
- return intervalCallback();
361
- }
362
- }
@@ -1,43 +0,0 @@
1
- import type { LoadSignal, LoadSignalStartContext, LoadSnapshot } from './load_signal.js';
2
- import type { SystemInfo } from './system_status.js';
3
- /**
4
- * A snapshot produced by the built-in CPU signal.
5
- * @internal
6
- */
7
- export interface CpuSnapshot extends LoadSnapshot {
8
- usedRatio: number;
9
- ticks?: {
10
- idle: number;
11
- total: number;
12
- };
13
- }
14
- /**
15
- * Tuning for the built-in **CPU** load signal, as accepted both by {@link CpuLoadSignal} and by the
16
- * {@link LoadSignalsOptions.cpu|`cpu`} shorthand on {@link LoadSignalsOptions}.
17
- */
18
- export interface CpuLoadSignalOptions {
19
- /**
20
- * Maximum ratio of overloaded snapshots in a sample before the CPU counts as overloaded.
21
- * @default 0.4
22
- */
23
- overloadedRatio?: number;
24
- }
25
- /**
26
- * Tracks CPU usage via `SYSTEM_INFO` events and reports overload when the platform or local OS metrics indicate the
27
- * CPU is overloaded.
28
- *
29
- * Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
30
- *
31
- * @category Scaling
32
- */
33
- export declare class CpuLoadSignal implements LoadSignal {
34
- #private;
35
- readonly name = "cpuInfo";
36
- readonly overloadedRatio: number;
37
- constructor(options?: CpuLoadSignalOptions);
38
- start(context: LoadSignalStartContext): Promise<void>;
39
- stop(): Promise<void>;
40
- getSample(sampleDurationMillis?: number): LoadSnapshot[];
41
- /** @internal Records a snapshot from a `SYSTEM_INFO` payload. Exposed for tests. */
42
- handle(systemInfo: SystemInfo): void;
43
- }
@@ -1,47 +0,0 @@
1
- import { EventType } from '../events/event_manager.js';
2
- import { serviceLocator } from '../service_locator.js';
3
- import { SnapshotStore } from './load_signal.js';
4
- /**
5
- * Tracks CPU usage via `SYSTEM_INFO` events and reports overload when the platform or local OS metrics indicate the
6
- * CPU is overloaded.
7
- *
8
- * Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
9
- *
10
- * @category Scaling
11
- */
12
- export class CpuLoadSignal {
13
- name = 'cpuInfo';
14
- overloadedRatio;
15
- #store = new SnapshotStore();
16
- #events;
17
- constructor(options = {}) {
18
- this.overloadedRatio = options.overloadedRatio ?? 0.4;
19
- this.handle = this.handle.bind(this);
20
- }
21
- async start(context) {
22
- this.#store.useSampleWindow(context.maxSampleWindowMillis);
23
- // A new session starts from a clean slate, so it is not judged on measurements from before the downtime.
24
- this.#store.clear();
25
- // Resolved here rather than in the constructor, so an instance built ahead of time (to be wrapped, or shared
26
- // between systems) cannot capture whichever event manager happened to be registered at that moment.
27
- this.#events = serviceLocator.getEventManager();
28
- this.#events.on(EventType.SYSTEM_INFO, this.handle);
29
- }
30
- async stop() {
31
- this.#events?.off(EventType.SYSTEM_INFO, this.handle);
32
- this.#events = undefined;
33
- }
34
- getSample(sampleDurationMillis) {
35
- return this.#store.getSample(sampleDurationMillis);
36
- }
37
- /** @internal Records a snapshot from a `SYSTEM_INFO` payload. Exposed for tests. */
38
- handle(systemInfo) {
39
- const { cpuCurrentUsage, isCpuOverloaded } = systemInfo;
40
- const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
41
- this.#store.push({
42
- createdAt,
43
- isOverloaded: isCpuOverloaded,
44
- usedRatio: Math.ceil(cpuCurrentUsage / 100),
45
- }, createdAt);
46
- }
47
- }
@@ -1,51 +0,0 @@
1
- import type { LoadSignal, LoadSignalStartContext, LoadSnapshot } from './load_signal.js';
2
- /**
3
- * A snapshot produced by the built-in event loop signal.
4
- * @internal
5
- */
6
- export interface EventLoopSnapshot extends LoadSnapshot {
7
- exceededMillis: number;
8
- }
9
- /**
10
- * Tuning for the built-in **event loop** load signal, as accepted both by {@link EventLoopLoadSignal} and by the
11
- * {@link LoadSignalsOptions.eventLoop|`eventLoop`} shorthand on {@link LoadSignalsOptions}.
12
- */
13
- export interface EventLoopLoadSignalOptions {
14
- /**
15
- * Defines the interval of measuring the event loop response time, in seconds.
16
- * @default 0.5
17
- */
18
- snapshotIntervalSecs?: number;
19
- /**
20
- * Maximum allowed delay of the event loop in milliseconds.
21
- * Exceeding this limit overloads the event loop.
22
- * @default 50
23
- */
24
- maxBlockedMillis?: number;
25
- /**
26
- * Maximum ratio of overloaded snapshots in a sample before the event loop counts as overloaded.
27
- * @default 0.6
28
- */
29
- overloadedRatio?: number;
30
- }
31
- /**
32
- * Periodically measures event loop delay and reports overload when the delay exceeds a configured threshold.
33
- *
34
- * Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
35
- *
36
- * @category Scaling
37
- */
38
- export declare class EventLoopLoadSignal implements LoadSignal {
39
- #private;
40
- readonly name = "eventLoopInfo";
41
- readonly overloadedRatio: number;
42
- constructor(options?: EventLoopLoadSignalOptions);
43
- start(context: LoadSignalStartContext): Promise<void>;
44
- stop(): Promise<void>;
45
- getSample(sampleDurationMillis?: number): LoadSnapshot[];
46
- /**
47
- * Records one snapshot: how much later than scheduled this tick ran is how long the loop was blocked.
48
- * @internal Also lets tests drive the measurement without waiting on a timer.
49
- */
50
- handle(intervalCallback: () => unknown): void;
51
- }
@@ -1,60 +0,0 @@
1
- import { betterClearInterval, betterSetInterval } from '@apify/utilities';
2
- import { SnapshotStore } from './load_signal.js';
3
- /**
4
- * Periodically measures event loop delay and reports overload when the delay exceeds a configured threshold.
5
- *
6
- * Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
7
- *
8
- * @category Scaling
9
- */
10
- export class EventLoopLoadSignal {
11
- name = 'eventLoopInfo';
12
- overloadedRatio;
13
- #store = new SnapshotStore();
14
- #intervalMillis;
15
- #maxBlockedMillis;
16
- #interval;
17
- constructor(options = {}) {
18
- this.overloadedRatio = options.overloadedRatio ?? 0.6;
19
- this.#intervalMillis = (options.snapshotIntervalSecs ?? 0.5) * 1000;
20
- this.#maxBlockedMillis = options.maxBlockedMillis ?? 50;
21
- this.handle = this.handle.bind(this);
22
- }
23
- async start(context) {
24
- this.#store.useSampleWindow(context.maxSampleWindowMillis);
25
- // A new session starts from a clean slate, or the downtime gets charged to the event loop: `handle()` measures
26
- // the gap since the previous snapshot, which across a restart is however long the system was stopped.
27
- this.#store.clear();
28
- this.#interval = betterSetInterval(this.handle, this.#intervalMillis);
29
- }
30
- async stop() {
31
- if (this.#interval)
32
- betterClearInterval(this.#interval);
33
- this.#interval = undefined;
34
- }
35
- getSample(sampleDurationMillis) {
36
- return this.#store.getSample(sampleDurationMillis);
37
- }
38
- /**
39
- * Records one snapshot: how much later than scheduled this tick ran is how long the loop was blocked.
40
- * @internal Also lets tests drive the measurement without waiting on a timer.
41
- */
42
- handle(intervalCallback) {
43
- const now = new Date();
44
- const snapshot = {
45
- createdAt: now,
46
- isOverloaded: false,
47
- exceededMillis: 0,
48
- };
49
- const all = this.#store.getAll();
50
- const previousSnapshot = all[all.length - 1];
51
- if (previousSnapshot) {
52
- const delta = now.getTime() - +previousSnapshot.createdAt - this.#intervalMillis;
53
- if (delta > this.#maxBlockedMillis)
54
- snapshot.isOverloaded = true;
55
- snapshot.exceededMillis = Math.max(delta - this.#maxBlockedMillis, 0);
56
- }
57
- this.#store.push(snapshot, now);
58
- intervalCallback();
59
- }
60
- }
@@ -1,9 +0,0 @@
1
- export * from './autoscaled_pool.js';
2
- export * from './concurrency_system.js';
3
- export * from './cpu_load_signal.js';
4
- export * from './event_loop_load_signal.js';
5
- export * from './load_signal.js';
6
- export * from './memory_load_signal.js';
7
- export * from './snapshotter.js';
8
- export * from './storage_backend_load_signal.js';
9
- export * from './system_status.js';
@@ -1,9 +0,0 @@
1
- export * from './autoscaled_pool.js';
2
- export * from './concurrency_system.js';
3
- export * from './cpu_load_signal.js';
4
- export * from './event_loop_load_signal.js';
5
- export * from './load_signal.js';
6
- export * from './memory_load_signal.js';
7
- export * from './snapshotter.js';
8
- export * from './storage_backend_load_signal.js';
9
- export * from './system_status.js';