@crawlee/core 4.0.0-rc.0 → 4.0.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/configuration.d.ts +15 -46
- package/configuration.js +8 -20
- package/errors.d.ts +12 -53
- package/errors.js +13 -66
- package/events/index.d.ts +1 -0
- package/events/local_event_manager.d.ts +0 -7
- package/events/local_event_manager.js +8 -8
- package/events/system_info.d.ts +38 -0
- package/index.d.ts +2 -8
- package/index.js +4 -8
- package/internal.d.ts +8 -0
- package/internal.js +9 -0
- package/log.d.ts +10 -11
- package/log.js +53 -25
- package/memory-storage/memory-storage.d.ts +12 -7
- package/memory-storage/memory-storage.js +45 -17
- package/memory-storage/resource-clients/dataset.d.ts +0 -5
- package/memory-storage/resource-clients/dataset.js +17 -20
- package/memory-storage/resource-clients/key-value-store.d.ts +0 -9
- package/memory-storage/resource-clients/key-value-store.js +11 -33
- package/memory-storage/resource-clients/request-queue.d.ts +0 -22
- package/memory-storage/resource-clients/request-queue.js +57 -53
- package/package.json +16 -18
- package/proxy_configuration.d.ts +20 -23
- package/proxy_configuration.js +18 -12
- package/recoverable_state.d.ts +28 -6
- package/recoverable_state.js +51 -14
- package/request.d.ts +18 -104
- package/request.js +41 -220
- package/serialization.js +3 -3
- package/service_locator.d.ts +3 -0
- package/service_locator.js +2 -0
- package/storages/dataset.d.ts +9 -15
- package/storages/dataset.js +22 -14
- package/storages/index.d.ts +3 -4
- package/storages/index.js +1 -4
- package/storages/key_value_store.d.ts +12 -46
- package/storages/key_value_store.js +31 -47
- package/storages/key_value_store_codec.js +6 -11
- package/storages/request_dedup_cache.d.ts +0 -2
- package/storages/request_dedup_cache.js +8 -8
- package/storages/request_list.d.ts +6 -82
- package/storages/request_list.js +175 -179
- package/storages/request_loader.d.ts +48 -22
- package/storages/request_loader.js +36 -1
- package/storages/request_manager.d.ts +86 -0
- package/storages/request_manager_tandem.d.ts +13 -28
- package/storages/request_manager_tandem.js +46 -43
- package/storages/request_queue.d.ts +19 -49
- package/storages/request_queue.js +92 -88
- package/storages/storage_instance_manager.d.ts +1 -2
- package/storages/storage_instance_manager.js +4 -4
- package/storages/transaction.d.ts +27 -9
- package/storages/transaction.js +56 -11
- package/storages/utils.d.ts +2 -2
- package/validators.d.ts +3 -2
- package/validators.js +3 -2
- package/autoscaling/autoscaled_pool.d.ts +0 -195
- package/autoscaling/autoscaled_pool.js +0 -386
- package/autoscaling/concurrency_system.d.ts +0 -268
- package/autoscaling/concurrency_system.js +0 -362
- package/autoscaling/cpu_load_signal.d.ts +0 -43
- package/autoscaling/cpu_load_signal.js +0 -47
- package/autoscaling/event_loop_load_signal.d.ts +0 -51
- package/autoscaling/event_loop_load_signal.js +0 -60
- package/autoscaling/index.d.ts +0 -9
- package/autoscaling/index.js +0 -9
- package/autoscaling/load_signal.d.ts +0 -100
- package/autoscaling/load_signal.js +0 -105
- package/autoscaling/memory_load_signal.d.ts +0 -47
- package/autoscaling/memory_load_signal.js +0 -106
- package/autoscaling/snapshotter.d.ts +0 -84
- package/autoscaling/snapshotter.js +0 -67
- package/autoscaling/storage_backend_load_signal.d.ts +0 -56
- package/autoscaling/storage_backend_load_signal.js +0 -73
- package/autoscaling/system_status.d.ts +0 -159
- package/autoscaling/system_status.js +0 -139
- package/autoscaling/weighted_avg.d.ts +0 -5
- package/autoscaling/weighted_avg.js +0 -14
- package/cookie_utils.d.ts +0 -44
- package/cookie_utils.js +0 -122
- package/crawlers/context_pipeline.d.ts +0 -70
- package/crawlers/context_pipeline.js +0 -122
- package/crawlers/crawler_commons.d.ts +0 -159
- package/crawlers/error_snapshotter.d.ts +0 -57
- package/crawlers/error_snapshotter.js +0 -117
- package/crawlers/error_tracker.d.ts +0 -54
- package/crawlers/error_tracker.js +0 -308
- package/crawlers/index.d.ts +0 -5
- package/crawlers/index.js +0 -4
- package/crawlers/internals/types.d.ts +0 -7
- package/crawlers/internals/types.js +0 -1
- package/crawlers/statistics.d.ts +0 -328
- package/crawlers/statistics.js +0 -536
- package/enqueue_links/enqueue_links.d.ts +0 -156
- package/enqueue_links/enqueue_links.js +0 -78
- package/enqueue_links/index.d.ts +0 -2
- package/enqueue_links/index.js +0 -2
- package/enqueue_links/shared.d.ts +0 -93
- package/enqueue_links/shared.js +0 -239
- package/http.d.ts +0 -9
- package/http.js +0 -28
- package/router.d.ts +0 -306
- package/router.js +0 -309
- package/session_pool/consts.d.ts +0 -3
- package/session_pool/consts.js +0 -3
- package/session_pool/errors.d.ts +0 -7
- package/session_pool/errors.js +0 -11
- package/session_pool/fingerprint.d.ts +0 -9
- package/session_pool/fingerprint.js +0 -30
- package/session_pool/index.d.ts +0 -4
- package/session_pool/index.js +0 -4
- package/session_pool/session.d.ts +0 -150
- package/session_pool/session.js +0 -220
- package/session_pool/session_pool.d.ts +0 -240
- package/session_pool/session_pool.js +0 -394
- package/storages/sitemap_request_loader.d.ts +0 -201
- package/storages/sitemap_request_loader.js +0 -438
- package/storages/throttling_request_manager.d.ts +0 -239
- package/storages/throttling_request_manager.js +0 -646
- /package/{crawlers/crawler_commons.js → events/system_info.js} +0 -0
|
@@ -1,362 +0,0 @@
|
|
|
1
|
-
import { z } from 'zod';
|
|
2
|
-
import { betterClearInterval, betterSetInterval } from '@apify/utilities';
|
|
3
|
-
import { serviceLocator } from '../service_locator.js';
|
|
4
|
-
import { parseArgument, schemas, validators } from '../validators.js';
|
|
5
|
-
import { Snapshotter } from './snapshotter.js';
|
|
6
|
-
import { SystemStatus } from './system_status.js';
|
|
7
|
-
const concurrencySchema = z.number().int().gte(1).optional();
|
|
8
|
-
// `schemas.anyObject` passes values through by reference, so the load signal instances inside
|
|
9
|
-
// `loadSignals` and class instances like loggers keep their prototype.
|
|
10
|
-
const concurrencySystemOptionsSchema = z.strictObject({
|
|
11
|
-
maxConcurrency: z.number().int().gte(1).default(200),
|
|
12
|
-
minConcurrency: z.number().int().gte(1).default(1),
|
|
13
|
-
desiredConcurrency: z.number().int().gte(1).optional(),
|
|
14
|
-
desiredConcurrencyRatio: z.number().gt(0).lt(1).default(0.9),
|
|
15
|
-
scaleUpStepRatio: z.number().gt(0).lt(1).default(0.05),
|
|
16
|
-
scaleDownStepRatio: z.number().gt(0).lt(1).default(0.05),
|
|
17
|
-
loggingIntervalSecs: schemas.anyNumber
|
|
18
|
-
.refine((value) => value > 0, 'Expected a number greater than 0')
|
|
19
|
-
.nullish()
|
|
20
|
-
.default(60),
|
|
21
|
-
autoscaleIntervalSecs: schemas.anyNumber
|
|
22
|
-
.refine((value) => value > 0, 'Expected a number greater than 0')
|
|
23
|
-
.default(10),
|
|
24
|
-
loadSignals: schemas.anyObject.default(() => ({})),
|
|
25
|
-
snapshotHistorySecs: schemas.anyNumber.refine((value) => value > 0, 'Expected a number greater than 0').optional(),
|
|
26
|
-
currentHistorySecs: schemas.anyNumber.refine((value) => value > 0, 'Expected a number greater than 0').optional(),
|
|
27
|
-
log: validators.logger.default(() => serviceLocator.getLogger()),
|
|
28
|
-
maxTasksPerMinute: z
|
|
29
|
-
.union([z.number().int().gte(1), z.literal(Number.POSITIVE_INFINITY)])
|
|
30
|
-
.default(Number.POSITIVE_INFINITY),
|
|
31
|
-
});
|
|
32
|
-
/**
|
|
33
|
-
* The shareable "governor" behind an {@link AutoscaledPool}: it decides whether there is free compute for one more
|
|
34
|
-
* task by combining live system load (via an internal {@link Snapshotter}) with a concurrency budget it autoscales
|
|
35
|
-
* over time.
|
|
36
|
-
*
|
|
37
|
-
* Sharing one instance between several pools (and therefore several crawlers) caps their *combined* compute, instead
|
|
38
|
-
* of letting each scale independently and oversubscribe the machine.
|
|
39
|
-
*
|
|
40
|
-
* Whoever builds the instance owns its lifecycle: call {@link ConcurrencySystem.start|`start()`} before any
|
|
41
|
-
* borrowing pool runs and {@link ConcurrencySystem.stop|`stop()`} once they are all done (crawlers do this for the
|
|
42
|
-
* default system they build, never for an injected one). Both calls are idempotent, and the first `stop()` tears the
|
|
43
|
-
* system down for every borrower.
|
|
44
|
-
* @category Scaling
|
|
45
|
-
*/
|
|
46
|
-
export class ConcurrencySystem {
|
|
47
|
-
log;
|
|
48
|
-
desiredConcurrencyRatio;
|
|
49
|
-
scaleUpStepRatio;
|
|
50
|
-
scaleDownStepRatio;
|
|
51
|
-
#loggingIntervalMillis;
|
|
52
|
-
#autoscaleIntervalMillis;
|
|
53
|
-
maxTasksPerMinute;
|
|
54
|
-
#minConcurrency;
|
|
55
|
-
#maxConcurrency;
|
|
56
|
-
#desiredConcurrency;
|
|
57
|
-
// kept as TS-private _-prefixed: autoscaled_pool tests write this backing field directly
|
|
58
|
-
_currentConcurrency = 0;
|
|
59
|
-
#lastLoggingTime;
|
|
60
|
-
#tasksPerMinute = Array.from({ length: 60 }, () => 0);
|
|
61
|
-
snapshotter;
|
|
62
|
-
#loadSignals;
|
|
63
|
-
systemStatus;
|
|
64
|
-
#autoscaleInterval;
|
|
65
|
-
#tasksDonePerSecondInterval;
|
|
66
|
-
/** Whether the snapshotter and autoscaling intervals are currently running. */
|
|
67
|
-
#running = false;
|
|
68
|
-
/** The in-flight (or completed) startup, memoized so concurrent `start()` calls await one boot. */
|
|
69
|
-
#startPromise;
|
|
70
|
-
/** Set once per session, so a pool outliving `stop()` is reported once rather than every half second. */
|
|
71
|
-
#warnedAboutQueryWhileStopped = false;
|
|
72
|
-
constructor(options = {}) {
|
|
73
|
-
const { maxConcurrency, minConcurrency, desiredConcurrency, desiredConcurrencyRatio, scaleUpStepRatio, scaleDownStepRatio, loggingIntervalSecs, autoscaleIntervalSecs, loadSignals, snapshotHistorySecs, currentHistorySecs, log, maxTasksPerMinute, } = parseArgument(options, concurrencySystemOptionsSchema, 'ConcurrencySystemOptions');
|
|
74
|
-
this.log = log.child({ prefix: 'ConcurrencySystem' });
|
|
75
|
-
this.desiredConcurrencyRatio = desiredConcurrencyRatio;
|
|
76
|
-
this.scaleUpStepRatio = scaleUpStepRatio;
|
|
77
|
-
this.scaleDownStepRatio = scaleDownStepRatio;
|
|
78
|
-
this.#loggingIntervalMillis = (loggingIntervalSecs ?? 0) * 1000;
|
|
79
|
-
this.#autoscaleIntervalMillis = autoscaleIntervalSecs * 1000;
|
|
80
|
-
this.maxTasksPerMinute = maxTasksPerMinute;
|
|
81
|
-
this.#minConcurrency = minConcurrency;
|
|
82
|
-
this.#maxConcurrency = maxConcurrency;
|
|
83
|
-
this.#desiredConcurrency = desiredConcurrency ?? minConcurrency;
|
|
84
|
-
this.clampDesiredConcurrency();
|
|
85
|
-
this.autoscale = this.autoscale.bind(this);
|
|
86
|
-
this.incrementTasksDonePerSecond = this.incrementTasksDonePerSecond.bind(this);
|
|
87
|
-
// The built-in signals are collected by the snapshotter; custom ones are simply evaluated alongside them.
|
|
88
|
-
const { custom: customLoadSignals = [], ...builtinSignalOptions } = loadSignals;
|
|
89
|
-
this.snapshotter = new Snapshotter(builtinSignalOptions);
|
|
90
|
-
this.#loadSignals = customLoadSignals;
|
|
91
|
-
this.systemStatus = new SystemStatus({
|
|
92
|
-
snapshotter: this.snapshotter,
|
|
93
|
-
loadSignals: customLoadSignals,
|
|
94
|
-
currentHistorySecs,
|
|
95
|
-
// Both windows are requested from the signals explicitly, so a signal's own retention can neither widen
|
|
96
|
-
// nor (given the start context below) narrow what it contributes.
|
|
97
|
-
historySecs: snapshotHistorySecs,
|
|
98
|
-
});
|
|
99
|
-
}
|
|
100
|
-
/**
|
|
101
|
-
* Gets the minimum number of tasks running in parallel.
|
|
102
|
-
*/
|
|
103
|
-
get minConcurrency() {
|
|
104
|
-
return this.#minConcurrency;
|
|
105
|
-
}
|
|
106
|
-
/**
|
|
107
|
-
* Sets the minimum number of tasks running in parallel.
|
|
108
|
-
*
|
|
109
|
-
* *WARNING:* If you set this value too high with respect to the available system memory and CPU, your code might run extremely slow or crash.
|
|
110
|
-
* If you're not sure, just keep the default value and the concurrency will scale up automatically.
|
|
111
|
-
*/
|
|
112
|
-
set minConcurrency(value) {
|
|
113
|
-
parseArgument(value, concurrencySchema);
|
|
114
|
-
this.#minConcurrency = value;
|
|
115
|
-
this.clampDesiredConcurrency();
|
|
116
|
-
}
|
|
117
|
-
/**
|
|
118
|
-
* Gets the maximum number of tasks running in parallel.
|
|
119
|
-
*/
|
|
120
|
-
get maxConcurrency() {
|
|
121
|
-
return this.#maxConcurrency;
|
|
122
|
-
}
|
|
123
|
-
/**
|
|
124
|
-
* Sets the maximum number of tasks running in parallel. Lowering it below the current
|
|
125
|
-
* {@link ConcurrencySystem.desiredConcurrency|`desiredConcurrency`} pulls that down to the new ceiling too, so
|
|
126
|
-
* the change takes effect immediately (in-flight tasks are never cancelled — the budget simply drains to the new
|
|
127
|
-
* limit as they settle).
|
|
128
|
-
*/
|
|
129
|
-
set maxConcurrency(value) {
|
|
130
|
-
parseArgument(value, concurrencySchema);
|
|
131
|
-
this.#maxConcurrency = value;
|
|
132
|
-
this.clampDesiredConcurrency();
|
|
133
|
-
}
|
|
134
|
-
/**
|
|
135
|
-
* Gets the desired concurrency for the system,
|
|
136
|
-
* which is an estimated number of parallel tasks that the system can currently support.
|
|
137
|
-
*/
|
|
138
|
-
get desiredConcurrency() {
|
|
139
|
-
return this.#desiredConcurrency;
|
|
140
|
-
}
|
|
141
|
-
/**
|
|
142
|
-
* Sets the desired concurrency for the system, i.e. the number of tasks that should be running
|
|
143
|
-
* in parallel if there's large enough supply of tasks.
|
|
144
|
-
*/
|
|
145
|
-
set desiredConcurrency(value) {
|
|
146
|
-
parseArgument(value, concurrencySchema);
|
|
147
|
-
this.#desiredConcurrency = value;
|
|
148
|
-
this.clampDesiredConcurrency();
|
|
149
|
-
}
|
|
150
|
-
/**
|
|
151
|
-
* Re-establishes `minConcurrency <= desiredConcurrency <= maxConcurrency` after any of the three is retuned.
|
|
152
|
-
* Dispatch gates on the desired value alone, so one stranded above `maxConcurrency` would make the ceiling
|
|
153
|
-
* meaningless. A contradictory pair (`minConcurrency > maxConcurrency`) resolves in favour of the maximum, since
|
|
154
|
-
* that is the limit callers set in order to protect something.
|
|
155
|
-
*/
|
|
156
|
-
clampDesiredConcurrency() {
|
|
157
|
-
const atLeastMin = Math.max(this.#desiredConcurrency, this.#minConcurrency);
|
|
158
|
-
this.#desiredConcurrency = Math.min(atLeastMin, this.#maxConcurrency);
|
|
159
|
-
}
|
|
160
|
-
get currentConcurrency() {
|
|
161
|
-
return this._currentConcurrency;
|
|
162
|
-
}
|
|
163
|
-
/** Whether the system is currently monitoring load and autoscaling the budget. */
|
|
164
|
-
get isRunning() {
|
|
165
|
-
return this.#running;
|
|
166
|
-
}
|
|
167
|
-
/**
|
|
168
|
-
* Boots the underlying snapshotter and the autoscaling interval. Idempotent, so a shared system isn't restarted
|
|
169
|
-
* when handed to another consumer; concurrent callers await one startup. Rejects, leaving nothing running, if a
|
|
170
|
-
* signal fails to start.
|
|
171
|
-
*/
|
|
172
|
-
async start() {
|
|
173
|
-
// Unwound and dropped again on failure, so a later `start()` retries instead of resolving instantly against
|
|
174
|
-
// a system that is down.
|
|
175
|
-
this.#startPromise ??= this.boot().catch(async (error) => {
|
|
176
|
-
this.#startPromise = undefined;
|
|
177
|
-
await this.shutDown();
|
|
178
|
-
throw error;
|
|
179
|
-
});
|
|
180
|
-
await this.#startPromise;
|
|
181
|
-
}
|
|
182
|
-
async boot() {
|
|
183
|
-
// Per-session measurement state, reset so a restarted system isn't judged on the previous session. The
|
|
184
|
-
// per-minute window matters most: its ageing interval is cleared while we are down, so starts from before an
|
|
185
|
-
// arbitrarily long stop would otherwise still count against "this minute" and trip the cap immediately.
|
|
186
|
-
this.#tasksPerMinute = Array.from({ length: 60 }, () => 0);
|
|
187
|
-
this.#lastLoggingTime = undefined;
|
|
188
|
-
this.#warnedAboutQueryWhileStopped = false;
|
|
189
|
-
// Signals are told how much history to keep when they start: exactly the longest window they will be sampled
|
|
190
|
-
// over, so nobody has to guess a retention value that matches this system's configuration.
|
|
191
|
-
const startContext = { maxSampleWindowMillis: this.systemStatus.maxSampleWindowMillis };
|
|
192
|
-
await this.snapshotter.start(startContext);
|
|
193
|
-
await Promise.all(this.#loadSignals.map(async (s) => s.start(startContext)));
|
|
194
|
-
this.#autoscaleInterval = betterSetInterval(this.autoscale, this.#autoscaleIntervalMillis);
|
|
195
|
-
if (this.maxTasksPerMinute !== Infinity) {
|
|
196
|
-
this.#tasksDonePerSecondInterval = betterSetInterval(this.incrementTasksDonePerSecond, 1000);
|
|
197
|
-
}
|
|
198
|
-
// Last, so `isRunning` never claims a system whose signals aren't collecting yet.
|
|
199
|
-
this.#running = true;
|
|
200
|
-
}
|
|
201
|
-
/**
|
|
202
|
-
* Stops the snapshotter and intervals. Idempotent and safe to call even if the system was never started.
|
|
203
|
-
*/
|
|
204
|
-
async stop() {
|
|
205
|
-
if (this.#startPromise === undefined) {
|
|
206
|
-
return;
|
|
207
|
-
}
|
|
208
|
-
// Waited out rather than interrupted, or the intervals a starting signal is about to register outlive us.
|
|
209
|
-
await this.#startPromise.catch(() => { });
|
|
210
|
-
this.#startPromise = undefined;
|
|
211
|
-
this.#running = false;
|
|
212
|
-
await this.shutDown();
|
|
213
|
-
}
|
|
214
|
-
async shutDown() {
|
|
215
|
-
if (this.#autoscaleInterval)
|
|
216
|
-
betterClearInterval(this.#autoscaleInterval);
|
|
217
|
-
if (this.#tasksDonePerSecondInterval)
|
|
218
|
-
betterClearInterval(this.#tasksDonePerSecondInterval);
|
|
219
|
-
await this.snapshotter.stop();
|
|
220
|
-
await Promise.all(this.#loadSignals.map(async (s) => s.stop()));
|
|
221
|
-
}
|
|
222
|
-
/**
|
|
223
|
-
* Reports, once per session, that capacity is being queried on a system that isn't running — a mistake nothing
|
|
224
|
-
* else catches, since {@link AutoscaledPool.run|`pool.run()`} only checks
|
|
225
|
-
* {@link ConcurrencySystem.isRunning|`isRunning`} on the way in. Both the overload verdict and
|
|
226
|
-
* `desiredConcurrency` are frozen at that point, so the borrowing pool would otherwise just quietly mis-scale.
|
|
227
|
-
*/
|
|
228
|
-
warnIfNotRunning() {
|
|
229
|
-
if (this.#running || this.#warnedAboutQueryWhileStopped) {
|
|
230
|
-
return;
|
|
231
|
-
}
|
|
232
|
-
this.#warnedAboutQueryWhileStopped = true;
|
|
233
|
-
this.log.warning('Capacity is being queried on a ConcurrencySystem that is not running, so system load is no longer being ' +
|
|
234
|
-
'monitored and the concurrency will no longer be adjusted. Whoever creates a ConcurrencySystem owns ' +
|
|
235
|
-
'its lifecycle: call `await concurrencySystem.stop()` only once every pool and crawler borrowing it ' +
|
|
236
|
-
'has finished.');
|
|
237
|
-
}
|
|
238
|
-
/**
|
|
239
|
-
* May **one more** task start right now? Returns `false` when the shared budget is spent (desired concurrency
|
|
240
|
-
* reached) or when the machine is overloaded past `minConcurrency`.
|
|
241
|
-
*
|
|
242
|
-
* One budget for the whole machine, so the asking consumer is ignored — and therefore optional here, unlike in the
|
|
243
|
-
* interface, letting the answer be queried directly.
|
|
244
|
-
*/
|
|
245
|
-
hasCapacityForTask(_consumer) {
|
|
246
|
-
this.warnIfNotRunning();
|
|
247
|
-
if (this._currentConcurrency >= this.#desiredConcurrency) {
|
|
248
|
-
this.log.perf('Task will not run. Desired concurrency achieved.');
|
|
249
|
-
return false;
|
|
250
|
-
}
|
|
251
|
-
const currentStatus = this.systemStatus.getCurrentStatus();
|
|
252
|
-
const { isSystemIdle } = currentStatus;
|
|
253
|
-
if (!isSystemIdle && this._currentConcurrency >= this.#minConcurrency) {
|
|
254
|
-
this.log.perf('Task will not be run. System is overloaded.', currentStatus);
|
|
255
|
-
return false;
|
|
256
|
-
}
|
|
257
|
-
return true;
|
|
258
|
-
}
|
|
259
|
-
/** Whether the per-minute task cap has been reached. */
|
|
260
|
-
get isOverMaxRequestLimit() {
|
|
261
|
-
if (this.maxTasksPerMinute === Infinity) {
|
|
262
|
-
return false;
|
|
263
|
-
}
|
|
264
|
-
return this.#tasksPerMinute.reduce((acc, curr) => acc + curr, 0) >= this.maxTasksPerMinute;
|
|
265
|
-
}
|
|
266
|
-
/**
|
|
267
|
-
* Atomically books a task against the shared budget: re-checks
|
|
268
|
-
* {@link ConcurrencySystem.hasCapacityForTask|`hasCapacityForTask()`} plus the per-minute task cap and
|
|
269
|
-
* increments the current concurrency in one synchronous step, returning `false` (without booking) when there is no
|
|
270
|
-
* room. Call right before the task actually runs.
|
|
271
|
-
*
|
|
272
|
-
* The cap is enforced here rather than in the pre-check so that an empty queue never blocks the pool for a whole
|
|
273
|
-
* extra minute.
|
|
274
|
-
*/
|
|
275
|
-
tryRegisterTaskStart(consumer) {
|
|
276
|
-
if (!this.hasCapacityForTask(consumer)) {
|
|
277
|
-
return false;
|
|
278
|
-
}
|
|
279
|
-
if (this.isOverMaxRequestLimit) {
|
|
280
|
-
this.log.perf('Task will not run. Maximum tasks per minute reached.');
|
|
281
|
-
return false;
|
|
282
|
-
}
|
|
283
|
-
this._currentConcurrency++;
|
|
284
|
-
this.#tasksPerMinute[0]++;
|
|
285
|
-
return true;
|
|
286
|
-
}
|
|
287
|
-
/** Returns a slot to the shared budget, whoever booked it. */
|
|
288
|
-
registerTaskEnd(_consumer) {
|
|
289
|
-
this._currentConcurrency--;
|
|
290
|
-
}
|
|
291
|
-
/**
|
|
292
|
-
* What the system currently makes of the machine: the per-signal overload verdicts, evaluated over the
|
|
293
|
-
* task-gating window, exactly as {@link ConcurrencySystem.hasCapacityForTask|`hasCapacityForTask()`} sees them.
|
|
294
|
-
* The one public window into load monitoring — useful for answering *why* a crawl is not scaling up.
|
|
295
|
-
*/
|
|
296
|
-
getCurrentStatus() {
|
|
297
|
-
return this.systemStatus.getCurrentStatus();
|
|
298
|
-
}
|
|
299
|
-
/**
|
|
300
|
-
* Evaluates the historical system status and scales the shared desired concurrency up or down accordingly. Driven
|
|
301
|
-
* by the autoscaling interval started in {@link ConcurrencySystem.start|`start()`}.
|
|
302
|
-
*/
|
|
303
|
-
autoscale(intervalCallback) {
|
|
304
|
-
if (this.isOverMaxRequestLimit)
|
|
305
|
-
return intervalCallback();
|
|
306
|
-
const systemStatus = this.systemStatus.getHistoricalStatus();
|
|
307
|
-
const { isSystemIdle } = systemStatus;
|
|
308
|
-
const weAreNotAtMax = this.#desiredConcurrency < this.#maxConcurrency;
|
|
309
|
-
const minCurrentConcurrency = Math.floor(this.#desiredConcurrency * this.desiredConcurrencyRatio);
|
|
310
|
-
const weAreReachingDesiredConcurrency = this._currentConcurrency >= minCurrentConcurrency;
|
|
311
|
-
if (isSystemIdle && weAreNotAtMax && weAreReachingDesiredConcurrency)
|
|
312
|
-
this.scaleUp(systemStatus);
|
|
313
|
-
const isSystemOverloaded = !isSystemIdle;
|
|
314
|
-
const weAreNotAtMin = this.#desiredConcurrency > this.#minConcurrency;
|
|
315
|
-
if (isSystemOverloaded && weAreNotAtMin)
|
|
316
|
-
this.scaleDown(systemStatus);
|
|
317
|
-
if (this.#loggingIntervalMillis > 0) {
|
|
318
|
-
const now = Date.now();
|
|
319
|
-
if (this.#lastLoggingTime == null) {
|
|
320
|
-
this.#lastLoggingTime = now;
|
|
321
|
-
}
|
|
322
|
-
else if (now > this.#lastLoggingTime + this.#loggingIntervalMillis) {
|
|
323
|
-
this.#lastLoggingTime = now;
|
|
324
|
-
this.log.info('state', {
|
|
325
|
-
currentConcurrency: this._currentConcurrency,
|
|
326
|
-
desiredConcurrency: this.#desiredConcurrency,
|
|
327
|
-
systemStatus,
|
|
328
|
-
});
|
|
329
|
-
}
|
|
330
|
-
}
|
|
331
|
-
return intervalCallback();
|
|
332
|
-
}
|
|
333
|
-
/**
|
|
334
|
-
* Scales the system up by increasing the desired concurrency by the scaleUpStepRatio.
|
|
335
|
-
*/
|
|
336
|
-
scaleUp(systemStatus) {
|
|
337
|
-
const step = Math.ceil(this.#desiredConcurrency * this.scaleUpStepRatio);
|
|
338
|
-
this.#desiredConcurrency = Math.min(this.#maxConcurrency, this.#desiredConcurrency + step);
|
|
339
|
-
this.log.debug('scaling up', {
|
|
340
|
-
oldConcurrency: this.#desiredConcurrency - step,
|
|
341
|
-
newConcurrency: this.#desiredConcurrency,
|
|
342
|
-
systemStatus,
|
|
343
|
-
});
|
|
344
|
-
}
|
|
345
|
-
/**
|
|
346
|
-
* Scales the system down by decreasing the desired concurrency by the scaleDownStepRatio.
|
|
347
|
-
*/
|
|
348
|
-
scaleDown(systemStatus) {
|
|
349
|
-
const step = Math.ceil(this.#desiredConcurrency * this.scaleDownStepRatio);
|
|
350
|
-
this.#desiredConcurrency = Math.max(this.#minConcurrency, this.#desiredConcurrency - step);
|
|
351
|
-
this.log.debug('scaling down', {
|
|
352
|
-
oldConcurrency: this.#desiredConcurrency + step,
|
|
353
|
-
newConcurrency: this.#desiredConcurrency,
|
|
354
|
-
systemStatus,
|
|
355
|
-
});
|
|
356
|
-
}
|
|
357
|
-
incrementTasksDonePerSecond(intervalCallback) {
|
|
358
|
-
this.#tasksPerMinute.unshift(0);
|
|
359
|
-
this.#tasksPerMinute.pop();
|
|
360
|
-
return intervalCallback();
|
|
361
|
-
}
|
|
362
|
-
}
|
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
import type { LoadSignal, LoadSignalStartContext, LoadSnapshot } from './load_signal.js';
|
|
2
|
-
import type { SystemInfo } from './system_status.js';
|
|
3
|
-
/**
|
|
4
|
-
* A snapshot produced by the built-in CPU signal.
|
|
5
|
-
* @internal
|
|
6
|
-
*/
|
|
7
|
-
export interface CpuSnapshot extends LoadSnapshot {
|
|
8
|
-
usedRatio: number;
|
|
9
|
-
ticks?: {
|
|
10
|
-
idle: number;
|
|
11
|
-
total: number;
|
|
12
|
-
};
|
|
13
|
-
}
|
|
14
|
-
/**
|
|
15
|
-
* Tuning for the built-in **CPU** load signal, as accepted both by {@link CpuLoadSignal} and by the
|
|
16
|
-
* {@link LoadSignalsOptions.cpu|`cpu`} shorthand on {@link LoadSignalsOptions}.
|
|
17
|
-
*/
|
|
18
|
-
export interface CpuLoadSignalOptions {
|
|
19
|
-
/**
|
|
20
|
-
* Maximum ratio of overloaded snapshots in a sample before the CPU counts as overloaded.
|
|
21
|
-
* @default 0.4
|
|
22
|
-
*/
|
|
23
|
-
overloadedRatio?: number;
|
|
24
|
-
}
|
|
25
|
-
/**
|
|
26
|
-
* Tracks CPU usage via `SYSTEM_INFO` events and reports overload when the platform or local OS metrics indicate the
|
|
27
|
-
* CPU is overloaded.
|
|
28
|
-
*
|
|
29
|
-
* Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
|
|
30
|
-
*
|
|
31
|
-
* @category Scaling
|
|
32
|
-
*/
|
|
33
|
-
export declare class CpuLoadSignal implements LoadSignal {
|
|
34
|
-
#private;
|
|
35
|
-
readonly name = "cpuInfo";
|
|
36
|
-
readonly overloadedRatio: number;
|
|
37
|
-
constructor(options?: CpuLoadSignalOptions);
|
|
38
|
-
start(context: LoadSignalStartContext): Promise<void>;
|
|
39
|
-
stop(): Promise<void>;
|
|
40
|
-
getSample(sampleDurationMillis?: number): LoadSnapshot[];
|
|
41
|
-
/** @internal Records a snapshot from a `SYSTEM_INFO` payload. Exposed for tests. */
|
|
42
|
-
handle(systemInfo: SystemInfo): void;
|
|
43
|
-
}
|
|
@@ -1,47 +0,0 @@
|
|
|
1
|
-
import { EventType } from '../events/event_manager.js';
|
|
2
|
-
import { serviceLocator } from '../service_locator.js';
|
|
3
|
-
import { SnapshotStore } from './load_signal.js';
|
|
4
|
-
/**
|
|
5
|
-
* Tracks CPU usage via `SYSTEM_INFO` events and reports overload when the platform or local OS metrics indicate the
|
|
6
|
-
* CPU is overloaded.
|
|
7
|
-
*
|
|
8
|
-
* Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
|
|
9
|
-
*
|
|
10
|
-
* @category Scaling
|
|
11
|
-
*/
|
|
12
|
-
export class CpuLoadSignal {
|
|
13
|
-
name = 'cpuInfo';
|
|
14
|
-
overloadedRatio;
|
|
15
|
-
#store = new SnapshotStore();
|
|
16
|
-
#events;
|
|
17
|
-
constructor(options = {}) {
|
|
18
|
-
this.overloadedRatio = options.overloadedRatio ?? 0.4;
|
|
19
|
-
this.handle = this.handle.bind(this);
|
|
20
|
-
}
|
|
21
|
-
async start(context) {
|
|
22
|
-
this.#store.useSampleWindow(context.maxSampleWindowMillis);
|
|
23
|
-
// A new session starts from a clean slate, so it is not judged on measurements from before the downtime.
|
|
24
|
-
this.#store.clear();
|
|
25
|
-
// Resolved here rather than in the constructor, so an instance built ahead of time (to be wrapped, or shared
|
|
26
|
-
// between systems) cannot capture whichever event manager happened to be registered at that moment.
|
|
27
|
-
this.#events = serviceLocator.getEventManager();
|
|
28
|
-
this.#events.on(EventType.SYSTEM_INFO, this.handle);
|
|
29
|
-
}
|
|
30
|
-
async stop() {
|
|
31
|
-
this.#events?.off(EventType.SYSTEM_INFO, this.handle);
|
|
32
|
-
this.#events = undefined;
|
|
33
|
-
}
|
|
34
|
-
getSample(sampleDurationMillis) {
|
|
35
|
-
return this.#store.getSample(sampleDurationMillis);
|
|
36
|
-
}
|
|
37
|
-
/** @internal Records a snapshot from a `SYSTEM_INFO` payload. Exposed for tests. */
|
|
38
|
-
handle(systemInfo) {
|
|
39
|
-
const { cpuCurrentUsage, isCpuOverloaded } = systemInfo;
|
|
40
|
-
const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
|
|
41
|
-
this.#store.push({
|
|
42
|
-
createdAt,
|
|
43
|
-
isOverloaded: isCpuOverloaded,
|
|
44
|
-
usedRatio: Math.ceil(cpuCurrentUsage / 100),
|
|
45
|
-
}, createdAt);
|
|
46
|
-
}
|
|
47
|
-
}
|
|
@@ -1,51 +0,0 @@
|
|
|
1
|
-
import type { LoadSignal, LoadSignalStartContext, LoadSnapshot } from './load_signal.js';
|
|
2
|
-
/**
|
|
3
|
-
* A snapshot produced by the built-in event loop signal.
|
|
4
|
-
* @internal
|
|
5
|
-
*/
|
|
6
|
-
export interface EventLoopSnapshot extends LoadSnapshot {
|
|
7
|
-
exceededMillis: number;
|
|
8
|
-
}
|
|
9
|
-
/**
|
|
10
|
-
* Tuning for the built-in **event loop** load signal, as accepted both by {@link EventLoopLoadSignal} and by the
|
|
11
|
-
* {@link LoadSignalsOptions.eventLoop|`eventLoop`} shorthand on {@link LoadSignalsOptions}.
|
|
12
|
-
*/
|
|
13
|
-
export interface EventLoopLoadSignalOptions {
|
|
14
|
-
/**
|
|
15
|
-
* Defines the interval of measuring the event loop response time, in seconds.
|
|
16
|
-
* @default 0.5
|
|
17
|
-
*/
|
|
18
|
-
snapshotIntervalSecs?: number;
|
|
19
|
-
/**
|
|
20
|
-
* Maximum allowed delay of the event loop in milliseconds.
|
|
21
|
-
* Exceeding this limit overloads the event loop.
|
|
22
|
-
* @default 50
|
|
23
|
-
*/
|
|
24
|
-
maxBlockedMillis?: number;
|
|
25
|
-
/**
|
|
26
|
-
* Maximum ratio of overloaded snapshots in a sample before the event loop counts as overloaded.
|
|
27
|
-
* @default 0.6
|
|
28
|
-
*/
|
|
29
|
-
overloadedRatio?: number;
|
|
30
|
-
}
|
|
31
|
-
/**
|
|
32
|
-
* Periodically measures event loop delay and reports overload when the delay exceeds a configured threshold.
|
|
33
|
-
*
|
|
34
|
-
* Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
|
|
35
|
-
*
|
|
36
|
-
* @category Scaling
|
|
37
|
-
*/
|
|
38
|
-
export declare class EventLoopLoadSignal implements LoadSignal {
|
|
39
|
-
#private;
|
|
40
|
-
readonly name = "eventLoopInfo";
|
|
41
|
-
readonly overloadedRatio: number;
|
|
42
|
-
constructor(options?: EventLoopLoadSignalOptions);
|
|
43
|
-
start(context: LoadSignalStartContext): Promise<void>;
|
|
44
|
-
stop(): Promise<void>;
|
|
45
|
-
getSample(sampleDurationMillis?: number): LoadSnapshot[];
|
|
46
|
-
/**
|
|
47
|
-
* Records one snapshot: how much later than scheduled this tick ran is how long the loop was blocked.
|
|
48
|
-
* @internal Also lets tests drive the measurement without waiting on a timer.
|
|
49
|
-
*/
|
|
50
|
-
handle(intervalCallback: () => unknown): void;
|
|
51
|
-
}
|
|
@@ -1,60 +0,0 @@
|
|
|
1
|
-
import { betterClearInterval, betterSetInterval } from '@apify/utilities';
|
|
2
|
-
import { SnapshotStore } from './load_signal.js';
|
|
3
|
-
/**
|
|
4
|
-
* Periodically measures event loop delay and reports overload when the delay exceeds a configured threshold.
|
|
5
|
-
*
|
|
6
|
-
* Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
|
|
7
|
-
*
|
|
8
|
-
* @category Scaling
|
|
9
|
-
*/
|
|
10
|
-
export class EventLoopLoadSignal {
|
|
11
|
-
name = 'eventLoopInfo';
|
|
12
|
-
overloadedRatio;
|
|
13
|
-
#store = new SnapshotStore();
|
|
14
|
-
#intervalMillis;
|
|
15
|
-
#maxBlockedMillis;
|
|
16
|
-
#interval;
|
|
17
|
-
constructor(options = {}) {
|
|
18
|
-
this.overloadedRatio = options.overloadedRatio ?? 0.6;
|
|
19
|
-
this.#intervalMillis = (options.snapshotIntervalSecs ?? 0.5) * 1000;
|
|
20
|
-
this.#maxBlockedMillis = options.maxBlockedMillis ?? 50;
|
|
21
|
-
this.handle = this.handle.bind(this);
|
|
22
|
-
}
|
|
23
|
-
async start(context) {
|
|
24
|
-
this.#store.useSampleWindow(context.maxSampleWindowMillis);
|
|
25
|
-
// A new session starts from a clean slate, or the downtime gets charged to the event loop: `handle()` measures
|
|
26
|
-
// the gap since the previous snapshot, which across a restart is however long the system was stopped.
|
|
27
|
-
this.#store.clear();
|
|
28
|
-
this.#interval = betterSetInterval(this.handle, this.#intervalMillis);
|
|
29
|
-
}
|
|
30
|
-
async stop() {
|
|
31
|
-
if (this.#interval)
|
|
32
|
-
betterClearInterval(this.#interval);
|
|
33
|
-
this.#interval = undefined;
|
|
34
|
-
}
|
|
35
|
-
getSample(sampleDurationMillis) {
|
|
36
|
-
return this.#store.getSample(sampleDurationMillis);
|
|
37
|
-
}
|
|
38
|
-
/**
|
|
39
|
-
* Records one snapshot: how much later than scheduled this tick ran is how long the loop was blocked.
|
|
40
|
-
* @internal Also lets tests drive the measurement without waiting on a timer.
|
|
41
|
-
*/
|
|
42
|
-
handle(intervalCallback) {
|
|
43
|
-
const now = new Date();
|
|
44
|
-
const snapshot = {
|
|
45
|
-
createdAt: now,
|
|
46
|
-
isOverloaded: false,
|
|
47
|
-
exceededMillis: 0,
|
|
48
|
-
};
|
|
49
|
-
const all = this.#store.getAll();
|
|
50
|
-
const previousSnapshot = all[all.length - 1];
|
|
51
|
-
if (previousSnapshot) {
|
|
52
|
-
const delta = now.getTime() - +previousSnapshot.createdAt - this.#intervalMillis;
|
|
53
|
-
if (delta > this.#maxBlockedMillis)
|
|
54
|
-
snapshot.isOverloaded = true;
|
|
55
|
-
snapshot.exceededMillis = Math.max(delta - this.#maxBlockedMillis, 0);
|
|
56
|
-
}
|
|
57
|
-
this.#store.push(snapshot, now);
|
|
58
|
-
intervalCallback();
|
|
59
|
-
}
|
|
60
|
-
}
|
package/autoscaling/index.d.ts
DELETED
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
export * from './autoscaled_pool.js';
|
|
2
|
-
export * from './concurrency_system.js';
|
|
3
|
-
export * from './cpu_load_signal.js';
|
|
4
|
-
export * from './event_loop_load_signal.js';
|
|
5
|
-
export * from './load_signal.js';
|
|
6
|
-
export * from './memory_load_signal.js';
|
|
7
|
-
export * from './snapshotter.js';
|
|
8
|
-
export * from './storage_backend_load_signal.js';
|
|
9
|
-
export * from './system_status.js';
|
package/autoscaling/index.js
DELETED
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
export * from './autoscaled_pool.js';
|
|
2
|
-
export * from './concurrency_system.js';
|
|
3
|
-
export * from './cpu_load_signal.js';
|
|
4
|
-
export * from './event_loop_load_signal.js';
|
|
5
|
-
export * from './load_signal.js';
|
|
6
|
-
export * from './memory_load_signal.js';
|
|
7
|
-
export * from './snapshotter.js';
|
|
8
|
-
export * from './storage_backend_load_signal.js';
|
|
9
|
-
export * from './system_status.js';
|