@crawlee/core 3.16.1-beta.34 → 3.16.1-beta.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/autoscaling/autoscaled_pool.d.ts +2 -0
- package/autoscaling/autoscaled_pool.d.ts.map +1 -1
- package/autoscaling/autoscaled_pool.js +10 -0
- package/autoscaling/autoscaled_pool.js.map +1 -1
- package/autoscaling/client_load_signal.d.ts +26 -0
- package/autoscaling/client_load_signal.d.ts.map +1 -0
- package/autoscaling/client_load_signal.js +40 -0
- package/autoscaling/client_load_signal.js.map +1 -0
- package/autoscaling/cpu_load_signal.d.ts +29 -0
- package/autoscaling/cpu_load_signal.d.ts.map +1 -0
- package/autoscaling/cpu_load_signal.js +27 -0
- package/autoscaling/cpu_load_signal.js.map +1 -0
- package/autoscaling/event_loop_load_signal.d.ts +24 -0
- package/autoscaling/event_loop_load_signal.d.ts.map +1 -0
- package/autoscaling/event_loop_load_signal.js +39 -0
- package/autoscaling/event_loop_load_signal.js.map +1 -0
- package/autoscaling/index.d.ts +5 -0
- package/autoscaling/index.d.ts.map +1 -1
- package/autoscaling/index.js +5 -0
- package/autoscaling/index.js.map +1 -1
- package/autoscaling/load_signal.d.ts +100 -0
- package/autoscaling/load_signal.d.ts.map +1 -0
- package/autoscaling/load_signal.js +151 -0
- package/autoscaling/load_signal.js.map +1 -0
- package/autoscaling/memory_load_signal.d.ts +44 -0
- package/autoscaling/memory_load_signal.d.ts.map +1 -0
- package/autoscaling/memory_load_signal.js +159 -0
- package/autoscaling/memory_load_signal.js.map +1 -0
- package/autoscaling/snapshotter.d.ts +27 -63
- package/autoscaling/snapshotter.d.ts.map +1 -1
- package/autoscaling/snapshotter.js +77 -243
- package/autoscaling/snapshotter.js.map +1 -1
- package/autoscaling/system_status.d.ts +20 -32
- package/autoscaling/system_status.d.ts.map +1 -1
- package/autoscaling/system_status.js +46 -92
- package/autoscaling/system_status.js.map +1 -1
- package/index.mjs +6 -0
- package/package.json +5 -5
|
@@ -2,14 +2,13 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.Snapshotter = void 0;
|
|
4
4
|
const tslib_1 = require("tslib");
|
|
5
|
-
const utils_1 = require("@crawlee/utils");
|
|
6
5
|
const ow_1 = tslib_1.__importDefault(require("ow"));
|
|
7
|
-
const utilities_1 = require("@apify/utilities");
|
|
8
6
|
const configuration_1 = require("../configuration");
|
|
9
7
|
const log_1 = require("../log");
|
|
10
|
-
const
|
|
11
|
-
const
|
|
12
|
-
const
|
|
8
|
+
const client_load_signal_1 = require("./client_load_signal");
|
|
9
|
+
const cpu_load_signal_1 = require("./cpu_load_signal");
|
|
10
|
+
const event_loop_load_signal_1 = require("./event_loop_load_signal");
|
|
11
|
+
const memory_load_signal_1 = require("./memory_load_signal");
|
|
13
12
|
/**
|
|
14
13
|
* Creates snapshots of system resources at given intervals and marks the resource
|
|
15
14
|
* as either overloaded or not during the last interval. Keeps a history of the snapshots.
|
|
@@ -33,9 +32,30 @@ const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10000;
|
|
|
33
32
|
*
|
|
34
33
|
* Client becomes overloaded when rate limit errors (429 - Too Many Requests),
|
|
35
34
|
* typically received from the request queue, exceed the set limit within the set interval.
|
|
35
|
+
*
|
|
36
36
|
* @category Scaling
|
|
37
37
|
*/
|
|
38
38
|
class Snapshotter {
|
|
39
|
+
/**
|
|
40
|
+
* Returns the four built-in signals as an array, so `SystemStatus` can
|
|
41
|
+
* iterate them alongside any custom `LoadSignal` instances.
|
|
42
|
+
*/
|
|
43
|
+
getLoadSignals() {
|
|
44
|
+
return [this.memorySignal, this.eventLoopSignal, this.cpuSignal, this.clientSignal];
|
|
45
|
+
}
|
|
46
|
+
// Legacy public properties kept for backward compat (tests read these directly)
|
|
47
|
+
get cpuSnapshots() {
|
|
48
|
+
return this.cpuSignal.store.getAll();
|
|
49
|
+
}
|
|
50
|
+
get eventLoopSnapshots() {
|
|
51
|
+
return this.eventLoopSignal.store.getAll();
|
|
52
|
+
}
|
|
53
|
+
get memorySnapshots() {
|
|
54
|
+
return this.memorySignal.getMemorySnapshots();
|
|
55
|
+
}
|
|
56
|
+
get clientSnapshots() {
|
|
57
|
+
return this.clientSignal.store.getAll();
|
|
58
|
+
}
|
|
39
59
|
/**
|
|
40
60
|
* @param [options] All `Snapshotter` configuration options.
|
|
41
61
|
*/
|
|
@@ -64,96 +84,30 @@ class Snapshotter {
|
|
|
64
84
|
writable: true,
|
|
65
85
|
value: void 0
|
|
66
86
|
});
|
|
67
|
-
Object.defineProperty(this, "
|
|
87
|
+
Object.defineProperty(this, "memorySignal", {
|
|
68
88
|
enumerable: true,
|
|
69
89
|
configurable: true,
|
|
70
90
|
writable: true,
|
|
71
91
|
value: void 0
|
|
72
92
|
});
|
|
73
|
-
Object.defineProperty(this, "
|
|
93
|
+
Object.defineProperty(this, "eventLoopSignal", {
|
|
74
94
|
enumerable: true,
|
|
75
95
|
configurable: true,
|
|
76
96
|
writable: true,
|
|
77
97
|
value: void 0
|
|
78
98
|
});
|
|
79
|
-
Object.defineProperty(this, "
|
|
99
|
+
Object.defineProperty(this, "cpuSignal", {
|
|
80
100
|
enumerable: true,
|
|
81
101
|
configurable: true,
|
|
82
102
|
writable: true,
|
|
83
103
|
value: void 0
|
|
84
104
|
});
|
|
85
|
-
Object.defineProperty(this, "
|
|
105
|
+
Object.defineProperty(this, "clientSignal", {
|
|
86
106
|
enumerable: true,
|
|
87
107
|
configurable: true,
|
|
88
108
|
writable: true,
|
|
89
109
|
value: void 0
|
|
90
110
|
});
|
|
91
|
-
Object.defineProperty(this, "maxUsedMemoryRatio", {
|
|
92
|
-
enumerable: true,
|
|
93
|
-
configurable: true,
|
|
94
|
-
writable: true,
|
|
95
|
-
value: void 0
|
|
96
|
-
});
|
|
97
|
-
Object.defineProperty(this, "maxClientErrors", {
|
|
98
|
-
enumerable: true,
|
|
99
|
-
configurable: true,
|
|
100
|
-
writable: true,
|
|
101
|
-
value: void 0
|
|
102
|
-
});
|
|
103
|
-
Object.defineProperty(this, "maxMemoryBytes", {
|
|
104
|
-
enumerable: true,
|
|
105
|
-
configurable: true,
|
|
106
|
-
writable: true,
|
|
107
|
-
value: void 0
|
|
108
|
-
});
|
|
109
|
-
Object.defineProperty(this, "maxMemoryRatio", {
|
|
110
|
-
enumerable: true,
|
|
111
|
-
configurable: true,
|
|
112
|
-
writable: true,
|
|
113
|
-
value: void 0
|
|
114
|
-
});
|
|
115
|
-
Object.defineProperty(this, "cpuSnapshots", {
|
|
116
|
-
enumerable: true,
|
|
117
|
-
configurable: true,
|
|
118
|
-
writable: true,
|
|
119
|
-
value: []
|
|
120
|
-
});
|
|
121
|
-
Object.defineProperty(this, "eventLoopSnapshots", {
|
|
122
|
-
enumerable: true,
|
|
123
|
-
configurable: true,
|
|
124
|
-
writable: true,
|
|
125
|
-
value: []
|
|
126
|
-
});
|
|
127
|
-
Object.defineProperty(this, "memorySnapshots", {
|
|
128
|
-
enumerable: true,
|
|
129
|
-
configurable: true,
|
|
130
|
-
writable: true,
|
|
131
|
-
value: []
|
|
132
|
-
});
|
|
133
|
-
Object.defineProperty(this, "clientSnapshots", {
|
|
134
|
-
enumerable: true,
|
|
135
|
-
configurable: true,
|
|
136
|
-
writable: true,
|
|
137
|
-
value: []
|
|
138
|
-
});
|
|
139
|
-
Object.defineProperty(this, "eventLoopInterval", {
|
|
140
|
-
enumerable: true,
|
|
141
|
-
configurable: true,
|
|
142
|
-
writable: true,
|
|
143
|
-
value: null
|
|
144
|
-
});
|
|
145
|
-
Object.defineProperty(this, "clientInterval", {
|
|
146
|
-
enumerable: true,
|
|
147
|
-
configurable: true,
|
|
148
|
-
writable: true,
|
|
149
|
-
value: null
|
|
150
|
-
});
|
|
151
|
-
Object.defineProperty(this, "lastLoggedCriticalMemoryOverloadAt", {
|
|
152
|
-
enumerable: true,
|
|
153
|
-
configurable: true,
|
|
154
|
-
writable: true,
|
|
155
|
-
value: null
|
|
156
|
-
});
|
|
157
111
|
(0, ow_1.default)(options, ow_1.default.object.exactShape({
|
|
158
112
|
eventLoopSnapshotIntervalSecs: ow_1.default.optional.number,
|
|
159
113
|
clientSnapshotIntervalSecs: ow_1.default.optional.number,
|
|
@@ -170,53 +124,46 @@ class Snapshotter {
|
|
|
170
124
|
this.client = client;
|
|
171
125
|
this.config = config;
|
|
172
126
|
this.events = this.config.getEventManager();
|
|
173
|
-
|
|
174
|
-
this.
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
this.
|
|
181
|
-
|
|
127
|
+
const snapshotHistoryMillis = snapshotHistorySecs * 1000;
|
|
128
|
+
this.memorySignal = new memory_load_signal_1.MemoryLoadSignal({
|
|
129
|
+
maxUsedMemoryRatio,
|
|
130
|
+
snapshotHistoryMillis,
|
|
131
|
+
config: this.config,
|
|
132
|
+
log: this.log,
|
|
133
|
+
});
|
|
134
|
+
this.eventLoopSignal = (0, event_loop_load_signal_1.createEventLoopLoadSignal)({
|
|
135
|
+
eventLoopSnapshotIntervalSecs,
|
|
136
|
+
maxBlockedMillis,
|
|
137
|
+
snapshotHistoryMillis,
|
|
138
|
+
});
|
|
139
|
+
this.cpuSignal = (0, cpu_load_signal_1.createCpuLoadSignal)({
|
|
140
|
+
snapshotHistoryMillis,
|
|
141
|
+
config: this.config,
|
|
142
|
+
});
|
|
143
|
+
this.clientSignal = (0, client_load_signal_1.createClientLoadSignal)({
|
|
144
|
+
client: this.client,
|
|
145
|
+
clientSnapshotIntervalSecs,
|
|
146
|
+
maxClientErrors,
|
|
147
|
+
snapshotHistoryMillis,
|
|
148
|
+
});
|
|
182
149
|
}
|
|
183
150
|
/**
|
|
184
151
|
* Starts capturing snapshots at configured intervals.
|
|
185
152
|
*/
|
|
186
153
|
async start() {
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
else {
|
|
192
|
-
this.maxMemoryRatio = this.config.get('availableMemoryRatio');
|
|
193
|
-
if (!this.maxMemoryRatio) {
|
|
194
|
-
throw new Error('availableMemoryRatio is not set in configuration.');
|
|
195
|
-
}
|
|
196
|
-
else {
|
|
197
|
-
this.log.debug(`Setting max memory of this run to ${this.maxMemoryRatio * 100} % of available memory. ` +
|
|
198
|
-
'Use the CRAWLEE_MEMORY_MBYTES or CRAWLEE_AVAILABLE_MEMORY_RATIO environment variable to override it.');
|
|
199
|
-
}
|
|
200
|
-
// Create a fallback memory measurement in case of missing memTotalBytes in SystemInfo. Weak types of
|
|
201
|
-
// SystemInfo do not guarantee that memTotalBytes is always present, and without it, we cannot compute the
|
|
202
|
-
// maxMemoryBytes.
|
|
203
|
-
// This does not happen in practice, but code allows it.
|
|
204
|
-
this.maxMemoryBytes = await this.getTotalMemoryBytes();
|
|
205
|
-
}
|
|
206
|
-
// Start snapshotting.
|
|
207
|
-
this.eventLoopInterval = (0, utilities_1.betterSetInterval)(this._snapshotEventLoop.bind(this), this.eventLoopSnapshotIntervalMillis);
|
|
208
|
-
this.clientInterval = (0, utilities_1.betterSetInterval)(this._snapshotClient.bind(this), this.clientSnapshotIntervalMillis);
|
|
209
|
-
this.events.on("systemInfo" /* EventType.SYSTEM_INFO */, this._snapshotCpu);
|
|
210
|
-
this.events.on("systemInfo" /* EventType.SYSTEM_INFO */, this._snapshotMemory);
|
|
154
|
+
await this.memorySignal.start();
|
|
155
|
+
await this.eventLoopSignal.start();
|
|
156
|
+
await this.cpuSignal.start();
|
|
157
|
+
await this.clientSignal.start();
|
|
211
158
|
}
|
|
212
159
|
/**
|
|
213
160
|
* Stops all resource capturing.
|
|
214
161
|
*/
|
|
215
162
|
async stop() {
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
this.
|
|
219
|
-
this.
|
|
163
|
+
await this.memorySignal.stop();
|
|
164
|
+
await this.eventLoopSignal.stop();
|
|
165
|
+
await this.cpuSignal.stop();
|
|
166
|
+
await this.clientSignal.stop();
|
|
220
167
|
// Allow microtask queue to unwind before stop returns.
|
|
221
168
|
await new Promise((resolve) => {
|
|
222
169
|
setImmediate(resolve);
|
|
@@ -227,177 +174,64 @@ class Snapshotter {
|
|
|
227
174
|
* by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
|
|
228
175
|
*/
|
|
229
176
|
getMemorySample(sampleDurationMillis) {
|
|
230
|
-
return this.
|
|
177
|
+
return this.memorySignal.getSample(sampleDurationMillis);
|
|
231
178
|
}
|
|
232
179
|
/**
|
|
233
180
|
* Returns a sample of latest event loop snapshots, with the size of the sample defined
|
|
234
181
|
* by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
|
|
235
182
|
*/
|
|
236
183
|
getEventLoopSample(sampleDurationMillis) {
|
|
237
|
-
return this.
|
|
184
|
+
return this.eventLoopSignal.getSample(sampleDurationMillis);
|
|
238
185
|
}
|
|
239
186
|
/**
|
|
240
187
|
* Returns a sample of latest CPU snapshots, with the size of the sample defined
|
|
241
188
|
* by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
|
|
242
189
|
*/
|
|
243
190
|
getCpuSample(sampleDurationMillis) {
|
|
244
|
-
return this.
|
|
191
|
+
return this.cpuSignal.getSample(sampleDurationMillis);
|
|
245
192
|
}
|
|
246
193
|
/**
|
|
247
194
|
* Returns a sample of latest Client snapshots, with the size of the sample defined
|
|
248
195
|
* by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
|
|
249
196
|
*/
|
|
250
197
|
getClientSample(sampleDurationMillis) {
|
|
251
|
-
return this.
|
|
198
|
+
return this.clientSignal.getSample(sampleDurationMillis);
|
|
252
199
|
}
|
|
253
200
|
/**
|
|
254
|
-
*
|
|
255
|
-
*/
|
|
256
|
-
_getSample(snapshots, sampleDurationMillis) {
|
|
257
|
-
if (!sampleDurationMillis)
|
|
258
|
-
return snapshots;
|
|
259
|
-
const sample = [];
|
|
260
|
-
let idx = snapshots.length;
|
|
261
|
-
if (!idx)
|
|
262
|
-
return sample;
|
|
263
|
-
const latestTime = snapshots[idx - 1].createdAt;
|
|
264
|
-
while (idx--) {
|
|
265
|
-
const snapshot = snapshots[idx];
|
|
266
|
-
if (+latestTime - +snapshot.createdAt <= sampleDurationMillis) {
|
|
267
|
-
sample.unshift(snapshot);
|
|
268
|
-
}
|
|
269
|
-
else {
|
|
270
|
-
break;
|
|
271
|
-
}
|
|
272
|
-
}
|
|
273
|
-
return sample;
|
|
274
|
-
}
|
|
275
|
-
/**
|
|
276
|
-
* Creates a snapshot of current memory usage
|
|
277
|
-
* using the Apify platform `systemInfo` event.
|
|
201
|
+
* @deprecated Kept for backward compatibility.
|
|
278
202
|
*/
|
|
279
203
|
_snapshotMemory(systemInfo) {
|
|
280
|
-
|
|
281
|
-
this._pruneSnapshots(this.memorySnapshots, createdAt);
|
|
282
|
-
const { memCurrentBytes, memTotalBytes } = systemInfo;
|
|
283
|
-
let maxMemoryBytes = this.maxMemoryBytes;
|
|
284
|
-
if (this.maxMemoryRatio !== undefined && this.maxMemoryRatio > 0) {
|
|
285
|
-
maxMemoryBytes = this.maxMemoryRatio * (memTotalBytes ?? this.maxMemoryBytes);
|
|
286
|
-
}
|
|
287
|
-
const snapshot = {
|
|
288
|
-
createdAt,
|
|
289
|
-
isOverloaded: memCurrentBytes / maxMemoryBytes > this.maxUsedMemoryRatio,
|
|
290
|
-
usedBytes: memCurrentBytes,
|
|
291
|
-
};
|
|
292
|
-
this.memorySnapshots.push(snapshot);
|
|
293
|
-
this._memoryOverloadWarning(systemInfo, maxMemoryBytes);
|
|
204
|
+
this.memorySignal._onSystemInfo(systemInfo);
|
|
294
205
|
}
|
|
295
206
|
/**
|
|
296
|
-
*
|
|
207
|
+
* @deprecated Kept for backward compatibility.
|
|
297
208
|
*/
|
|
298
|
-
_memoryOverloadWarning(systemInfo
|
|
299
|
-
|
|
300
|
-
const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
|
|
301
|
-
if (this.lastLoggedCriticalMemoryOverloadAt &&
|
|
302
|
-
+createdAt < +this.lastLoggedCriticalMemoryOverloadAt + CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS)
|
|
303
|
-
return;
|
|
304
|
-
const maxDesiredMemoryBytes = this.maxUsedMemoryRatio * maxMemoryBytes;
|
|
305
|
-
const reserveMemory = maxMemoryBytes * (1 - this.maxUsedMemoryRatio) * RESERVE_MEMORY_RATIO;
|
|
306
|
-
const criticalOverloadBytes = maxDesiredMemoryBytes + reserveMemory;
|
|
307
|
-
const isCriticalOverload = memCurrentBytes > criticalOverloadBytes;
|
|
308
|
-
if (isCriticalOverload) {
|
|
309
|
-
const usedPercentage = Math.round((memCurrentBytes / maxMemoryBytes) * 100);
|
|
310
|
-
const toMb = (bytes) => Math.round(bytes / 1024 ** 2);
|
|
311
|
-
this.log.warning('Memory is critically overloaded. ' +
|
|
312
|
-
`Using ${toMb(memCurrentBytes)} MB of ${toMb(maxMemoryBytes)} MB (${usedPercentage}%). Consider increasing available memory.`);
|
|
313
|
-
this.lastLoggedCriticalMemoryOverloadAt = createdAt;
|
|
314
|
-
}
|
|
209
|
+
_memoryOverloadWarning(systemInfo) {
|
|
210
|
+
this.memorySignal._memoryOverloadWarning(systemInfo);
|
|
315
211
|
}
|
|
316
212
|
/**
|
|
317
|
-
*
|
|
213
|
+
* @deprecated Kept for backward compatibility.
|
|
318
214
|
*/
|
|
319
215
|
_snapshotEventLoop(intervalCallback) {
|
|
320
|
-
|
|
321
|
-
this._pruneSnapshots(this.eventLoopSnapshots, now);
|
|
322
|
-
const snapshot = {
|
|
323
|
-
createdAt: now,
|
|
324
|
-
isOverloaded: false,
|
|
325
|
-
exceededMillis: 0,
|
|
326
|
-
};
|
|
327
|
-
const previousSnapshot = this.eventLoopSnapshots[this.eventLoopSnapshots.length - 1];
|
|
328
|
-
if (previousSnapshot) {
|
|
329
|
-
const { createdAt } = previousSnapshot;
|
|
330
|
-
const delta = now.getTime() - +createdAt - this.eventLoopSnapshotIntervalMillis;
|
|
331
|
-
if (delta > this.maxBlockedMillis)
|
|
332
|
-
snapshot.isOverloaded = true;
|
|
333
|
-
snapshot.exceededMillis = Math.max(delta - this.maxBlockedMillis, 0);
|
|
334
|
-
}
|
|
335
|
-
this.eventLoopSnapshots.push(snapshot);
|
|
336
|
-
intervalCallback();
|
|
216
|
+
this.eventLoopSignal.handle(intervalCallback);
|
|
337
217
|
}
|
|
338
218
|
/**
|
|
339
|
-
*
|
|
219
|
+
* @deprecated Kept for backward compatibility.
|
|
340
220
|
*/
|
|
341
221
|
_snapshotCpu(systemInfo) {
|
|
342
|
-
|
|
343
|
-
const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
|
|
344
|
-
this._pruneSnapshots(this.cpuSnapshots, createdAt);
|
|
345
|
-
this.cpuSnapshots.push({
|
|
346
|
-
createdAt,
|
|
347
|
-
isOverloaded: isCpuOverloaded,
|
|
348
|
-
usedRatio: Math.ceil(cpuCurrentUsage / 100),
|
|
349
|
-
});
|
|
222
|
+
this.cpuSignal.handle(systemInfo);
|
|
350
223
|
}
|
|
351
224
|
/**
|
|
352
|
-
*
|
|
353
|
-
* rate limit errors. Only errors produced by a 2nd retry
|
|
354
|
-
* of the API call are considered for snapshotting since
|
|
355
|
-
* earlier errors may just be caused by a random spike in
|
|
356
|
-
* number of requests and do not necessarily signify API
|
|
357
|
-
* overloading.
|
|
225
|
+
* @deprecated Kept for backward compatibility.
|
|
358
226
|
*/
|
|
359
227
|
_snapshotClient(intervalCallback) {
|
|
360
|
-
|
|
361
|
-
this._pruneSnapshots(this.clientSnapshots, now);
|
|
362
|
-
const allErrorCounts = this.client.stats?.rateLimitErrors ?? []; // storage client might not support this
|
|
363
|
-
const currentErrCount = allErrorCounts[CLIENT_RATE_LIMIT_ERROR_RETRY_COUNT] || 0;
|
|
364
|
-
// Handle empty snapshots array
|
|
365
|
-
const snapshot = {
|
|
366
|
-
createdAt: now,
|
|
367
|
-
isOverloaded: false,
|
|
368
|
-
rateLimitErrorCount: currentErrCount,
|
|
369
|
-
};
|
|
370
|
-
const previousSnapshot = this.clientSnapshots[this.clientSnapshots.length - 1];
|
|
371
|
-
if (previousSnapshot) {
|
|
372
|
-
const { rateLimitErrorCount } = previousSnapshot;
|
|
373
|
-
const delta = currentErrCount - rateLimitErrorCount;
|
|
374
|
-
if (delta > this.maxClientErrors)
|
|
375
|
-
snapshot.isOverloaded = true;
|
|
376
|
-
}
|
|
377
|
-
this.clientSnapshots.push(snapshot);
|
|
378
|
-
intervalCallback();
|
|
228
|
+
this.clientSignal.handle(intervalCallback);
|
|
379
229
|
}
|
|
380
230
|
/**
|
|
381
|
-
*
|
|
382
|
-
* from the array (destructively - in place).
|
|
231
|
+
* @deprecated Pruning is now handled by individual signals.
|
|
383
232
|
*/
|
|
384
|
-
_pruneSnapshots(
|
|
385
|
-
|
|
386
|
-
for (let i = 0; i < snapshots.length; i++) {
|
|
387
|
-
const { createdAt } = snapshots[i];
|
|
388
|
-
if (now.getTime() - new Date(createdAt).getTime() > this.snapshotHistoryMillis)
|
|
389
|
-
oldCount++;
|
|
390
|
-
else
|
|
391
|
-
break;
|
|
392
|
-
}
|
|
393
|
-
snapshots.splice(0, oldCount);
|
|
394
|
-
}
|
|
395
|
-
async getTotalMemoryBytes() {
|
|
396
|
-
if (this.config.get('systemInfoV2')) {
|
|
397
|
-
const containerized = this.config.get('containerized', await (0, utils_1.isContainerized)());
|
|
398
|
-
return (await (0, utils_1.getMemoryInfoV2)(containerized)).totalBytes;
|
|
399
|
-
}
|
|
400
|
-
return (await (0, utils_1.getMemoryInfo)()).totalBytes;
|
|
233
|
+
_pruneSnapshots(_snapshots, _now) {
|
|
234
|
+
// no-op — signals prune themselves
|
|
401
235
|
}
|
|
402
236
|
}
|
|
403
237
|
exports.Snapshotter = Snapshotter;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"snapshotter.js","sourceRoot":"","sources":["../../src/autoscaling/snapshotter.ts"],"names":[],"mappings":";;;;AACA,
|
|
1
|
+
{"version":3,"file":"snapshotter.js","sourceRoot":"","sources":["../../src/autoscaling/snapshotter.ts"],"names":[],"mappings":";;;;AACA,oDAAoB;AAIpB,oDAAiD;AAEjD,gCAA2C;AAE3C,6DAA8D;AAE9D,uDAAwD;AAExD,qEAAqE;AAGrE,6DAAwD;AAuDxD;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAa,WAAW;IAWpB;;;OAGG;IACH,cAAc;QACV,OAAO,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IACxF,CAAC;IAED,gFAAgF;IAChF,IAAI,YAAY;QACZ,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;IACzC,CAAC;IAED,IAAI,kBAAkB;QAClB,OAAO,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;IAC/C,CAAC;IAED,IAAI,eAAe;QACf,OAAO,IAAI,CAAC,YAAY,CAAC,kBAAkB,EAAE,CAAC;IAClD,CAAC;IAED,IAAI,eAAe;QACf,OAAO,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;IAC5C,CAAC;IAED;;OAEG;IACH,YAAY,UAA8B,EAAE;QAtC5C;;;;;WAAS;QACT;;;;;WAAsB;QACtB;;;;;WAAsB;QACtB;;;;;WAAqB;QAEJ;;;;;WAA+B;QAC/B;;;;;WAAqC;QACrC;;;;;WAAyB;QACzB;;;;;WAA+B;QA+B5C,IAAA,YAAE,EACE,OAAO,EACP,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,6BAA6B,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACjD,0BAA0B,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9C,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,gBAAgB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACpC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACvB,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC1B,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SAC7B,CAAC,CACL,CAAC;QAEF,MAAM,EACF,6BAA6B,GAAG,GAAG,EACnC,0BAA0B,GAAG,CAAC,EAC9B,mBAAmB,GAAG,EAAE,EACxB,gBAAgB,GAAG,EAAE,EACrB,kBAAkB,GAAG,GAAG,EACxB,eAAe,GAAG,CAAC,EACnB,GAAG,GAAG,SAAU,EAChB,MAAM,GAAG,6BAAa,CAAC,eAAe,EAAE,EACxC,MAAM,GAAG,MAAM,CAAC,gBAAgB,EAAE,GACrC,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC;QAE5C,MAAM,qBAAqB,GAAG,mBAAmB,GAAG,IAAI,CAAC;QAEzD,IAAI,CAAC,YAAY,GAAG,IAAI,qCAAgB,CAAC;YACrC,kBAAkB;YAClB,qBAAqB;YACrB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,GAAG,EAAE,IAAI,CAAC,GAAG;SAChB,CAAC,CAAC;QAEH,IAAI,CAAC,eAAe,GAAG,IAAA,kDAAyB,EAAC;YAC7C,6BAA6B;YAC7B,gBAAgB;YAChB,qBAAqB;SACxB,CAAC,CAAC;QAEH,IAAI,CAAC,SAAS,GAAG,IAAA,qCAAmB,EAAC;YACjC,qBAAqB;YACrB,MAAM,EAAE,IAAI,CAAC,MAAM;SACtB,CAAC,CAAC;QAEH,IAAI,CAAC,YAAY,GAAG,IAAA,2CAAsB,EAAC;YACvC,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,0BAA0B;YAC1B,eAAe;YACf,qBAAqB;SACxB,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACP,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;QAChC,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QACnC,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QAC7B,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;IACpC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACN,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;QAC/B,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;QAC5B,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;QAC/B,uDAAuD;QACvD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC1B,YAAY,CAAC,OAAO,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACH,eAAe,CAAC,oBAA6B;QACzC,OAAO,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,oBAAoB,CAAC,CAAC;IAC7D,CAAC;IAED;;;OAGG;IACH,kBAAkB,CAAC,oBAA6B;QAC5C,OAAO,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,oBAAoB,CAAC,CAAC;IAChE,CAAC;IAED;;;OAGG;IACH,YAAY,CAAC,oBAA6B;QACtC,OAAO,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,oBAAoB,CAAC,CAAC;IAC1D,CAAC;IAED;;;OAGG;IACH,eAAe,CAAC,oBAA6B;QACzC,OAAO,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,oBAAoB,CAAC,CAAC;IAC7D,CAAC;IAED;;OAEG;IACO,eAAe,CAAC,UAAsB;QAC5C,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;IAChD,CAAC;IAED;;OAEG;IACO,sBAAsB,CAAC,UAAsB;QACnD,IAAI,CAAC,YAAY,CAAC,sBAAsB,CAAC,UAAU,CAAC,CAAC;IACzD,CAAC;IAED;;OAEG;IACO,kBAAkB,CAAC,gBAA+B;QACxD,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;IAClD,CAAC;IAED;;OAEG;IACO,YAAY,CAAC,UAAsB;QACzC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IACtC,CAAC;IAED;;OAEG;IACO,eAAe,CAAC,gBAA+B;QACrD,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;IAC/C,CAAC;IAED;;OAEG;IACO,eAAe,CAAC,UAAiB,EAAE,IAAU;QACnD,mCAAmC;IACvC,CAAC;CACJ;AArMD,kCAqMC"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { Configuration } from '../configuration';
|
|
2
|
+
import type { LoadSignal } from './load_signal';
|
|
2
3
|
import { Snapshotter } from './snapshotter';
|
|
3
4
|
/**
|
|
4
5
|
* Represents the current status of the system.
|
|
@@ -27,6 +28,11 @@ export interface SystemInfo {
|
|
|
27
28
|
* @internal
|
|
28
29
|
*/
|
|
29
30
|
createdAt?: Date;
|
|
31
|
+
/**
|
|
32
|
+
* Status of additional load signals beyond the built-in four.
|
|
33
|
+
* Keys are `LoadSignal.name` values, values are overload info.
|
|
34
|
+
*/
|
|
35
|
+
loadSignalInfo?: Record<string, ClientInfo>;
|
|
30
36
|
}
|
|
31
37
|
export interface SystemStatusOptions {
|
|
32
38
|
/**
|
|
@@ -62,6 +68,13 @@ export interface SystemStatusOptions {
|
|
|
62
68
|
* The `Snapshotter` instance to be queried for `SystemStatus`.
|
|
63
69
|
*/
|
|
64
70
|
snapshotter?: Snapshotter;
|
|
71
|
+
/**
|
|
72
|
+
* Additional load signals to include in the system status evaluation.
|
|
73
|
+
* These are evaluated alongside the built-in memory, CPU, event loop,
|
|
74
|
+
* and client signals. If any signal reports overload, the system is
|
|
75
|
+
* considered overloaded.
|
|
76
|
+
*/
|
|
77
|
+
loadSignals?: LoadSignal[];
|
|
65
78
|
/** @internal */
|
|
66
79
|
config?: Configuration;
|
|
67
80
|
}
|
|
@@ -106,11 +119,14 @@ export interface FinalStatistics {
|
|
|
106
119
|
*/
|
|
107
120
|
export declare class SystemStatus {
|
|
108
121
|
private readonly currentHistoryMillis;
|
|
109
|
-
private readonly maxMemoryOverloadedRatio;
|
|
110
|
-
private readonly maxEventLoopOverloadedRatio;
|
|
111
|
-
private readonly maxCpuOverloadedRatio;
|
|
112
|
-
private readonly maxClientOverloadedRatio;
|
|
113
122
|
private readonly snapshotter;
|
|
123
|
+
private readonly signals;
|
|
124
|
+
/**
|
|
125
|
+
* Per-signal ratio overrides. The built-in four get their overrides from
|
|
126
|
+
* the legacy `max*OverloadedRatio` options; custom signals use their own
|
|
127
|
+
* `overloadedRatio`.
|
|
128
|
+
*/
|
|
129
|
+
private ratioOverrides;
|
|
114
130
|
constructor(options?: SystemStatusOptions);
|
|
115
131
|
/**
|
|
116
132
|
* Returns an {@link SystemInfo} object with the following structure:
|
|
@@ -150,33 +166,5 @@ export declare class SystemStatus {
|
|
|
150
166
|
* Returns a system status object.
|
|
151
167
|
*/
|
|
152
168
|
protected _isSystemIdle(sampleDurationMillis?: number): SystemInfo;
|
|
153
|
-
/**
|
|
154
|
-
* Returns an object with an isOverloaded property set to true
|
|
155
|
-
* if the memory has been overloaded in the last sampleDurationMillis.
|
|
156
|
-
*/
|
|
157
|
-
protected _isMemoryOverloaded(sampleDurationMillis?: number): ClientInfo;
|
|
158
|
-
/**
|
|
159
|
-
* Returns an object with an isOverloaded property set to true
|
|
160
|
-
* if the event loop has been overloaded in the last sampleDurationMillis.
|
|
161
|
-
*/
|
|
162
|
-
protected _isEventLoopOverloaded(sampleDurationMillis?: number): ClientInfo;
|
|
163
|
-
/**
|
|
164
|
-
* Returns an object with an isOverloaded property set to true
|
|
165
|
-
* if the CPU has been overloaded in the last sampleDurationMillis.
|
|
166
|
-
*/
|
|
167
|
-
protected _isCpuOverloaded(sampleDurationMillis?: number): ClientInfo;
|
|
168
|
-
/**
|
|
169
|
-
* Returns an object with an isOverloaded property set to true
|
|
170
|
-
* if the client has been overloaded in the last sampleDurationMillis.
|
|
171
|
-
*/
|
|
172
|
-
protected _isClientOverloaded(sampleDurationMillis?: number): ClientInfo;
|
|
173
|
-
/**
|
|
174
|
-
* Returns an object with sample information and an isOverloaded property
|
|
175
|
-
* set to true if at least the ratio of snapshots in the sample are overloaded.
|
|
176
|
-
*/
|
|
177
|
-
protected _isSampleOverloaded<T extends {
|
|
178
|
-
createdAt: Date;
|
|
179
|
-
isOverloaded: boolean;
|
|
180
|
-
}>(sample: T[], ratio: number): ClientInfo;
|
|
181
169
|
}
|
|
182
170
|
//# sourceMappingURL=system_status.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"system_status.d.ts","sourceRoot":"","sources":["../../src/autoscaling/system_status.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"system_status.d.ts","sourceRoot":"","sources":["../../src/autoscaling/system_status.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAEhD,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAE5C;;GAEG;AACH,MAAM,WAAW,UAAU;IACvB,4CAA4C;IAC5C,YAAY,EAAE,OAAO,CAAC;IACtB,OAAO,EAAE,UAAU,CAAC;IACpB,aAAa,EAAE,UAAU,CAAC;IAC1B,OAAO,EAAE,UAAU,CAAC;IACpB,UAAU,EAAE,UAAU,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B;;;OAGG;IACH,SAAS,CAAC,EAAE,IAAI,CAAC;IAEjB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;CAC/C;AAED,MAAM,WAAW,mBAAmB;IAChC;;;OAGG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;OAIG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAElC;;;;OAIG;IACH,2BAA2B,CAAC,EAAE,MAAM,CAAC;IAErC;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;;OAIG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAElC;;OAEG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B;;;;;OAKG;IACH,WAAW,CAAC,EAAE,UAAU,EAAE,CAAC;IAE3B,gBAAgB;IAChB,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED,MAAM,WAAW,UAAU;IACvB,YAAY,EAAE,OAAO,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,eAAe;IAC5B,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,8BAA8B,EAAE,MAAM,CAAC;IACvC,gCAAgC,EAAE,MAAM,CAAC;IACzC,yBAAyB,EAAE,MAAM,CAAC;IAClC,uBAAuB,EAAE,MAAM,CAAC;IAChC,0BAA0B,EAAE,MAAM,CAAC;IACnC,aAAa,EAAE,MAAM,CAAC;IACtB,oBAAoB,EAAE,MAAM,CAAC;CAChC;AAKD;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,qBAAa,YAAY;IACrB,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAS;IAC9C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;IAC1C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAe;IAEvC;;;;OAIG;IACH,OAAO,CAAC,cAAc,CAAyB;gBAEnC,OAAO,GAAE,mBAAwB;IAyC7C;;;;;;;;;;;;;;;OAeG;IACH,gBAAgB,IAAI,UAAU;IAI9B;;;;;;;;;;;;;;;OAeG;IACH,mBAAmB,IAAI,UAAU;IAIjC;;OAEG;IACH,SAAS,CAAC,aAAa,CAAC,oBAAoB,CAAC,EAAE,MAAM,GAAG,UAAU;CAkCrE"}
|