@crawlee/core 3.16.1-beta.4 → 3.16.1-beta.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/autoscaling/autoscaled_pool.d.ts +2 -0
- package/autoscaling/autoscaled_pool.d.ts.map +1 -1
- package/autoscaling/autoscaled_pool.js +10 -0
- package/autoscaling/autoscaled_pool.js.map +1 -1
- package/autoscaling/client_load_signal.d.ts +26 -0
- package/autoscaling/client_load_signal.d.ts.map +1 -0
- package/autoscaling/client_load_signal.js +40 -0
- package/autoscaling/client_load_signal.js.map +1 -0
- package/autoscaling/cpu_load_signal.d.ts +29 -0
- package/autoscaling/cpu_load_signal.d.ts.map +1 -0
- package/autoscaling/cpu_load_signal.js +27 -0
- package/autoscaling/cpu_load_signal.js.map +1 -0
- package/autoscaling/event_loop_load_signal.d.ts +24 -0
- package/autoscaling/event_loop_load_signal.d.ts.map +1 -0
- package/autoscaling/event_loop_load_signal.js +39 -0
- package/autoscaling/event_loop_load_signal.js.map +1 -0
- package/autoscaling/index.d.ts +5 -0
- package/autoscaling/index.d.ts.map +1 -1
- package/autoscaling/index.js +5 -0
- package/autoscaling/index.js.map +1 -1
- package/autoscaling/load_signal.d.ts +100 -0
- package/autoscaling/load_signal.d.ts.map +1 -0
- package/autoscaling/load_signal.js +151 -0
- package/autoscaling/load_signal.js.map +1 -0
- package/autoscaling/memory_load_signal.d.ts +44 -0
- package/autoscaling/memory_load_signal.d.ts.map +1 -0
- package/autoscaling/memory_load_signal.js +159 -0
- package/autoscaling/memory_load_signal.js.map +1 -0
- package/autoscaling/snapshotter.d.ts +26 -60
- package/autoscaling/snapshotter.d.ts.map +1 -1
- package/autoscaling/snapshotter.js +76 -225
- package/autoscaling/snapshotter.js.map +1 -1
- package/autoscaling/system_status.d.ts +21 -32
- package/autoscaling/system_status.d.ts.map +1 -1
- package/autoscaling/system_status.js +46 -92
- package/autoscaling/system_status.js.map +1 -1
- package/events/local_event_manager.d.ts +1 -0
- package/events/local_event_manager.d.ts.map +1 -1
- package/events/local_event_manager.js +8 -7
- package/events/local_event_manager.js.map +1 -1
- package/index.mjs +6 -0
- package/package.json +6 -6
|
@@ -2,14 +2,13 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.Snapshotter = void 0;
|
|
4
4
|
const tslib_1 = require("tslib");
|
|
5
|
-
const utils_1 = require("@crawlee/utils");
|
|
6
5
|
const ow_1 = tslib_1.__importDefault(require("ow"));
|
|
7
|
-
const utilities_1 = require("@apify/utilities");
|
|
8
6
|
const configuration_1 = require("../configuration");
|
|
9
7
|
const log_1 = require("../log");
|
|
10
|
-
const
|
|
11
|
-
const
|
|
12
|
-
const
|
|
8
|
+
const client_load_signal_1 = require("./client_load_signal");
|
|
9
|
+
const cpu_load_signal_1 = require("./cpu_load_signal");
|
|
10
|
+
const event_loop_load_signal_1 = require("./event_loop_load_signal");
|
|
11
|
+
const memory_load_signal_1 = require("./memory_load_signal");
|
|
13
12
|
/**
|
|
14
13
|
* Creates snapshots of system resources at given intervals and marks the resource
|
|
15
14
|
* as either overloaded or not during the last interval. Keeps a history of the snapshots.
|
|
@@ -33,9 +32,30 @@ const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10000;
|
|
|
33
32
|
*
|
|
34
33
|
* Client becomes overloaded when rate limit errors (429 - Too Many Requests),
|
|
35
34
|
* typically received from the request queue, exceed the set limit within the set interval.
|
|
35
|
+
*
|
|
36
36
|
* @category Scaling
|
|
37
37
|
*/
|
|
38
38
|
class Snapshotter {
|
|
39
|
+
/**
|
|
40
|
+
* Returns the four built-in signals as an array, so `SystemStatus` can
|
|
41
|
+
* iterate them alongside any custom `LoadSignal` instances.
|
|
42
|
+
*/
|
|
43
|
+
getLoadSignals() {
|
|
44
|
+
return [this.memorySignal, this.eventLoopSignal, this.cpuSignal, this.clientSignal];
|
|
45
|
+
}
|
|
46
|
+
// Legacy public properties kept for backward compat (tests read these directly)
|
|
47
|
+
get cpuSnapshots() {
|
|
48
|
+
return this.cpuSignal.store.getAll();
|
|
49
|
+
}
|
|
50
|
+
get eventLoopSnapshots() {
|
|
51
|
+
return this.eventLoopSignal.store.getAll();
|
|
52
|
+
}
|
|
53
|
+
get memorySnapshots() {
|
|
54
|
+
return this.memorySignal.getMemorySnapshots();
|
|
55
|
+
}
|
|
56
|
+
get clientSnapshots() {
|
|
57
|
+
return this.clientSignal.store.getAll();
|
|
58
|
+
}
|
|
39
59
|
/**
|
|
40
60
|
* @param [options] All `Snapshotter` configuration options.
|
|
41
61
|
*/
|
|
@@ -64,90 +84,30 @@ class Snapshotter {
|
|
|
64
84
|
writable: true,
|
|
65
85
|
value: void 0
|
|
66
86
|
});
|
|
67
|
-
Object.defineProperty(this, "
|
|
68
|
-
enumerable: true,
|
|
69
|
-
configurable: true,
|
|
70
|
-
writable: true,
|
|
71
|
-
value: void 0
|
|
72
|
-
});
|
|
73
|
-
Object.defineProperty(this, "clientSnapshotIntervalMillis", {
|
|
74
|
-
enumerable: true,
|
|
75
|
-
configurable: true,
|
|
76
|
-
writable: true,
|
|
77
|
-
value: void 0
|
|
78
|
-
});
|
|
79
|
-
Object.defineProperty(this, "snapshotHistoryMillis", {
|
|
80
|
-
enumerable: true,
|
|
81
|
-
configurable: true,
|
|
82
|
-
writable: true,
|
|
83
|
-
value: void 0
|
|
84
|
-
});
|
|
85
|
-
Object.defineProperty(this, "maxBlockedMillis", {
|
|
87
|
+
Object.defineProperty(this, "memorySignal", {
|
|
86
88
|
enumerable: true,
|
|
87
89
|
configurable: true,
|
|
88
90
|
writable: true,
|
|
89
91
|
value: void 0
|
|
90
92
|
});
|
|
91
|
-
Object.defineProperty(this, "
|
|
93
|
+
Object.defineProperty(this, "eventLoopSignal", {
|
|
92
94
|
enumerable: true,
|
|
93
95
|
configurable: true,
|
|
94
96
|
writable: true,
|
|
95
97
|
value: void 0
|
|
96
98
|
});
|
|
97
|
-
Object.defineProperty(this, "
|
|
99
|
+
Object.defineProperty(this, "cpuSignal", {
|
|
98
100
|
enumerable: true,
|
|
99
101
|
configurable: true,
|
|
100
102
|
writable: true,
|
|
101
103
|
value: void 0
|
|
102
104
|
});
|
|
103
|
-
Object.defineProperty(this, "
|
|
105
|
+
Object.defineProperty(this, "clientSignal", {
|
|
104
106
|
enumerable: true,
|
|
105
107
|
configurable: true,
|
|
106
108
|
writable: true,
|
|
107
109
|
value: void 0
|
|
108
110
|
});
|
|
109
|
-
Object.defineProperty(this, "cpuSnapshots", {
|
|
110
|
-
enumerable: true,
|
|
111
|
-
configurable: true,
|
|
112
|
-
writable: true,
|
|
113
|
-
value: []
|
|
114
|
-
});
|
|
115
|
-
Object.defineProperty(this, "eventLoopSnapshots", {
|
|
116
|
-
enumerable: true,
|
|
117
|
-
configurable: true,
|
|
118
|
-
writable: true,
|
|
119
|
-
value: []
|
|
120
|
-
});
|
|
121
|
-
Object.defineProperty(this, "memorySnapshots", {
|
|
122
|
-
enumerable: true,
|
|
123
|
-
configurable: true,
|
|
124
|
-
writable: true,
|
|
125
|
-
value: []
|
|
126
|
-
});
|
|
127
|
-
Object.defineProperty(this, "clientSnapshots", {
|
|
128
|
-
enumerable: true,
|
|
129
|
-
configurable: true,
|
|
130
|
-
writable: true,
|
|
131
|
-
value: []
|
|
132
|
-
});
|
|
133
|
-
Object.defineProperty(this, "eventLoopInterval", {
|
|
134
|
-
enumerable: true,
|
|
135
|
-
configurable: true,
|
|
136
|
-
writable: true,
|
|
137
|
-
value: null
|
|
138
|
-
});
|
|
139
|
-
Object.defineProperty(this, "clientInterval", {
|
|
140
|
-
enumerable: true,
|
|
141
|
-
configurable: true,
|
|
142
|
-
writable: true,
|
|
143
|
-
value: null
|
|
144
|
-
});
|
|
145
|
-
Object.defineProperty(this, "lastLoggedCriticalMemoryOverloadAt", {
|
|
146
|
-
enumerable: true,
|
|
147
|
-
configurable: true,
|
|
148
|
-
writable: true,
|
|
149
|
-
value: null
|
|
150
|
-
});
|
|
151
111
|
(0, ow_1.default)(options, ow_1.default.object.exactShape({
|
|
152
112
|
eventLoopSnapshotIntervalSecs: ow_1.default.optional.number,
|
|
153
113
|
clientSnapshotIntervalSecs: ow_1.default.optional.number,
|
|
@@ -164,53 +124,46 @@ class Snapshotter {
|
|
|
164
124
|
this.client = client;
|
|
165
125
|
this.config = config;
|
|
166
126
|
this.events = this.config.getEventManager();
|
|
167
|
-
|
|
168
|
-
this.
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
this.
|
|
175
|
-
|
|
127
|
+
const snapshotHistoryMillis = snapshotHistorySecs * 1000;
|
|
128
|
+
this.memorySignal = new memory_load_signal_1.MemoryLoadSignal({
|
|
129
|
+
maxUsedMemoryRatio,
|
|
130
|
+
snapshotHistoryMillis,
|
|
131
|
+
config: this.config,
|
|
132
|
+
log: this.log,
|
|
133
|
+
});
|
|
134
|
+
this.eventLoopSignal = (0, event_loop_load_signal_1.createEventLoopLoadSignal)({
|
|
135
|
+
eventLoopSnapshotIntervalSecs,
|
|
136
|
+
maxBlockedMillis,
|
|
137
|
+
snapshotHistoryMillis,
|
|
138
|
+
});
|
|
139
|
+
this.cpuSignal = (0, cpu_load_signal_1.createCpuLoadSignal)({
|
|
140
|
+
snapshotHistoryMillis,
|
|
141
|
+
config: this.config,
|
|
142
|
+
});
|
|
143
|
+
this.clientSignal = (0, client_load_signal_1.createClientLoadSignal)({
|
|
144
|
+
client: this.client,
|
|
145
|
+
clientSnapshotIntervalSecs,
|
|
146
|
+
maxClientErrors,
|
|
147
|
+
snapshotHistoryMillis,
|
|
148
|
+
});
|
|
176
149
|
}
|
|
177
150
|
/**
|
|
178
151
|
* Starts capturing snapshots at configured intervals.
|
|
179
152
|
*/
|
|
180
153
|
async start() {
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
else {
|
|
186
|
-
let totalBytes;
|
|
187
|
-
if (this.config.get('systemInfoV2')) {
|
|
188
|
-
const containerized = this.config.get('containerized', await (0, utils_1.isContainerized)());
|
|
189
|
-
const memInfo = await (0, utils_1.getMemoryInfoV2)(containerized);
|
|
190
|
-
totalBytes = memInfo.totalBytes;
|
|
191
|
-
}
|
|
192
|
-
else {
|
|
193
|
-
const memInfo = await (0, utils_1.getMemoryInfo)();
|
|
194
|
-
totalBytes = memInfo.totalBytes;
|
|
195
|
-
}
|
|
196
|
-
this.maxMemoryBytes = Math.ceil(totalBytes * this.config.get('availableMemoryRatio'));
|
|
197
|
-
this.log.debug(`Setting max memory of this run to ${Math.round(this.maxMemoryBytes / 1024 / 1024)} MB. ` +
|
|
198
|
-
'Use the CRAWLEE_MEMORY_MBYTES or CRAWLEE_AVAILABLE_MEMORY_RATIO environment variable to override it.');
|
|
199
|
-
}
|
|
200
|
-
// Start snapshotting.
|
|
201
|
-
this.eventLoopInterval = (0, utilities_1.betterSetInterval)(this._snapshotEventLoop.bind(this), this.eventLoopSnapshotIntervalMillis);
|
|
202
|
-
this.clientInterval = (0, utilities_1.betterSetInterval)(this._snapshotClient.bind(this), this.clientSnapshotIntervalMillis);
|
|
203
|
-
this.events.on("systemInfo" /* EventType.SYSTEM_INFO */, this._snapshotCpu);
|
|
204
|
-
this.events.on("systemInfo" /* EventType.SYSTEM_INFO */, this._snapshotMemory);
|
|
154
|
+
await this.memorySignal.start();
|
|
155
|
+
await this.eventLoopSignal.start();
|
|
156
|
+
await this.cpuSignal.start();
|
|
157
|
+
await this.clientSignal.start();
|
|
205
158
|
}
|
|
206
159
|
/**
|
|
207
160
|
* Stops all resource capturing.
|
|
208
161
|
*/
|
|
209
162
|
async stop() {
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
this.
|
|
213
|
-
this.
|
|
163
|
+
await this.memorySignal.stop();
|
|
164
|
+
await this.eventLoopSignal.stop();
|
|
165
|
+
await this.cpuSignal.stop();
|
|
166
|
+
await this.clientSignal.stop();
|
|
214
167
|
// Allow microtask queue to unwind before stop returns.
|
|
215
168
|
await new Promise((resolve) => {
|
|
216
169
|
setImmediate(resolve);
|
|
@@ -221,166 +174,64 @@ class Snapshotter {
|
|
|
221
174
|
* by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
|
|
222
175
|
*/
|
|
223
176
|
getMemorySample(sampleDurationMillis) {
|
|
224
|
-
return this.
|
|
177
|
+
return this.memorySignal.getSample(sampleDurationMillis);
|
|
225
178
|
}
|
|
226
179
|
/**
|
|
227
180
|
* Returns a sample of latest event loop snapshots, with the size of the sample defined
|
|
228
181
|
* by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
|
|
229
182
|
*/
|
|
230
183
|
getEventLoopSample(sampleDurationMillis) {
|
|
231
|
-
return this.
|
|
184
|
+
return this.eventLoopSignal.getSample(sampleDurationMillis);
|
|
232
185
|
}
|
|
233
186
|
/**
|
|
234
187
|
* Returns a sample of latest CPU snapshots, with the size of the sample defined
|
|
235
188
|
* by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
|
|
236
189
|
*/
|
|
237
190
|
getCpuSample(sampleDurationMillis) {
|
|
238
|
-
return this.
|
|
191
|
+
return this.cpuSignal.getSample(sampleDurationMillis);
|
|
239
192
|
}
|
|
240
193
|
/**
|
|
241
194
|
* Returns a sample of latest Client snapshots, with the size of the sample defined
|
|
242
195
|
* by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
|
|
243
196
|
*/
|
|
244
197
|
getClientSample(sampleDurationMillis) {
|
|
245
|
-
return this.
|
|
198
|
+
return this.clientSignal.getSample(sampleDurationMillis);
|
|
246
199
|
}
|
|
247
200
|
/**
|
|
248
|
-
*
|
|
249
|
-
*/
|
|
250
|
-
_getSample(snapshots, sampleDurationMillis) {
|
|
251
|
-
if (!sampleDurationMillis)
|
|
252
|
-
return snapshots;
|
|
253
|
-
const sample = [];
|
|
254
|
-
let idx = snapshots.length;
|
|
255
|
-
if (!idx)
|
|
256
|
-
return sample;
|
|
257
|
-
const latestTime = snapshots[idx - 1].createdAt;
|
|
258
|
-
while (idx--) {
|
|
259
|
-
const snapshot = snapshots[idx];
|
|
260
|
-
if (+latestTime - +snapshot.createdAt <= sampleDurationMillis) {
|
|
261
|
-
sample.unshift(snapshot);
|
|
262
|
-
}
|
|
263
|
-
else {
|
|
264
|
-
break;
|
|
265
|
-
}
|
|
266
|
-
}
|
|
267
|
-
return sample;
|
|
268
|
-
}
|
|
269
|
-
/**
|
|
270
|
-
* Creates a snapshot of current memory usage
|
|
271
|
-
* using the Apify platform `systemInfo` event.
|
|
201
|
+
* @deprecated Kept for backward compatibility.
|
|
272
202
|
*/
|
|
273
203
|
_snapshotMemory(systemInfo) {
|
|
274
|
-
|
|
275
|
-
this._pruneSnapshots(this.memorySnapshots, createdAt);
|
|
276
|
-
const { memCurrentBytes } = systemInfo;
|
|
277
|
-
const snapshot = {
|
|
278
|
-
createdAt,
|
|
279
|
-
isOverloaded: memCurrentBytes / this.maxMemoryBytes > this.maxUsedMemoryRatio,
|
|
280
|
-
usedBytes: memCurrentBytes,
|
|
281
|
-
};
|
|
282
|
-
this.memorySnapshots.push(snapshot);
|
|
283
|
-
this._memoryOverloadWarning(systemInfo);
|
|
204
|
+
this.memorySignal._onSystemInfo(systemInfo);
|
|
284
205
|
}
|
|
285
206
|
/**
|
|
286
|
-
*
|
|
207
|
+
* @deprecated Kept for backward compatibility.
|
|
287
208
|
*/
|
|
288
209
|
_memoryOverloadWarning(systemInfo) {
|
|
289
|
-
|
|
290
|
-
const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
|
|
291
|
-
if (this.lastLoggedCriticalMemoryOverloadAt &&
|
|
292
|
-
+createdAt < +this.lastLoggedCriticalMemoryOverloadAt + CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS)
|
|
293
|
-
return;
|
|
294
|
-
const maxDesiredMemoryBytes = this.maxUsedMemoryRatio * this.maxMemoryBytes;
|
|
295
|
-
const reserveMemory = this.maxMemoryBytes * (1 - this.maxUsedMemoryRatio) * RESERVE_MEMORY_RATIO;
|
|
296
|
-
const criticalOverloadBytes = maxDesiredMemoryBytes + reserveMemory;
|
|
297
|
-
const isCriticalOverload = memCurrentBytes > criticalOverloadBytes;
|
|
298
|
-
if (isCriticalOverload) {
|
|
299
|
-
const usedPercentage = Math.round((memCurrentBytes / this.maxMemoryBytes) * 100);
|
|
300
|
-
const toMb = (bytes) => Math.round(bytes / 1024 ** 2);
|
|
301
|
-
this.log.warning('Memory is critically overloaded. ' +
|
|
302
|
-
`Using ${toMb(memCurrentBytes)} MB of ${toMb(this.maxMemoryBytes)} MB (${usedPercentage}%). Consider increasing available memory.`);
|
|
303
|
-
this.lastLoggedCriticalMemoryOverloadAt = createdAt;
|
|
304
|
-
}
|
|
210
|
+
this.memorySignal._memoryOverloadWarning(systemInfo);
|
|
305
211
|
}
|
|
306
212
|
/**
|
|
307
|
-
*
|
|
213
|
+
* @deprecated Kept for backward compatibility.
|
|
308
214
|
*/
|
|
309
215
|
_snapshotEventLoop(intervalCallback) {
|
|
310
|
-
|
|
311
|
-
this._pruneSnapshots(this.eventLoopSnapshots, now);
|
|
312
|
-
const snapshot = {
|
|
313
|
-
createdAt: now,
|
|
314
|
-
isOverloaded: false,
|
|
315
|
-
exceededMillis: 0,
|
|
316
|
-
};
|
|
317
|
-
const previousSnapshot = this.eventLoopSnapshots[this.eventLoopSnapshots.length - 1];
|
|
318
|
-
if (previousSnapshot) {
|
|
319
|
-
const { createdAt } = previousSnapshot;
|
|
320
|
-
const delta = now.getTime() - +createdAt - this.eventLoopSnapshotIntervalMillis;
|
|
321
|
-
if (delta > this.maxBlockedMillis)
|
|
322
|
-
snapshot.isOverloaded = true;
|
|
323
|
-
snapshot.exceededMillis = Math.max(delta - this.maxBlockedMillis, 0);
|
|
324
|
-
}
|
|
325
|
-
this.eventLoopSnapshots.push(snapshot);
|
|
326
|
-
intervalCallback();
|
|
216
|
+
this.eventLoopSignal.handle(intervalCallback);
|
|
327
217
|
}
|
|
328
218
|
/**
|
|
329
|
-
*
|
|
219
|
+
* @deprecated Kept for backward compatibility.
|
|
330
220
|
*/
|
|
331
221
|
_snapshotCpu(systemInfo) {
|
|
332
|
-
|
|
333
|
-
const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
|
|
334
|
-
this._pruneSnapshots(this.cpuSnapshots, createdAt);
|
|
335
|
-
this.cpuSnapshots.push({
|
|
336
|
-
createdAt,
|
|
337
|
-
isOverloaded: isCpuOverloaded,
|
|
338
|
-
usedRatio: Math.ceil(cpuCurrentUsage / 100),
|
|
339
|
-
});
|
|
222
|
+
this.cpuSignal.handle(systemInfo);
|
|
340
223
|
}
|
|
341
224
|
/**
|
|
342
|
-
*
|
|
343
|
-
* rate limit errors. Only errors produced by a 2nd retry
|
|
344
|
-
* of the API call are considered for snapshotting since
|
|
345
|
-
* earlier errors may just be caused by a random spike in
|
|
346
|
-
* number of requests and do not necessarily signify API
|
|
347
|
-
* overloading.
|
|
225
|
+
* @deprecated Kept for backward compatibility.
|
|
348
226
|
*/
|
|
349
227
|
_snapshotClient(intervalCallback) {
|
|
350
|
-
|
|
351
|
-
this._pruneSnapshots(this.clientSnapshots, now);
|
|
352
|
-
const allErrorCounts = this.client.stats?.rateLimitErrors ?? []; // storage client might not support this
|
|
353
|
-
const currentErrCount = allErrorCounts[CLIENT_RATE_LIMIT_ERROR_RETRY_COUNT] || 0;
|
|
354
|
-
// Handle empty snapshots array
|
|
355
|
-
const snapshot = {
|
|
356
|
-
createdAt: now,
|
|
357
|
-
isOverloaded: false,
|
|
358
|
-
rateLimitErrorCount: currentErrCount,
|
|
359
|
-
};
|
|
360
|
-
const previousSnapshot = this.clientSnapshots[this.clientSnapshots.length - 1];
|
|
361
|
-
if (previousSnapshot) {
|
|
362
|
-
const { rateLimitErrorCount } = previousSnapshot;
|
|
363
|
-
const delta = currentErrCount - rateLimitErrorCount;
|
|
364
|
-
if (delta > this.maxClientErrors)
|
|
365
|
-
snapshot.isOverloaded = true;
|
|
366
|
-
}
|
|
367
|
-
this.clientSnapshots.push(snapshot);
|
|
368
|
-
intervalCallback();
|
|
228
|
+
this.clientSignal.handle(intervalCallback);
|
|
369
229
|
}
|
|
370
230
|
/**
|
|
371
|
-
*
|
|
372
|
-
* from the array (destructively - in place).
|
|
231
|
+
* @deprecated Pruning is now handled by individual signals.
|
|
373
232
|
*/
|
|
374
|
-
_pruneSnapshots(
|
|
375
|
-
|
|
376
|
-
for (let i = 0; i < snapshots.length; i++) {
|
|
377
|
-
const { createdAt } = snapshots[i];
|
|
378
|
-
if (now.getTime() - new Date(createdAt).getTime() > this.snapshotHistoryMillis)
|
|
379
|
-
oldCount++;
|
|
380
|
-
else
|
|
381
|
-
break;
|
|
382
|
-
}
|
|
383
|
-
snapshots.splice(0, oldCount);
|
|
233
|
+
_pruneSnapshots(_snapshots, _now) {
|
|
234
|
+
// no-op — signals prune themselves
|
|
384
235
|
}
|
|
385
236
|
}
|
|
386
237
|
exports.Snapshotter = Snapshotter;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"snapshotter.js","sourceRoot":"","sources":["../../src/autoscaling/snapshotter.ts"],"names":[],"mappings":";;;;AACA,
|
|
1
|
+
{"version":3,"file":"snapshotter.js","sourceRoot":"","sources":["../../src/autoscaling/snapshotter.ts"],"names":[],"mappings":";;;;AACA,oDAAoB;AAIpB,oDAAiD;AAEjD,gCAA2C;AAE3C,6DAA8D;AAE9D,uDAAwD;AAExD,qEAAqE;AAGrE,6DAAwD;AAuDxD;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAa,WAAW;IAWpB;;;OAGG;IACH,cAAc;QACV,OAAO,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IACxF,CAAC;IAED,gFAAgF;IAChF,IAAI,YAAY;QACZ,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;IACzC,CAAC;IAED,IAAI,kBAAkB;QAClB,OAAO,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;IAC/C,CAAC;IAED,IAAI,eAAe;QACf,OAAO,IAAI,CAAC,YAAY,CAAC,kBAAkB,EAAE,CAAC;IAClD,CAAC;IAED,IAAI,eAAe;QACf,OAAO,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;IAC5C,CAAC;IAED;;OAEG;IACH,YAAY,UAA8B,EAAE;QAtC5C;;;;;WAAS;QACT;;;;;WAAsB;QACtB;;;;;WAAsB;QACtB;;;;;WAAqB;QAEJ;;;;;WAA+B;QAC/B;;;;;WAAqC;QACrC;;;;;WAAyB;QACzB;;;;;WAA+B;QA+B5C,IAAA,YAAE,EACE,OAAO,EACP,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,6BAA6B,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACjD,0BAA0B,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9C,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,gBAAgB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACpC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACvB,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC1B,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SAC7B,CAAC,CACL,CAAC;QAEF,MAAM,EACF,6BAA6B,GAAG,GAAG,EACnC,0BAA0B,GAAG,CAAC,EAC9B,mBAAmB,GAAG,EAAE,EACxB,gBAAgB,GAAG,EAAE,EACrB,kBAAkB,GAAG,GAAG,EACxB,eAAe,GAAG,CAAC,EACnB,GAAG,GAAG,SAAU,EAChB,MAAM,GAAG,6BAAa,CAAC,eAAe,EAAE,EACxC,MAAM,GAAG,MAAM,CAAC,gBAAgB,EAAE,GACrC,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC;QAE5C,MAAM,qBAAqB,GAAG,mBAAmB,GAAG,IAAI,CAAC;QAEzD,IAAI,CAAC,YAAY,GAAG,IAAI,qCAAgB,CAAC;YACrC,kBAAkB;YAClB,qBAAqB;YACrB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,GAAG,EAAE,IAAI,CAAC,GAAG;SAChB,CAAC,CAAC;QAEH,IAAI,CAAC,eAAe,GAAG,IAAA,kDAAyB,EAAC;YAC7C,6BAA6B;YAC7B,gBAAgB;YAChB,qBAAqB;SACxB,CAAC,CAAC;QAEH,IAAI,CAAC,SAAS,GAAG,IAAA,qCAAmB,EAAC;YACjC,qBAAqB;YACrB,MAAM,EAAE,IAAI,CAAC,MAAM;SACtB,CAAC,CAAC;QAEH,IAAI,CAAC,YAAY,GAAG,IAAA,2CAAsB,EAAC;YACvC,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,0BAA0B;YAC1B,eAAe;YACf,qBAAqB;SACxB,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACP,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;QAChC,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QACnC,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QAC7B,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;IACpC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACN,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;QAC/B,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;QAC5B,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;QAC/B,uDAAuD;QACvD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC1B,YAAY,CAAC,OAAO,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACH,eAAe,CAAC,oBAA6B;QACzC,OAAO,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,oBAAoB,CAAC,CAAC;IAC7D,CAAC;IAED;;;OAGG;IACH,kBAAkB,CAAC,oBAA6B;QAC5C,OAAO,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,oBAAoB,CAAC,CAAC;IAChE,CAAC;IAED;;;OAGG;IACH,YAAY,CAAC,oBAA6B;QACtC,OAAO,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,oBAAoB,CAAC,CAAC;IAC1D,CAAC;IAED;;;OAGG;IACH,eAAe,CAAC,oBAA6B;QACzC,OAAO,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,oBAAoB,CAAC,CAAC;IAC7D,CAAC;IAED;;OAEG;IACO,eAAe,CAAC,UAAsB;QAC5C,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;IAChD,CAAC;IAED;;OAEG;IACO,sBAAsB,CAAC,UAAsB;QACnD,IAAI,CAAC,YAAY,CAAC,sBAAsB,CAAC,UAAU,CAAC,CAAC;IACzD,CAAC;IAED;;OAEG;IACO,kBAAkB,CAAC,gBAA+B;QACxD,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;IAClD,CAAC;IAED;;OAEG;IACO,YAAY,CAAC,UAAsB;QACzC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IACtC,CAAC;IAED;;OAEG;IACO,eAAe,CAAC,gBAA+B;QACrD,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;IAC/C,CAAC;IAED;;OAEG;IACO,eAAe,CAAC,UAAiB,EAAE,IAAU;QACnD,mCAAmC;IACvC,CAAC;CACJ;AArMD,kCAqMC"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { Configuration } from '../configuration';
|
|
2
|
+
import type { LoadSignal } from './load_signal';
|
|
2
3
|
import { Snapshotter } from './snapshotter';
|
|
3
4
|
/**
|
|
4
5
|
* Represents the current status of the system.
|
|
@@ -10,6 +11,7 @@ export interface SystemInfo {
|
|
|
10
11
|
eventLoopInfo: ClientInfo;
|
|
11
12
|
cpuInfo: ClientInfo;
|
|
12
13
|
clientInfo: ClientInfo;
|
|
14
|
+
memTotalBytes?: number;
|
|
13
15
|
memCurrentBytes?: number;
|
|
14
16
|
/**
|
|
15
17
|
* Platform only property
|
|
@@ -26,6 +28,11 @@ export interface SystemInfo {
|
|
|
26
28
|
* @internal
|
|
27
29
|
*/
|
|
28
30
|
createdAt?: Date;
|
|
31
|
+
/**
|
|
32
|
+
* Status of additional load signals beyond the built-in four.
|
|
33
|
+
* Keys are `LoadSignal.name` values, values are overload info.
|
|
34
|
+
*/
|
|
35
|
+
loadSignalInfo?: Record<string, ClientInfo>;
|
|
29
36
|
}
|
|
30
37
|
export interface SystemStatusOptions {
|
|
31
38
|
/**
|
|
@@ -61,6 +68,13 @@ export interface SystemStatusOptions {
|
|
|
61
68
|
* The `Snapshotter` instance to be queried for `SystemStatus`.
|
|
62
69
|
*/
|
|
63
70
|
snapshotter?: Snapshotter;
|
|
71
|
+
/**
|
|
72
|
+
* Additional load signals to include in the system status evaluation.
|
|
73
|
+
* These are evaluated alongside the built-in memory, CPU, event loop,
|
|
74
|
+
* and client signals. If any signal reports overload, the system is
|
|
75
|
+
* considered overloaded.
|
|
76
|
+
*/
|
|
77
|
+
loadSignals?: LoadSignal[];
|
|
64
78
|
/** @internal */
|
|
65
79
|
config?: Configuration;
|
|
66
80
|
}
|
|
@@ -105,11 +119,14 @@ export interface FinalStatistics {
|
|
|
105
119
|
*/
|
|
106
120
|
export declare class SystemStatus {
|
|
107
121
|
private readonly currentHistoryMillis;
|
|
108
|
-
private readonly maxMemoryOverloadedRatio;
|
|
109
|
-
private readonly maxEventLoopOverloadedRatio;
|
|
110
|
-
private readonly maxCpuOverloadedRatio;
|
|
111
|
-
private readonly maxClientOverloadedRatio;
|
|
112
122
|
private readonly snapshotter;
|
|
123
|
+
private readonly signals;
|
|
124
|
+
/**
|
|
125
|
+
* Per-signal ratio overrides. The built-in four get their overrides from
|
|
126
|
+
* the legacy `max*OverloadedRatio` options; custom signals use their own
|
|
127
|
+
* `overloadedRatio`.
|
|
128
|
+
*/
|
|
129
|
+
private ratioOverrides;
|
|
113
130
|
constructor(options?: SystemStatusOptions);
|
|
114
131
|
/**
|
|
115
132
|
* Returns an {@link SystemInfo} object with the following structure:
|
|
@@ -149,33 +166,5 @@ export declare class SystemStatus {
|
|
|
149
166
|
* Returns a system status object.
|
|
150
167
|
*/
|
|
151
168
|
protected _isSystemIdle(sampleDurationMillis?: number): SystemInfo;
|
|
152
|
-
/**
|
|
153
|
-
* Returns an object with an isOverloaded property set to true
|
|
154
|
-
* if the memory has been overloaded in the last sampleDurationMillis.
|
|
155
|
-
*/
|
|
156
|
-
protected _isMemoryOverloaded(sampleDurationMillis?: number): ClientInfo;
|
|
157
|
-
/**
|
|
158
|
-
* Returns an object with an isOverloaded property set to true
|
|
159
|
-
* if the event loop has been overloaded in the last sampleDurationMillis.
|
|
160
|
-
*/
|
|
161
|
-
protected _isEventLoopOverloaded(sampleDurationMillis?: number): ClientInfo;
|
|
162
|
-
/**
|
|
163
|
-
* Returns an object with an isOverloaded property set to true
|
|
164
|
-
* if the CPU has been overloaded in the last sampleDurationMillis.
|
|
165
|
-
*/
|
|
166
|
-
protected _isCpuOverloaded(sampleDurationMillis?: number): ClientInfo;
|
|
167
|
-
/**
|
|
168
|
-
* Returns an object with an isOverloaded property set to true
|
|
169
|
-
* if the client has been overloaded in the last sampleDurationMillis.
|
|
170
|
-
*/
|
|
171
|
-
protected _isClientOverloaded(sampleDurationMillis?: number): ClientInfo;
|
|
172
|
-
/**
|
|
173
|
-
* Returns an object with sample information and an isOverloaded property
|
|
174
|
-
* set to true if at least the ratio of snapshots in the sample are overloaded.
|
|
175
|
-
*/
|
|
176
|
-
protected _isSampleOverloaded<T extends {
|
|
177
|
-
createdAt: Date;
|
|
178
|
-
isOverloaded: boolean;
|
|
179
|
-
}>(sample: T[], ratio: number): ClientInfo;
|
|
180
169
|
}
|
|
181
170
|
//# sourceMappingURL=system_status.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"system_status.d.ts","sourceRoot":"","sources":["../../src/autoscaling/system_status.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"system_status.d.ts","sourceRoot":"","sources":["../../src/autoscaling/system_status.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAEhD,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAE5C;;GAEG;AACH,MAAM,WAAW,UAAU;IACvB,4CAA4C;IAC5C,YAAY,EAAE,OAAO,CAAC;IACtB,OAAO,EAAE,UAAU,CAAC;IACpB,aAAa,EAAE,UAAU,CAAC;IAC1B,OAAO,EAAE,UAAU,CAAC;IACpB,UAAU,EAAE,UAAU,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B;;;OAGG;IACH,SAAS,CAAC,EAAE,IAAI,CAAC;IAEjB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;CAC/C;AAED,MAAM,WAAW,mBAAmB;IAChC;;;OAGG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;OAIG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAElC;;;;OAIG;IACH,2BAA2B,CAAC,EAAE,MAAM,CAAC;IAErC;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;;OAIG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAElC;;OAEG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B;;;;;OAKG;IACH,WAAW,CAAC,EAAE,UAAU,EAAE,CAAC;IAE3B,gBAAgB;IAChB,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED,MAAM,WAAW,UAAU;IACvB,YAAY,EAAE,OAAO,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,eAAe;IAC5B,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,8BAA8B,EAAE,MAAM,CAAC;IACvC,gCAAgC,EAAE,MAAM,CAAC;IACzC,yBAAyB,EAAE,MAAM,CAAC;IAClC,uBAAuB,EAAE,MAAM,CAAC;IAChC,0BAA0B,EAAE,MAAM,CAAC;IACnC,aAAa,EAAE,MAAM,CAAC;IACtB,oBAAoB,EAAE,MAAM,CAAC;CAChC;AAKD;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,qBAAa,YAAY;IACrB,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAS;IAC9C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;IAC1C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAe;IAEvC;;;;OAIG;IACH,OAAO,CAAC,cAAc,CAAyB;gBAEnC,OAAO,GAAE,mBAAwB;IAyC7C;;;;;;;;;;;;;;;OAeG;IACH,gBAAgB,IAAI,UAAU;IAI9B;;;;;;;;;;;;;;;OAeG;IACH,mBAAmB,IAAI,UAAU;IAIjC;;OAEG;IACH,SAAS,CAAC,aAAa,CAAC,oBAAoB,CAAC,EAAE,MAAM,GAAG,UAAU;CAkCrE"}
|