@platformatic/watt-extra 1.16.0 → 1.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/test.yml +2 -2
- package/lib/errors.js +7 -1
- package/lib/utils.js +1 -10
- package/lib/watt.js +17 -2
- package/package.json +14 -15
- package/plugins/env.js +3 -1
- package/plugins/flamegraphs.js +386 -59
- package/plugins/update.js +15 -0
- package/test/profiling-toggle.test.js +746 -0
- package/test/scheduler-coordination.test.js +7 -2
- package/test/scheduler.test.js +15 -11
|
@@ -0,0 +1,746 @@
|
|
|
1
|
+
import { test } from 'node:test'
|
|
2
|
+
import { equal, ok, deepEqual } from 'node:assert'
|
|
3
|
+
import { once } from 'node:events'
|
|
4
|
+
import { setTimeout as sleep } from 'node:timers/promises'
|
|
5
|
+
import { WebSocketServer } from 'ws'
|
|
6
|
+
import fastify from 'fastify'
|
|
7
|
+
import { setUpEnvironment } from './helper.js'
|
|
8
|
+
import updatePlugin from '../plugins/update.js'
|
|
9
|
+
import flamegraphsPlugin from '../plugins/flamegraphs.js'
|
|
10
|
+
|
|
11
|
+
const port = 15100
|
|
12
|
+
|
|
13
|
+
function createMockApp (opts = {}) {
|
|
14
|
+
const {
|
|
15
|
+
scalerUrl,
|
|
16
|
+
iccPort = port,
|
|
17
|
+
env = {},
|
|
18
|
+
sendCommandToApplication
|
|
19
|
+
} = opts
|
|
20
|
+
|
|
21
|
+
const eventListeners = new Map()
|
|
22
|
+
const commandCalls = []
|
|
23
|
+
const commandWaiters = []
|
|
24
|
+
|
|
25
|
+
function notifyCommandWaiters () {
|
|
26
|
+
for (let i = commandWaiters.length - 1; i >= 0; i--) {
|
|
27
|
+
const { command, count, resolve } = commandWaiters[i]
|
|
28
|
+
if (commandCalls.filter((c) => c.command === command).length >= count) {
|
|
29
|
+
commandWaiters.splice(i, 1)
|
|
30
|
+
resolve()
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const mockWatt = {
|
|
36
|
+
runtime: {
|
|
37
|
+
getWorkers: async () => ({
|
|
38
|
+
'service-1:0': { application: 'service-1', worker: 0, status: 'started' },
|
|
39
|
+
'service-2:0': { application: 'service-2', worker: 0, status: 'started' }
|
|
40
|
+
}),
|
|
41
|
+
getApplications: async () => ({
|
|
42
|
+
applications: [{ id: 'service-1' }, { id: 'service-2' }]
|
|
43
|
+
}),
|
|
44
|
+
on: (event, listener) => {
|
|
45
|
+
if (!eventListeners.has(event)) {
|
|
46
|
+
eventListeners.set(event, [])
|
|
47
|
+
}
|
|
48
|
+
eventListeners.get(event).push(listener)
|
|
49
|
+
},
|
|
50
|
+
removeListener: (event, listener) => {
|
|
51
|
+
const listeners = eventListeners.get(event)
|
|
52
|
+
if (listeners) {
|
|
53
|
+
const index = listeners.indexOf(listener)
|
|
54
|
+
if (index !== -1) {
|
|
55
|
+
listeners.splice(index, 1)
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
},
|
|
59
|
+
emit: (event, ...args) => {
|
|
60
|
+
const listeners = eventListeners.get(event) || []
|
|
61
|
+
for (const listener of listeners) {
|
|
62
|
+
listener(...args)
|
|
63
|
+
}
|
|
64
|
+
},
|
|
65
|
+
getApplicationDetails: async (id) => {
|
|
66
|
+
return { id, sourceMaps: false }
|
|
67
|
+
},
|
|
68
|
+
sendCommandToApplication: async (workerId, command, options) => {
|
|
69
|
+
commandCalls.push({ workerId, command, options })
|
|
70
|
+
notifyCommandWaiters()
|
|
71
|
+
if (sendCommandToApplication) {
|
|
72
|
+
return sendCommandToApplication(workerId, command, options)
|
|
73
|
+
}
|
|
74
|
+
if (command === 'getProfilingState') {
|
|
75
|
+
return {
|
|
76
|
+
isCapturing: true,
|
|
77
|
+
hasProfile: false,
|
|
78
|
+
isProfilerRunning: false,
|
|
79
|
+
isPaused: true,
|
|
80
|
+
eluThreshold: 0.4,
|
|
81
|
+
latestProfileTimestamp: null
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
return { success: true }
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
const app = {
|
|
90
|
+
log: {
|
|
91
|
+
info: () => {},
|
|
92
|
+
error: () => {},
|
|
93
|
+
warn: () => {},
|
|
94
|
+
debug: () => {}
|
|
95
|
+
},
|
|
96
|
+
instanceConfig: {
|
|
97
|
+
applicationId: 'test-application-id',
|
|
98
|
+
iccServices: scalerUrl
|
|
99
|
+
? { scaler: { url: scalerUrl } }
|
|
100
|
+
: undefined
|
|
101
|
+
},
|
|
102
|
+
instanceId: 'test-pod-123',
|
|
103
|
+
getAuthorizationHeaders: async () => {
|
|
104
|
+
return { Authorization: 'Bearer test-token' }
|
|
105
|
+
},
|
|
106
|
+
getRuntimeId: () => 'test-runtime-id',
|
|
107
|
+
env: {
|
|
108
|
+
PLT_APP_NAME: 'test-app',
|
|
109
|
+
PLT_APP_DIR: '/path/to/app',
|
|
110
|
+
PLT_ICC_URL: `http://localhost:${iccPort}`,
|
|
111
|
+
PLT_DISABLE_FLAMEGRAPHS: false,
|
|
112
|
+
PLT_FLAMEGRAPHS_INTERVAL_SEC: 1,
|
|
113
|
+
PLT_FLAMEGRAPHS_ELU_THRESHOLD: 0,
|
|
114
|
+
PLT_FLAMEGRAPHS_GRACE_PERIOD: 0,
|
|
115
|
+
PLT_FLAMEGRAPHS_ATTEMPT_TIMEOUT: 1000,
|
|
116
|
+
PLT_FLAMEGRAPHS_CACHE_CLEANUP_INTERVAL: 120000,
|
|
117
|
+
PLT_FLAMEGRAPHS_STATE_REPORT_INTERVAL: 30000,
|
|
118
|
+
...env
|
|
119
|
+
},
|
|
120
|
+
watt: mockWatt,
|
|
121
|
+
commandCalls,
|
|
122
|
+
// Resolves when `count` calls of `command` have been recorded
|
|
123
|
+
waitForCommands: (command, count) => {
|
|
124
|
+
if (commandCalls.filter((c) => c.command === command).length >= count) {
|
|
125
|
+
return Promise.resolve()
|
|
126
|
+
}
|
|
127
|
+
const { promise, resolve } = Promise.withResolvers()
|
|
128
|
+
commandWaiters.push({ command, count, resolve })
|
|
129
|
+
return promise
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
return app
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
async function startScalerMock (t, opts = {}) {
|
|
137
|
+
const scaler = fastify({ keepAliveTimeout: 1, forceCloseConnections: true })
|
|
138
|
+
const statesReports = []
|
|
139
|
+
const reportWaiters = []
|
|
140
|
+
|
|
141
|
+
function notifyReportWaiters () {
|
|
142
|
+
for (let i = reportWaiters.length - 1; i >= 0; i--) {
|
|
143
|
+
const { count, resolve } = reportWaiters[i]
|
|
144
|
+
if (statesReports.length >= count) {
|
|
145
|
+
reportWaiters.splice(i, 1)
|
|
146
|
+
resolve()
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
let activeStatesRequests = 0
|
|
152
|
+
let maxConcurrentStatesRequests = 0
|
|
153
|
+
|
|
154
|
+
if (!opts.withoutStatesRoute) {
|
|
155
|
+
scaler.post('/scaler/flamegraphs/states', async (req) => {
|
|
156
|
+
activeStatesRequests++
|
|
157
|
+
maxConcurrentStatesRequests = Math.max(maxConcurrentStatesRequests, activeStatesRequests)
|
|
158
|
+
if (opts.statesDelay) {
|
|
159
|
+
await sleep(opts.statesDelay)
|
|
160
|
+
}
|
|
161
|
+
activeStatesRequests--
|
|
162
|
+
statesReports.push(req.body)
|
|
163
|
+
notifyReportWaiters()
|
|
164
|
+
return {}
|
|
165
|
+
})
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
await scaler.listen({ port: 0, host: '127.0.0.1' })
|
|
169
|
+
t.after(() => scaler.close())
|
|
170
|
+
|
|
171
|
+
// Resolves when `count` state reports have been received
|
|
172
|
+
function waitForReports (count = 1) {
|
|
173
|
+
if (statesReports.length >= count) {
|
|
174
|
+
return Promise.resolve()
|
|
175
|
+
}
|
|
176
|
+
const { promise, resolve } = Promise.withResolvers()
|
|
177
|
+
reportWaiters.push({ count, resolve })
|
|
178
|
+
return promise
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
const url = `http://127.0.0.1:${scaler.server.address().port}/scaler`
|
|
182
|
+
return {
|
|
183
|
+
scaler,
|
|
184
|
+
statesReports,
|
|
185
|
+
url,
|
|
186
|
+
waitForReports,
|
|
187
|
+
getMaxConcurrentStatesRequests: () => maxConcurrentStatesRequests
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
// The worker-started listener logs this message synchronously before arming a
|
|
192
|
+
// worker, so its absence right after a synchronous emit proves the listener
|
|
193
|
+
// took the deactivated branch, without waiting on wall-clock time.
|
|
194
|
+
const ARMING_LOG = 'Starting profiling on new worker'
|
|
195
|
+
|
|
196
|
+
function recordInfoLogs (app) {
|
|
197
|
+
const messages = []
|
|
198
|
+
app.log.info = (...args) => {
|
|
199
|
+
messages.push(args[args.length - 1])
|
|
200
|
+
}
|
|
201
|
+
return messages
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
test('setProfilingEnabled(false) stops profiling for both types on every worker', async (t) => {
|
|
205
|
+
setUpEnvironment()
|
|
206
|
+
|
|
207
|
+
const { statesReports, url, waitForReports } = await startScalerMock(t)
|
|
208
|
+
const app = createMockApp({ scalerUrl: url })
|
|
209
|
+
|
|
210
|
+
await flamegraphsPlugin(app)
|
|
211
|
+
|
|
212
|
+
const result = await app.setProfilingEnabled(false)
|
|
213
|
+
equal(result.success, true)
|
|
214
|
+
equal(result.enabled, false)
|
|
215
|
+
|
|
216
|
+
const stopCalls = app.commandCalls.filter((c) => c.command === 'stopProfiling')
|
|
217
|
+
equal(stopCalls.length, 4)
|
|
218
|
+
for (const workerId of ['service-1:0', 'service-2:0']) {
|
|
219
|
+
for (const type of ['cpu', 'heap']) {
|
|
220
|
+
ok(stopCalls.find((c) => c.workerId === workerId && c.options.type === type))
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
await waitForReports(1)
|
|
225
|
+
equal(statesReports.length, 1)
|
|
226
|
+
})
|
|
227
|
+
|
|
228
|
+
test('setProfilingEnabled(true) after a stop restarts profiling with the boot options', async (t) => {
|
|
229
|
+
setUpEnvironment()
|
|
230
|
+
|
|
231
|
+
const { url } = await startScalerMock(t)
|
|
232
|
+
const app = createMockApp({ scalerUrl: url })
|
|
233
|
+
|
|
234
|
+
await flamegraphsPlugin(app)
|
|
235
|
+
|
|
236
|
+
await app.setProfilingEnabled(false)
|
|
237
|
+
app.commandCalls.length = 0
|
|
238
|
+
|
|
239
|
+
const result = await app.setProfilingEnabled(true)
|
|
240
|
+
equal(result.success, true)
|
|
241
|
+
deepEqual(result.types.sort(), ['cpu', 'heap'])
|
|
242
|
+
|
|
243
|
+
const startCalls = app.commandCalls.filter((c) => c.command === 'startProfiling')
|
|
244
|
+
equal(startCalls.length, 4)
|
|
245
|
+
for (const call of startCalls) {
|
|
246
|
+
equal(call.options.durationMillis, 1000)
|
|
247
|
+
equal(call.options.eluThreshold, 0)
|
|
248
|
+
equal(call.options.sourceMaps, false)
|
|
249
|
+
}
|
|
250
|
+
})
|
|
251
|
+
|
|
252
|
+
test('already-started and not-started profiler errors are treated as success', async (t) => {
|
|
253
|
+
setUpEnvironment()
|
|
254
|
+
|
|
255
|
+
const { url } = await startScalerMock(t)
|
|
256
|
+
const app = createMockApp({
|
|
257
|
+
scalerUrl: url,
|
|
258
|
+
sendCommandToApplication: async (workerId, command) => {
|
|
259
|
+
if (command === 'startProfiling') {
|
|
260
|
+
const err = new Error('Profiling already started')
|
|
261
|
+
err.code = 'PLT_PPROF_PROFILING_ALREADY_STARTED'
|
|
262
|
+
throw err
|
|
263
|
+
}
|
|
264
|
+
if (command === 'stopProfiling') {
|
|
265
|
+
const err = new Error('Profiling not started')
|
|
266
|
+
err.code = 'PLT_PPROF_PROFILING_NOT_STARTED'
|
|
267
|
+
throw err
|
|
268
|
+
}
|
|
269
|
+
return {}
|
|
270
|
+
}
|
|
271
|
+
})
|
|
272
|
+
|
|
273
|
+
await flamegraphsPlugin(app)
|
|
274
|
+
|
|
275
|
+
const stopResult = await app.setProfilingEnabled(false)
|
|
276
|
+
equal(stopResult.success, true)
|
|
277
|
+
|
|
278
|
+
const startResult = await app.setProfilingEnabled(true)
|
|
279
|
+
equal(startResult.success, true)
|
|
280
|
+
})
|
|
281
|
+
|
|
282
|
+
test('a worker started while profiling is deactivated is not armed', async (t) => {
|
|
283
|
+
setUpEnvironment()
|
|
284
|
+
|
|
285
|
+
const { url } = await startScalerMock(t)
|
|
286
|
+
const app = createMockApp({ scalerUrl: url })
|
|
287
|
+
|
|
288
|
+
await flamegraphsPlugin(app)
|
|
289
|
+
await app.setupFlamegraphs()
|
|
290
|
+
t.after(() => app.cleanupFlamegraphs())
|
|
291
|
+
|
|
292
|
+
await app.setProfilingEnabled(false)
|
|
293
|
+
app.commandCalls.length = 0
|
|
294
|
+
const infoLogs = recordInfoLogs(app)
|
|
295
|
+
|
|
296
|
+
// The mock runtime dispatches listeners synchronously and the arming log
|
|
297
|
+
// happens before any await, so no waiting is needed for the negative check
|
|
298
|
+
app.watt.runtime.emit('application:worker:started', {
|
|
299
|
+
application: 'service-3',
|
|
300
|
+
worker: 0
|
|
301
|
+
})
|
|
302
|
+
|
|
303
|
+
ok(!infoLogs.includes(ARMING_LOG))
|
|
304
|
+
const startCalls = app.commandCalls.filter((c) => c.command === 'startProfiling')
|
|
305
|
+
equal(startCalls.length, 0)
|
|
306
|
+
})
|
|
307
|
+
|
|
308
|
+
test('a worker started while profiling is activated is armed', async (t) => {
|
|
309
|
+
setUpEnvironment()
|
|
310
|
+
|
|
311
|
+
const { url } = await startScalerMock(t)
|
|
312
|
+
const app = createMockApp({ scalerUrl: url })
|
|
313
|
+
|
|
314
|
+
await flamegraphsPlugin(app)
|
|
315
|
+
await app.setupFlamegraphs()
|
|
316
|
+
t.after(() => app.cleanupFlamegraphs())
|
|
317
|
+
|
|
318
|
+
await app.setProfilingEnabled(false)
|
|
319
|
+
await app.setProfilingEnabled(true)
|
|
320
|
+
app.commandCalls.length = 0
|
|
321
|
+
|
|
322
|
+
app.watt.runtime.emit('application:worker:started', {
|
|
323
|
+
application: 'service-3',
|
|
324
|
+
worker: 0
|
|
325
|
+
})
|
|
326
|
+
await app.waitForCommands('startProfiling', 2)
|
|
327
|
+
|
|
328
|
+
const startCalls = app.commandCalls.filter((c) => c.command === 'startProfiling')
|
|
329
|
+
equal(startCalls.length, 2)
|
|
330
|
+
ok(startCalls.every((c) => c.workerId === 'service-3:0'))
|
|
331
|
+
})
|
|
332
|
+
|
|
333
|
+
test('a re-setup after an ICC recovery does not re-enable a deactivated pod', async (t) => {
|
|
334
|
+
setUpEnvironment()
|
|
335
|
+
|
|
336
|
+
const { url } = await startScalerMock(t)
|
|
337
|
+
const app = createMockApp({ scalerUrl: url })
|
|
338
|
+
|
|
339
|
+
await flamegraphsPlugin(app)
|
|
340
|
+
await app.setupFlamegraphs()
|
|
341
|
+
t.after(() => app.cleanupFlamegraphs())
|
|
342
|
+
|
|
343
|
+
await app.setProfilingEnabled(false)
|
|
344
|
+
app.commandCalls.length = 0
|
|
345
|
+
|
|
346
|
+
// setupFlamegraphs awaits its start commands internally, so the check
|
|
347
|
+
// right after it is deterministic
|
|
348
|
+
await app.setupFlamegraphs()
|
|
349
|
+
|
|
350
|
+
const startCalls = app.commandCalls.filter((c) => c.command === 'startProfiling')
|
|
351
|
+
equal(startCalls.length, 0)
|
|
352
|
+
})
|
|
353
|
+
|
|
354
|
+
test('a pod started with PLT_DISABLE_FLAMEGRAPHS boots deactivated but reports its state', async (t) => {
|
|
355
|
+
setUpEnvironment()
|
|
356
|
+
|
|
357
|
+
const { statesReports, url, waitForReports } = await startScalerMock(t)
|
|
358
|
+
const app = createMockApp({
|
|
359
|
+
scalerUrl: url,
|
|
360
|
+
env: { PLT_DISABLE_FLAMEGRAPHS: true }
|
|
361
|
+
})
|
|
362
|
+
|
|
363
|
+
await flamegraphsPlugin(app)
|
|
364
|
+
await app.setupFlamegraphs()
|
|
365
|
+
t.after(() => app.cleanupFlamegraphs())
|
|
366
|
+
await waitForReports(1)
|
|
367
|
+
|
|
368
|
+
const startCalls = app.commandCalls.filter((c) => c.command === 'startProfiling')
|
|
369
|
+
equal(startCalls.length, 0)
|
|
370
|
+
|
|
371
|
+
equal(statesReports.length, 1)
|
|
372
|
+
ok(statesReports[0].states.every((s) => s.enabled === false))
|
|
373
|
+
})
|
|
374
|
+
|
|
375
|
+
test('a pod started with PLT_DISABLE_FLAMEGRAPHS can be activated at runtime', async (t) => {
|
|
376
|
+
setUpEnvironment()
|
|
377
|
+
|
|
378
|
+
const { url } = await startScalerMock(t)
|
|
379
|
+
const app = createMockApp({
|
|
380
|
+
scalerUrl: url,
|
|
381
|
+
env: { PLT_DISABLE_FLAMEGRAPHS: true }
|
|
382
|
+
})
|
|
383
|
+
|
|
384
|
+
await flamegraphsPlugin(app)
|
|
385
|
+
await app.setupFlamegraphs()
|
|
386
|
+
t.after(() => app.cleanupFlamegraphs())
|
|
387
|
+
|
|
388
|
+
const result = await app.setProfilingEnabled(true)
|
|
389
|
+
equal(result.success, true)
|
|
390
|
+
deepEqual(result.types.sort(), ['cpu', 'heap'])
|
|
391
|
+
|
|
392
|
+
let startCalls = app.commandCalls.filter((c) => c.command === 'startProfiling')
|
|
393
|
+
equal(startCalls.length, 4)
|
|
394
|
+
|
|
395
|
+
// A worker started after the activation is armed too
|
|
396
|
+
app.commandCalls.length = 0
|
|
397
|
+
app.watt.runtime.emit('application:worker:started', {
|
|
398
|
+
application: 'service-3',
|
|
399
|
+
worker: 0
|
|
400
|
+
})
|
|
401
|
+
await app.waitForCommands('startProfiling', 2)
|
|
402
|
+
|
|
403
|
+
startCalls = app.commandCalls.filter((c) => c.command === 'startProfiling')
|
|
404
|
+
equal(startCalls.length, 2)
|
|
405
|
+
|
|
406
|
+
// And it can be deactivated again
|
|
407
|
+
app.commandCalls.length = 0
|
|
408
|
+
await app.setProfilingEnabled(false)
|
|
409
|
+
const stopCalls = app.commandCalls.filter((c) => c.command === 'stopProfiling')
|
|
410
|
+
equal(stopCalls.length, 4)
|
|
411
|
+
})
|
|
412
|
+
|
|
413
|
+
test('sendFlamegraphs works on a runtime-activated pod started with PLT_DISABLE_FLAMEGRAPHS', async (t) => {
|
|
414
|
+
setUpEnvironment()
|
|
415
|
+
|
|
416
|
+
const { url } = await startScalerMock(t)
|
|
417
|
+
const app = createMockApp({
|
|
418
|
+
scalerUrl: url,
|
|
419
|
+
env: { PLT_DISABLE_FLAMEGRAPHS: true }
|
|
420
|
+
})
|
|
421
|
+
let profileRequests = 0
|
|
422
|
+
app.watt.runtime.getApplicationLastProfile = async () => {
|
|
423
|
+
profileRequests++
|
|
424
|
+
return { profile: new Uint8Array([1]), timestamp: Date.now(), preserved: false }
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
await flamegraphsPlugin(app)
|
|
428
|
+
|
|
429
|
+
await app.sendFlamegraphs({ profileType: 'cpu' })
|
|
430
|
+
equal(profileRequests, 0)
|
|
431
|
+
|
|
432
|
+
await app.setProfilingEnabled(true)
|
|
433
|
+
await app.sendFlamegraphs({ profileType: 'cpu' })
|
|
434
|
+
equal(profileRequests, 2)
|
|
435
|
+
})
|
|
436
|
+
|
|
437
|
+
test('sendFlamegraphs on a deactivated pod returns early without collecting', async (t) => {
|
|
438
|
+
setUpEnvironment()
|
|
439
|
+
|
|
440
|
+
const { url } = await startScalerMock(t)
|
|
441
|
+
const app = createMockApp({ scalerUrl: url })
|
|
442
|
+
let profileRequests = 0
|
|
443
|
+
app.watt.runtime.getApplicationLastProfile = async () => {
|
|
444
|
+
profileRequests++
|
|
445
|
+
return { profile: new Uint8Array([1]), timestamp: Date.now(), preserved: false }
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
await flamegraphsPlugin(app)
|
|
449
|
+
await app.setProfilingEnabled(false)
|
|
450
|
+
|
|
451
|
+
await app.sendFlamegraphs({ profileType: 'cpu' })
|
|
452
|
+
equal(profileRequests, 0)
|
|
453
|
+
})
|
|
454
|
+
|
|
455
|
+
test('reportProfilingStates posts the desired and real state of every worker', async (t) => {
|
|
456
|
+
setUpEnvironment()
|
|
457
|
+
|
|
458
|
+
const { statesReports, url } = await startScalerMock(t)
|
|
459
|
+
const app = createMockApp({ scalerUrl: url })
|
|
460
|
+
|
|
461
|
+
await flamegraphsPlugin(app)
|
|
462
|
+
await app.reportProfilingStates()
|
|
463
|
+
|
|
464
|
+
equal(statesReports.length, 1)
|
|
465
|
+
const report = statesReports[0]
|
|
466
|
+
equal(report.applicationId, 'test-application-id')
|
|
467
|
+
equal(report.podId, 'test-pod-123')
|
|
468
|
+
equal(report.expiresIn, 90000)
|
|
469
|
+
equal(report.states.length, 4)
|
|
470
|
+
|
|
471
|
+
const state = report.states.find(
|
|
472
|
+
(s) => s.workerId === 'service-1:0' && s.type === 'cpu'
|
|
473
|
+
)
|
|
474
|
+
equal(state.serviceId, 'service-1')
|
|
475
|
+
equal(state.enabled, true)
|
|
476
|
+
equal(state.isCapturing, true)
|
|
477
|
+
equal(state.isPaused, true)
|
|
478
|
+
})
|
|
479
|
+
|
|
480
|
+
test('reportProfilingStates disables itself when the scaler does not support it', async (t) => {
|
|
481
|
+
setUpEnvironment()
|
|
482
|
+
|
|
483
|
+
const { statesReports, url } = await startScalerMock(t, { withoutStatesRoute: true })
|
|
484
|
+
const app = createMockApp({ scalerUrl: url })
|
|
485
|
+
|
|
486
|
+
let warnings = 0
|
|
487
|
+
app.log.warn = () => { warnings++ }
|
|
488
|
+
|
|
489
|
+
await flamegraphsPlugin(app)
|
|
490
|
+
await app.reportProfilingStates()
|
|
491
|
+
await app.reportProfilingStates()
|
|
492
|
+
|
|
493
|
+
equal(statesReports.length, 0)
|
|
494
|
+
equal(warnings, 1)
|
|
495
|
+
})
|
|
496
|
+
|
|
497
|
+
test('a worker without the profiling state command is reported as unsupported', async (t) => {
|
|
498
|
+
setUpEnvironment()
|
|
499
|
+
|
|
500
|
+
const { statesReports, url } = await startScalerMock(t)
|
|
501
|
+
const app = createMockApp({
|
|
502
|
+
scalerUrl: url,
|
|
503
|
+
sendCommandToApplication: async (workerId, command) => {
|
|
504
|
+
if (command === 'getProfilingState') {
|
|
505
|
+
return {}
|
|
506
|
+
}
|
|
507
|
+
return { success: true }
|
|
508
|
+
}
|
|
509
|
+
})
|
|
510
|
+
|
|
511
|
+
await flamegraphsPlugin(app)
|
|
512
|
+
await app.reportProfilingStates()
|
|
513
|
+
|
|
514
|
+
equal(statesReports.length, 1)
|
|
515
|
+
const report = statesReports[0]
|
|
516
|
+
equal(report.states.length, 4)
|
|
517
|
+
for (const state of report.states) {
|
|
518
|
+
equal(state.unsupported, true)
|
|
519
|
+
equal(state.enabled, true)
|
|
520
|
+
}
|
|
521
|
+
})
|
|
522
|
+
|
|
523
|
+
test('handles start-profiling and stop-profiling commands from ICC', async (t) => {
|
|
524
|
+
setUpEnvironment()
|
|
525
|
+
|
|
526
|
+
const receivedMessages = []
|
|
527
|
+
const wsPort = port + 1
|
|
528
|
+
const wss = new WebSocketServer({ port: wsPort })
|
|
529
|
+
t.after(async () => wss.close())
|
|
530
|
+
|
|
531
|
+
let ws = null
|
|
532
|
+
const waitForClientSubscription = once(wss, 'connection').then(([socket]) => {
|
|
533
|
+
ws = socket
|
|
534
|
+
return new Promise((resolve) => {
|
|
535
|
+
socket.on('message', (data) => {
|
|
536
|
+
const message = JSON.parse(data.toString())
|
|
537
|
+
receivedMessages.push(message)
|
|
538
|
+
if (message.command === 'subscribe' && message.topic === '/config') {
|
|
539
|
+
socket.send(JSON.stringify({ command: 'ack' }))
|
|
540
|
+
resolve()
|
|
541
|
+
}
|
|
542
|
+
})
|
|
543
|
+
})
|
|
544
|
+
})
|
|
545
|
+
|
|
546
|
+
const { url } = await startScalerMock(t)
|
|
547
|
+
const app = createMockApp({ scalerUrl: url, iccPort: wsPort })
|
|
548
|
+
|
|
549
|
+
await updatePlugin(app)
|
|
550
|
+
await flamegraphsPlugin(app)
|
|
551
|
+
await app.connectToUpdates()
|
|
552
|
+
t.after(async () => {
|
|
553
|
+
await app.cleanupFlamegraphs()
|
|
554
|
+
await app.closeUpdates()
|
|
555
|
+
})
|
|
556
|
+
|
|
557
|
+
await waitForClientSubscription
|
|
558
|
+
|
|
559
|
+
ws.send(JSON.stringify({ command: 'stop-profiling', params: {} }))
|
|
560
|
+
await app.waitForCommands('stopProfiling', 4)
|
|
561
|
+
|
|
562
|
+
const stopCalls = app.commandCalls.filter((c) => c.command === 'stopProfiling')
|
|
563
|
+
equal(stopCalls.length, 4)
|
|
564
|
+
|
|
565
|
+
app.commandCalls.length = 0
|
|
566
|
+
ws.send(JSON.stringify({ command: 'start-profiling', params: { types: ['cpu'] } }))
|
|
567
|
+
await app.waitForCommands('startProfiling', 2)
|
|
568
|
+
|
|
569
|
+
const startCalls = app.commandCalls.filter((c) => c.command === 'startProfiling')
|
|
570
|
+
equal(startCalls.length, 2)
|
|
571
|
+
ok(startCalls.every((c) => c.options.type === 'cpu'))
|
|
572
|
+
|
|
573
|
+
app.commandCalls.length = 0
|
|
574
|
+
ws.send(JSON.stringify({ command: 'stop-profiling', params: { types: ['cpu'] } }))
|
|
575
|
+
await app.waitForCommands('stopProfiling', 2)
|
|
576
|
+
|
|
577
|
+
const cpuStopCalls = app.commandCalls.filter((c) => c.command === 'stopProfiling')
|
|
578
|
+
equal(cpuStopCalls.length, 2)
|
|
579
|
+
ok(cpuStopCalls.every((c) => c.options.type === 'cpu'))
|
|
580
|
+
})
|
|
581
|
+
|
|
582
|
+
test('a partial worker failure is not reported as success and the report carries the real state', async (t) => {
|
|
583
|
+
setUpEnvironment()
|
|
584
|
+
|
|
585
|
+
const { statesReports, url, waitForReports } = await startScalerMock(t)
|
|
586
|
+
const failingWorker = 'service-2:0'
|
|
587
|
+
const capturing = { 'service-1:0': false, 'service-2:0': false }
|
|
588
|
+
const app = createMockApp({
|
|
589
|
+
scalerUrl: url,
|
|
590
|
+
sendCommandToApplication: async (workerId, command, options) => {
|
|
591
|
+
if (command === 'startProfiling') {
|
|
592
|
+
if (workerId === failingWorker) {
|
|
593
|
+
throw new Error('boom')
|
|
594
|
+
}
|
|
595
|
+
capturing[workerId] = true
|
|
596
|
+
return {}
|
|
597
|
+
}
|
|
598
|
+
if (command === 'stopProfiling') {
|
|
599
|
+
capturing[workerId] = false
|
|
600
|
+
return {}
|
|
601
|
+
}
|
|
602
|
+
if (command === 'getProfilingState') {
|
|
603
|
+
return {
|
|
604
|
+
isCapturing: capturing[workerId],
|
|
605
|
+
isProfilerRunning: capturing[workerId],
|
|
606
|
+
isPaused: false
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
return {}
|
|
610
|
+
}
|
|
611
|
+
})
|
|
612
|
+
|
|
613
|
+
await flamegraphsPlugin(app)
|
|
614
|
+
|
|
615
|
+
await app.setProfilingEnabled(false)
|
|
616
|
+
const result = await app.setProfilingEnabled(true)
|
|
617
|
+
|
|
618
|
+
// The toggle is not a success and the intent is still recorded
|
|
619
|
+
equal(result.success, false)
|
|
620
|
+
equal(result.failures, 1)
|
|
621
|
+
deepEqual(result.types.sort(), ['cpu', 'heap'])
|
|
622
|
+
|
|
623
|
+
// The report distinguishes intent (enabled) from reality (isCapturing)
|
|
624
|
+
await waitForReports(2)
|
|
625
|
+
const report = statesReports[statesReports.length - 1]
|
|
626
|
+
const failedState = report.states.find((s) => s.workerId === failingWorker && s.type === 'cpu')
|
|
627
|
+
equal(failedState.enabled, true)
|
|
628
|
+
equal(failedState.isCapturing, false)
|
|
629
|
+
const okState = report.states.find((s) => s.workerId === 'service-1:0' && s.type === 'cpu')
|
|
630
|
+
equal(okState.enabled, true)
|
|
631
|
+
equal(okState.isCapturing, true)
|
|
632
|
+
})
|
|
633
|
+
|
|
634
|
+
test('a blocked worker does not stall the state report', async (t) => {
|
|
635
|
+
setUpEnvironment()
|
|
636
|
+
|
|
637
|
+
const { statesReports, url, waitForReports } = await startScalerMock(t)
|
|
638
|
+
const app = createMockApp({
|
|
639
|
+
scalerUrl: url,
|
|
640
|
+
env: { PLT_FLAMEGRAPHS_STATE_QUERY_TIMEOUT: 100 },
|
|
641
|
+
sendCommandToApplication: async (workerId, command) => {
|
|
642
|
+
if (command === 'getProfilingState' && workerId === 'service-1:0') {
|
|
643
|
+
// A blocked worker event loop: the query never settles
|
|
644
|
+
return new Promise(() => {})
|
|
645
|
+
}
|
|
646
|
+
if (command === 'getProfilingState') {
|
|
647
|
+
return {
|
|
648
|
+
isCapturing: true,
|
|
649
|
+
isProfilerRunning: true,
|
|
650
|
+
isPaused: false
|
|
651
|
+
}
|
|
652
|
+
}
|
|
653
|
+
return {}
|
|
654
|
+
}
|
|
655
|
+
})
|
|
656
|
+
|
|
657
|
+
await flamegraphsPlugin(app)
|
|
658
|
+
await app.reportProfilingStates()
|
|
659
|
+
await waitForReports(1)
|
|
660
|
+
|
|
661
|
+
// The blocked worker is reported as unresponsive, the healthy one normally
|
|
662
|
+
const report = statesReports[0]
|
|
663
|
+
equal(report.states.length, 4)
|
|
664
|
+
const blockedStates = report.states.filter((s) => s.workerId === 'service-1:0')
|
|
665
|
+
equal(blockedStates.length, 2)
|
|
666
|
+
ok(blockedStates.every((s) => s.unresponsive === true))
|
|
667
|
+
const healthyStates = report.states.filter((s) => s.workerId === 'service-2:0')
|
|
668
|
+
equal(healthyStates.length, 2)
|
|
669
|
+
ok(healthyStates.every((s) => s.isCapturing === true && s.unresponsive === undefined))
|
|
670
|
+
})
|
|
671
|
+
|
|
672
|
+
test('overlapping state reports are coalesced', async (t) => {
|
|
673
|
+
setUpEnvironment()
|
|
674
|
+
|
|
675
|
+
const mock = await startScalerMock(t, { statesDelay: 100 })
|
|
676
|
+
const app = createMockApp({ scalerUrl: mock.url })
|
|
677
|
+
|
|
678
|
+
await flamegraphsPlugin(app)
|
|
679
|
+
|
|
680
|
+
// The second call while the first is in flight queues a single re-run
|
|
681
|
+
const first = app.reportProfilingStates()
|
|
682
|
+
const second = app.reportProfilingStates()
|
|
683
|
+
equal(first, second)
|
|
684
|
+
|
|
685
|
+
await first
|
|
686
|
+
await mock.waitForReports(2)
|
|
687
|
+
|
|
688
|
+
equal(mock.statesReports.length, 2)
|
|
689
|
+
equal(mock.getMaxConcurrentStatesRequests(), 1)
|
|
690
|
+
})
|
|
691
|
+
|
|
692
|
+
test('a timed-out worker is not queried again until it restarts', async (t) => {
|
|
693
|
+
setUpEnvironment()
|
|
694
|
+
|
|
695
|
+
const { statesReports, url, waitForReports } = await startScalerMock(t)
|
|
696
|
+
const app = createMockApp({
|
|
697
|
+
scalerUrl: url,
|
|
698
|
+
env: { PLT_FLAMEGRAPHS_STATE_QUERY_TIMEOUT: 100 },
|
|
699
|
+
sendCommandToApplication: async (workerId, command) => {
|
|
700
|
+
if (command === 'getProfilingState' && workerId === 'service-1:0') {
|
|
701
|
+
// A blocked worker event loop: the query never settles
|
|
702
|
+
return new Promise(() => {})
|
|
703
|
+
}
|
|
704
|
+
if (command === 'getProfilingState') {
|
|
705
|
+
return {
|
|
706
|
+
isCapturing: true,
|
|
707
|
+
isProfilerRunning: true,
|
|
708
|
+
isPaused: false
|
|
709
|
+
}
|
|
710
|
+
}
|
|
711
|
+
return {}
|
|
712
|
+
}
|
|
713
|
+
})
|
|
714
|
+
|
|
715
|
+
const queriesToBlockedWorker = () => app.commandCalls.filter(
|
|
716
|
+
(c) => c.command === 'getProfilingState' && c.workerId === 'service-1:0'
|
|
717
|
+
).length
|
|
718
|
+
|
|
719
|
+
await flamegraphsPlugin(app)
|
|
720
|
+
await app.setupFlamegraphs()
|
|
721
|
+
t.after(() => app.cleanupFlamegraphs())
|
|
722
|
+
|
|
723
|
+
// The first report queries the blocked worker, times out, and suppresses it
|
|
724
|
+
await waitForReports(1)
|
|
725
|
+
equal(queriesToBlockedWorker(), 2)
|
|
726
|
+
|
|
727
|
+
// The next report does not query the suppressed worker but still reports
|
|
728
|
+
// it as unresponsive
|
|
729
|
+
await app.reportProfilingStates()
|
|
730
|
+
await waitForReports(2)
|
|
731
|
+
equal(queriesToBlockedWorker(), 2)
|
|
732
|
+
const suppressedReport = statesReports[statesReports.length - 1]
|
|
733
|
+
equal(suppressedReport.states.length, 4)
|
|
734
|
+
ok(suppressedReport.states
|
|
735
|
+
.filter((s) => s.workerId === 'service-1:0')
|
|
736
|
+
.every((s) => s.unresponsive === true))
|
|
737
|
+
|
|
738
|
+
// A worker restart clears the suppression and it is queried again
|
|
739
|
+
app.watt.runtime.emit('application:worker:started', {
|
|
740
|
+
application: 'service-1',
|
|
741
|
+
worker: 0
|
|
742
|
+
})
|
|
743
|
+
await app.reportProfilingStates()
|
|
744
|
+
await waitForReports(3)
|
|
745
|
+
equal(queriesToBlockedWorker(), 4)
|
|
746
|
+
})
|