@platformatic/watt-extra 1.16.0 → 1.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,746 @@
1
+ import { test } from 'node:test'
2
+ import { equal, ok, deepEqual } from 'node:assert'
3
+ import { once } from 'node:events'
4
+ import { setTimeout as sleep } from 'node:timers/promises'
5
+ import { WebSocketServer } from 'ws'
6
+ import fastify from 'fastify'
7
+ import { setUpEnvironment } from './helper.js'
8
+ import updatePlugin from '../plugins/update.js'
9
+ import flamegraphsPlugin from '../plugins/flamegraphs.js'
10
+
11
+ const port = 15100
12
+
13
+ function createMockApp (opts = {}) {
14
+ const {
15
+ scalerUrl,
16
+ iccPort = port,
17
+ env = {},
18
+ sendCommandToApplication
19
+ } = opts
20
+
21
+ const eventListeners = new Map()
22
+ const commandCalls = []
23
+ const commandWaiters = []
24
+
25
+ function notifyCommandWaiters () {
26
+ for (let i = commandWaiters.length - 1; i >= 0; i--) {
27
+ const { command, count, resolve } = commandWaiters[i]
28
+ if (commandCalls.filter((c) => c.command === command).length >= count) {
29
+ commandWaiters.splice(i, 1)
30
+ resolve()
31
+ }
32
+ }
33
+ }
34
+
35
+ const mockWatt = {
36
+ runtime: {
37
+ getWorkers: async () => ({
38
+ 'service-1:0': { application: 'service-1', worker: 0, status: 'started' },
39
+ 'service-2:0': { application: 'service-2', worker: 0, status: 'started' }
40
+ }),
41
+ getApplications: async () => ({
42
+ applications: [{ id: 'service-1' }, { id: 'service-2' }]
43
+ }),
44
+ on: (event, listener) => {
45
+ if (!eventListeners.has(event)) {
46
+ eventListeners.set(event, [])
47
+ }
48
+ eventListeners.get(event).push(listener)
49
+ },
50
+ removeListener: (event, listener) => {
51
+ const listeners = eventListeners.get(event)
52
+ if (listeners) {
53
+ const index = listeners.indexOf(listener)
54
+ if (index !== -1) {
55
+ listeners.splice(index, 1)
56
+ }
57
+ }
58
+ },
59
+ emit: (event, ...args) => {
60
+ const listeners = eventListeners.get(event) || []
61
+ for (const listener of listeners) {
62
+ listener(...args)
63
+ }
64
+ },
65
+ getApplicationDetails: async (id) => {
66
+ return { id, sourceMaps: false }
67
+ },
68
+ sendCommandToApplication: async (workerId, command, options) => {
69
+ commandCalls.push({ workerId, command, options })
70
+ notifyCommandWaiters()
71
+ if (sendCommandToApplication) {
72
+ return sendCommandToApplication(workerId, command, options)
73
+ }
74
+ if (command === 'getProfilingState') {
75
+ return {
76
+ isCapturing: true,
77
+ hasProfile: false,
78
+ isProfilerRunning: false,
79
+ isPaused: true,
80
+ eluThreshold: 0.4,
81
+ latestProfileTimestamp: null
82
+ }
83
+ }
84
+ return { success: true }
85
+ }
86
+ }
87
+ }
88
+
89
+ const app = {
90
+ log: {
91
+ info: () => {},
92
+ error: () => {},
93
+ warn: () => {},
94
+ debug: () => {}
95
+ },
96
+ instanceConfig: {
97
+ applicationId: 'test-application-id',
98
+ iccServices: scalerUrl
99
+ ? { scaler: { url: scalerUrl } }
100
+ : undefined
101
+ },
102
+ instanceId: 'test-pod-123',
103
+ getAuthorizationHeaders: async () => {
104
+ return { Authorization: 'Bearer test-token' }
105
+ },
106
+ getRuntimeId: () => 'test-runtime-id',
107
+ env: {
108
+ PLT_APP_NAME: 'test-app',
109
+ PLT_APP_DIR: '/path/to/app',
110
+ PLT_ICC_URL: `http://localhost:${iccPort}`,
111
+ PLT_DISABLE_FLAMEGRAPHS: false,
112
+ PLT_FLAMEGRAPHS_INTERVAL_SEC: 1,
113
+ PLT_FLAMEGRAPHS_ELU_THRESHOLD: 0,
114
+ PLT_FLAMEGRAPHS_GRACE_PERIOD: 0,
115
+ PLT_FLAMEGRAPHS_ATTEMPT_TIMEOUT: 1000,
116
+ PLT_FLAMEGRAPHS_CACHE_CLEANUP_INTERVAL: 120000,
117
+ PLT_FLAMEGRAPHS_STATE_REPORT_INTERVAL: 30000,
118
+ ...env
119
+ },
120
+ watt: mockWatt,
121
+ commandCalls,
122
+ // Resolves when `count` calls of `command` have been recorded
123
+ waitForCommands: (command, count) => {
124
+ if (commandCalls.filter((c) => c.command === command).length >= count) {
125
+ return Promise.resolve()
126
+ }
127
+ const { promise, resolve } = Promise.withResolvers()
128
+ commandWaiters.push({ command, count, resolve })
129
+ return promise
130
+ }
131
+ }
132
+
133
+ return app
134
+ }
135
+
136
+ async function startScalerMock (t, opts = {}) {
137
+ const scaler = fastify({ keepAliveTimeout: 1, forceCloseConnections: true })
138
+ const statesReports = []
139
+ const reportWaiters = []
140
+
141
+ function notifyReportWaiters () {
142
+ for (let i = reportWaiters.length - 1; i >= 0; i--) {
143
+ const { count, resolve } = reportWaiters[i]
144
+ if (statesReports.length >= count) {
145
+ reportWaiters.splice(i, 1)
146
+ resolve()
147
+ }
148
+ }
149
+ }
150
+
151
+ let activeStatesRequests = 0
152
+ let maxConcurrentStatesRequests = 0
153
+
154
+ if (!opts.withoutStatesRoute) {
155
+ scaler.post('/scaler/flamegraphs/states', async (req) => {
156
+ activeStatesRequests++
157
+ maxConcurrentStatesRequests = Math.max(maxConcurrentStatesRequests, activeStatesRequests)
158
+ if (opts.statesDelay) {
159
+ await sleep(opts.statesDelay)
160
+ }
161
+ activeStatesRequests--
162
+ statesReports.push(req.body)
163
+ notifyReportWaiters()
164
+ return {}
165
+ })
166
+ }
167
+
168
+ await scaler.listen({ port: 0, host: '127.0.0.1' })
169
+ t.after(() => scaler.close())
170
+
171
+ // Resolves when `count` state reports have been received
172
+ function waitForReports (count = 1) {
173
+ if (statesReports.length >= count) {
174
+ return Promise.resolve()
175
+ }
176
+ const { promise, resolve } = Promise.withResolvers()
177
+ reportWaiters.push({ count, resolve })
178
+ return promise
179
+ }
180
+
181
+ const url = `http://127.0.0.1:${scaler.server.address().port}/scaler`
182
+ return {
183
+ scaler,
184
+ statesReports,
185
+ url,
186
+ waitForReports,
187
+ getMaxConcurrentStatesRequests: () => maxConcurrentStatesRequests
188
+ }
189
+ }
190
+
191
+ // The worker-started listener logs this message synchronously before arming a
192
+ // worker, so its absence right after a synchronous emit proves the listener
193
+ // took the deactivated branch, without waiting on wall-clock time.
194
+ const ARMING_LOG = 'Starting profiling on new worker'
195
+
196
+ function recordInfoLogs (app) {
197
+ const messages = []
198
+ app.log.info = (...args) => {
199
+ messages.push(args[args.length - 1])
200
+ }
201
+ return messages
202
+ }
203
+
204
+ test('setProfilingEnabled(false) stops profiling for both types on every worker', async (t) => {
205
+ setUpEnvironment()
206
+
207
+ const { statesReports, url, waitForReports } = await startScalerMock(t)
208
+ const app = createMockApp({ scalerUrl: url })
209
+
210
+ await flamegraphsPlugin(app)
211
+
212
+ const result = await app.setProfilingEnabled(false)
213
+ equal(result.success, true)
214
+ equal(result.enabled, false)
215
+
216
+ const stopCalls = app.commandCalls.filter((c) => c.command === 'stopProfiling')
217
+ equal(stopCalls.length, 4)
218
+ for (const workerId of ['service-1:0', 'service-2:0']) {
219
+ for (const type of ['cpu', 'heap']) {
220
+ ok(stopCalls.find((c) => c.workerId === workerId && c.options.type === type))
221
+ }
222
+ }
223
+
224
+ await waitForReports(1)
225
+ equal(statesReports.length, 1)
226
+ })
227
+
228
+ test('setProfilingEnabled(true) after a stop restarts profiling with the boot options', async (t) => {
229
+ setUpEnvironment()
230
+
231
+ const { url } = await startScalerMock(t)
232
+ const app = createMockApp({ scalerUrl: url })
233
+
234
+ await flamegraphsPlugin(app)
235
+
236
+ await app.setProfilingEnabled(false)
237
+ app.commandCalls.length = 0
238
+
239
+ const result = await app.setProfilingEnabled(true)
240
+ equal(result.success, true)
241
+ deepEqual(result.types.sort(), ['cpu', 'heap'])
242
+
243
+ const startCalls = app.commandCalls.filter((c) => c.command === 'startProfiling')
244
+ equal(startCalls.length, 4)
245
+ for (const call of startCalls) {
246
+ equal(call.options.durationMillis, 1000)
247
+ equal(call.options.eluThreshold, 0)
248
+ equal(call.options.sourceMaps, false)
249
+ }
250
+ })
251
+
252
+ test('already-started and not-started profiler errors are treated as success', async (t) => {
253
+ setUpEnvironment()
254
+
255
+ const { url } = await startScalerMock(t)
256
+ const app = createMockApp({
257
+ scalerUrl: url,
258
+ sendCommandToApplication: async (workerId, command) => {
259
+ if (command === 'startProfiling') {
260
+ const err = new Error('Profiling already started')
261
+ err.code = 'PLT_PPROF_PROFILING_ALREADY_STARTED'
262
+ throw err
263
+ }
264
+ if (command === 'stopProfiling') {
265
+ const err = new Error('Profiling not started')
266
+ err.code = 'PLT_PPROF_PROFILING_NOT_STARTED'
267
+ throw err
268
+ }
269
+ return {}
270
+ }
271
+ })
272
+
273
+ await flamegraphsPlugin(app)
274
+
275
+ const stopResult = await app.setProfilingEnabled(false)
276
+ equal(stopResult.success, true)
277
+
278
+ const startResult = await app.setProfilingEnabled(true)
279
+ equal(startResult.success, true)
280
+ })
281
+
282
+ test('a worker started while profiling is deactivated is not armed', async (t) => {
283
+ setUpEnvironment()
284
+
285
+ const { url } = await startScalerMock(t)
286
+ const app = createMockApp({ scalerUrl: url })
287
+
288
+ await flamegraphsPlugin(app)
289
+ await app.setupFlamegraphs()
290
+ t.after(() => app.cleanupFlamegraphs())
291
+
292
+ await app.setProfilingEnabled(false)
293
+ app.commandCalls.length = 0
294
+ const infoLogs = recordInfoLogs(app)
295
+
296
+ // The mock runtime dispatches listeners synchronously and the arming log
297
+ // happens before any await, so no waiting is needed for the negative check
298
+ app.watt.runtime.emit('application:worker:started', {
299
+ application: 'service-3',
300
+ worker: 0
301
+ })
302
+
303
+ ok(!infoLogs.includes(ARMING_LOG))
304
+ const startCalls = app.commandCalls.filter((c) => c.command === 'startProfiling')
305
+ equal(startCalls.length, 0)
306
+ })
307
+
308
+ test('a worker started while profiling is activated is armed', async (t) => {
309
+ setUpEnvironment()
310
+
311
+ const { url } = await startScalerMock(t)
312
+ const app = createMockApp({ scalerUrl: url })
313
+
314
+ await flamegraphsPlugin(app)
315
+ await app.setupFlamegraphs()
316
+ t.after(() => app.cleanupFlamegraphs())
317
+
318
+ await app.setProfilingEnabled(false)
319
+ await app.setProfilingEnabled(true)
320
+ app.commandCalls.length = 0
321
+
322
+ app.watt.runtime.emit('application:worker:started', {
323
+ application: 'service-3',
324
+ worker: 0
325
+ })
326
+ await app.waitForCommands('startProfiling', 2)
327
+
328
+ const startCalls = app.commandCalls.filter((c) => c.command === 'startProfiling')
329
+ equal(startCalls.length, 2)
330
+ ok(startCalls.every((c) => c.workerId === 'service-3:0'))
331
+ })
332
+
333
+ test('a re-setup after an ICC recovery does not re-enable a deactivated pod', async (t) => {
334
+ setUpEnvironment()
335
+
336
+ const { url } = await startScalerMock(t)
337
+ const app = createMockApp({ scalerUrl: url })
338
+
339
+ await flamegraphsPlugin(app)
340
+ await app.setupFlamegraphs()
341
+ t.after(() => app.cleanupFlamegraphs())
342
+
343
+ await app.setProfilingEnabled(false)
344
+ app.commandCalls.length = 0
345
+
346
+ // setupFlamegraphs awaits its start commands internally, so the check
347
+ // right after it is deterministic
348
+ await app.setupFlamegraphs()
349
+
350
+ const startCalls = app.commandCalls.filter((c) => c.command === 'startProfiling')
351
+ equal(startCalls.length, 0)
352
+ })
353
+
354
+ test('a pod started with PLT_DISABLE_FLAMEGRAPHS boots deactivated but reports its state', async (t) => {
355
+ setUpEnvironment()
356
+
357
+ const { statesReports, url, waitForReports } = await startScalerMock(t)
358
+ const app = createMockApp({
359
+ scalerUrl: url,
360
+ env: { PLT_DISABLE_FLAMEGRAPHS: true }
361
+ })
362
+
363
+ await flamegraphsPlugin(app)
364
+ await app.setupFlamegraphs()
365
+ t.after(() => app.cleanupFlamegraphs())
366
+ await waitForReports(1)
367
+
368
+ const startCalls = app.commandCalls.filter((c) => c.command === 'startProfiling')
369
+ equal(startCalls.length, 0)
370
+
371
+ equal(statesReports.length, 1)
372
+ ok(statesReports[0].states.every((s) => s.enabled === false))
373
+ })
374
+
375
+ test('a pod started with PLT_DISABLE_FLAMEGRAPHS can be activated at runtime', async (t) => {
376
+ setUpEnvironment()
377
+
378
+ const { url } = await startScalerMock(t)
379
+ const app = createMockApp({
380
+ scalerUrl: url,
381
+ env: { PLT_DISABLE_FLAMEGRAPHS: true }
382
+ })
383
+
384
+ await flamegraphsPlugin(app)
385
+ await app.setupFlamegraphs()
386
+ t.after(() => app.cleanupFlamegraphs())
387
+
388
+ const result = await app.setProfilingEnabled(true)
389
+ equal(result.success, true)
390
+ deepEqual(result.types.sort(), ['cpu', 'heap'])
391
+
392
+ let startCalls = app.commandCalls.filter((c) => c.command === 'startProfiling')
393
+ equal(startCalls.length, 4)
394
+
395
+ // A worker started after the activation is armed too
396
+ app.commandCalls.length = 0
397
+ app.watt.runtime.emit('application:worker:started', {
398
+ application: 'service-3',
399
+ worker: 0
400
+ })
401
+ await app.waitForCommands('startProfiling', 2)
402
+
403
+ startCalls = app.commandCalls.filter((c) => c.command === 'startProfiling')
404
+ equal(startCalls.length, 2)
405
+
406
+ // And it can be deactivated again
407
+ app.commandCalls.length = 0
408
+ await app.setProfilingEnabled(false)
409
+ const stopCalls = app.commandCalls.filter((c) => c.command === 'stopProfiling')
410
+ equal(stopCalls.length, 4)
411
+ })
412
+
413
+ test('sendFlamegraphs works on a runtime-activated pod started with PLT_DISABLE_FLAMEGRAPHS', async (t) => {
414
+ setUpEnvironment()
415
+
416
+ const { url } = await startScalerMock(t)
417
+ const app = createMockApp({
418
+ scalerUrl: url,
419
+ env: { PLT_DISABLE_FLAMEGRAPHS: true }
420
+ })
421
+ let profileRequests = 0
422
+ app.watt.runtime.getApplicationLastProfile = async () => {
423
+ profileRequests++
424
+ return { profile: new Uint8Array([1]), timestamp: Date.now(), preserved: false }
425
+ }
426
+
427
+ await flamegraphsPlugin(app)
428
+
429
+ await app.sendFlamegraphs({ profileType: 'cpu' })
430
+ equal(profileRequests, 0)
431
+
432
+ await app.setProfilingEnabled(true)
433
+ await app.sendFlamegraphs({ profileType: 'cpu' })
434
+ equal(profileRequests, 2)
435
+ })
436
+
437
+ test('sendFlamegraphs on a deactivated pod returns early without collecting', async (t) => {
438
+ setUpEnvironment()
439
+
440
+ const { url } = await startScalerMock(t)
441
+ const app = createMockApp({ scalerUrl: url })
442
+ let profileRequests = 0
443
+ app.watt.runtime.getApplicationLastProfile = async () => {
444
+ profileRequests++
445
+ return { profile: new Uint8Array([1]), timestamp: Date.now(), preserved: false }
446
+ }
447
+
448
+ await flamegraphsPlugin(app)
449
+ await app.setProfilingEnabled(false)
450
+
451
+ await app.sendFlamegraphs({ profileType: 'cpu' })
452
+ equal(profileRequests, 0)
453
+ })
454
+
455
+ test('reportProfilingStates posts the desired and real state of every worker', async (t) => {
456
+ setUpEnvironment()
457
+
458
+ const { statesReports, url } = await startScalerMock(t)
459
+ const app = createMockApp({ scalerUrl: url })
460
+
461
+ await flamegraphsPlugin(app)
462
+ await app.reportProfilingStates()
463
+
464
+ equal(statesReports.length, 1)
465
+ const report = statesReports[0]
466
+ equal(report.applicationId, 'test-application-id')
467
+ equal(report.podId, 'test-pod-123')
468
+ equal(report.expiresIn, 90000)
469
+ equal(report.states.length, 4)
470
+
471
+ const state = report.states.find(
472
+ (s) => s.workerId === 'service-1:0' && s.type === 'cpu'
473
+ )
474
+ equal(state.serviceId, 'service-1')
475
+ equal(state.enabled, true)
476
+ equal(state.isCapturing, true)
477
+ equal(state.isPaused, true)
478
+ })
479
+
480
+ test('reportProfilingStates disables itself when the scaler does not support it', async (t) => {
481
+ setUpEnvironment()
482
+
483
+ const { statesReports, url } = await startScalerMock(t, { withoutStatesRoute: true })
484
+ const app = createMockApp({ scalerUrl: url })
485
+
486
+ let warnings = 0
487
+ app.log.warn = () => { warnings++ }
488
+
489
+ await flamegraphsPlugin(app)
490
+ await app.reportProfilingStates()
491
+ await app.reportProfilingStates()
492
+
493
+ equal(statesReports.length, 0)
494
+ equal(warnings, 1)
495
+ })
496
+
497
+ test('a worker without the profiling state command is reported as unsupported', async (t) => {
498
+ setUpEnvironment()
499
+
500
+ const { statesReports, url } = await startScalerMock(t)
501
+ const app = createMockApp({
502
+ scalerUrl: url,
503
+ sendCommandToApplication: async (workerId, command) => {
504
+ if (command === 'getProfilingState') {
505
+ return {}
506
+ }
507
+ return { success: true }
508
+ }
509
+ })
510
+
511
+ await flamegraphsPlugin(app)
512
+ await app.reportProfilingStates()
513
+
514
+ equal(statesReports.length, 1)
515
+ const report = statesReports[0]
516
+ equal(report.states.length, 4)
517
+ for (const state of report.states) {
518
+ equal(state.unsupported, true)
519
+ equal(state.enabled, true)
520
+ }
521
+ })
522
+
523
+ test('handles start-profiling and stop-profiling commands from ICC', async (t) => {
524
+ setUpEnvironment()
525
+
526
+ const receivedMessages = []
527
+ const wsPort = port + 1
528
+ const wss = new WebSocketServer({ port: wsPort })
529
+ t.after(async () => wss.close())
530
+
531
+ let ws = null
532
+ const waitForClientSubscription = once(wss, 'connection').then(([socket]) => {
533
+ ws = socket
534
+ return new Promise((resolve) => {
535
+ socket.on('message', (data) => {
536
+ const message = JSON.parse(data.toString())
537
+ receivedMessages.push(message)
538
+ if (message.command === 'subscribe' && message.topic === '/config') {
539
+ socket.send(JSON.stringify({ command: 'ack' }))
540
+ resolve()
541
+ }
542
+ })
543
+ })
544
+ })
545
+
546
+ const { url } = await startScalerMock(t)
547
+ const app = createMockApp({ scalerUrl: url, iccPort: wsPort })
548
+
549
+ await updatePlugin(app)
550
+ await flamegraphsPlugin(app)
551
+ await app.connectToUpdates()
552
+ t.after(async () => {
553
+ await app.cleanupFlamegraphs()
554
+ await app.closeUpdates()
555
+ })
556
+
557
+ await waitForClientSubscription
558
+
559
+ ws.send(JSON.stringify({ command: 'stop-profiling', params: {} }))
560
+ await app.waitForCommands('stopProfiling', 4)
561
+
562
+ const stopCalls = app.commandCalls.filter((c) => c.command === 'stopProfiling')
563
+ equal(stopCalls.length, 4)
564
+
565
+ app.commandCalls.length = 0
566
+ ws.send(JSON.stringify({ command: 'start-profiling', params: { types: ['cpu'] } }))
567
+ await app.waitForCommands('startProfiling', 2)
568
+
569
+ const startCalls = app.commandCalls.filter((c) => c.command === 'startProfiling')
570
+ equal(startCalls.length, 2)
571
+ ok(startCalls.every((c) => c.options.type === 'cpu'))
572
+
573
+ app.commandCalls.length = 0
574
+ ws.send(JSON.stringify({ command: 'stop-profiling', params: { types: ['cpu'] } }))
575
+ await app.waitForCommands('stopProfiling', 2)
576
+
577
+ const cpuStopCalls = app.commandCalls.filter((c) => c.command === 'stopProfiling')
578
+ equal(cpuStopCalls.length, 2)
579
+ ok(cpuStopCalls.every((c) => c.options.type === 'cpu'))
580
+ })
581
+
582
+ test('a partial worker failure is not reported as success and the report carries the real state', async (t) => {
583
+ setUpEnvironment()
584
+
585
+ const { statesReports, url, waitForReports } = await startScalerMock(t)
586
+ const failingWorker = 'service-2:0'
587
+ const capturing = { 'service-1:0': false, 'service-2:0': false }
588
+ const app = createMockApp({
589
+ scalerUrl: url,
590
+ sendCommandToApplication: async (workerId, command, options) => {
591
+ if (command === 'startProfiling') {
592
+ if (workerId === failingWorker) {
593
+ throw new Error('boom')
594
+ }
595
+ capturing[workerId] = true
596
+ return {}
597
+ }
598
+ if (command === 'stopProfiling') {
599
+ capturing[workerId] = false
600
+ return {}
601
+ }
602
+ if (command === 'getProfilingState') {
603
+ return {
604
+ isCapturing: capturing[workerId],
605
+ isProfilerRunning: capturing[workerId],
606
+ isPaused: false
607
+ }
608
+ }
609
+ return {}
610
+ }
611
+ })
612
+
613
+ await flamegraphsPlugin(app)
614
+
615
+ await app.setProfilingEnabled(false)
616
+ const result = await app.setProfilingEnabled(true)
617
+
618
+ // The toggle is not a success and the intent is still recorded
619
+ equal(result.success, false)
620
+ equal(result.failures, 1)
621
+ deepEqual(result.types.sort(), ['cpu', 'heap'])
622
+
623
+ // The report distinguishes intent (enabled) from reality (isCapturing)
624
+ await waitForReports(2)
625
+ const report = statesReports[statesReports.length - 1]
626
+ const failedState = report.states.find((s) => s.workerId === failingWorker && s.type === 'cpu')
627
+ equal(failedState.enabled, true)
628
+ equal(failedState.isCapturing, false)
629
+ const okState = report.states.find((s) => s.workerId === 'service-1:0' && s.type === 'cpu')
630
+ equal(okState.enabled, true)
631
+ equal(okState.isCapturing, true)
632
+ })
633
+
634
+ test('a blocked worker does not stall the state report', async (t) => {
635
+ setUpEnvironment()
636
+
637
+ const { statesReports, url, waitForReports } = await startScalerMock(t)
638
+ const app = createMockApp({
639
+ scalerUrl: url,
640
+ env: { PLT_FLAMEGRAPHS_STATE_QUERY_TIMEOUT: 100 },
641
+ sendCommandToApplication: async (workerId, command) => {
642
+ if (command === 'getProfilingState' && workerId === 'service-1:0') {
643
+ // A blocked worker event loop: the query never settles
644
+ return new Promise(() => {})
645
+ }
646
+ if (command === 'getProfilingState') {
647
+ return {
648
+ isCapturing: true,
649
+ isProfilerRunning: true,
650
+ isPaused: false
651
+ }
652
+ }
653
+ return {}
654
+ }
655
+ })
656
+
657
+ await flamegraphsPlugin(app)
658
+ await app.reportProfilingStates()
659
+ await waitForReports(1)
660
+
661
+ // The blocked worker is reported as unresponsive, the healthy one normally
662
+ const report = statesReports[0]
663
+ equal(report.states.length, 4)
664
+ const blockedStates = report.states.filter((s) => s.workerId === 'service-1:0')
665
+ equal(blockedStates.length, 2)
666
+ ok(blockedStates.every((s) => s.unresponsive === true))
667
+ const healthyStates = report.states.filter((s) => s.workerId === 'service-2:0')
668
+ equal(healthyStates.length, 2)
669
+ ok(healthyStates.every((s) => s.isCapturing === true && s.unresponsive === undefined))
670
+ })
671
+
672
+ test('overlapping state reports are coalesced', async (t) => {
673
+ setUpEnvironment()
674
+
675
+ const mock = await startScalerMock(t, { statesDelay: 100 })
676
+ const app = createMockApp({ scalerUrl: mock.url })
677
+
678
+ await flamegraphsPlugin(app)
679
+
680
+ // The second call while the first is in flight queues a single re-run
681
+ const first = app.reportProfilingStates()
682
+ const second = app.reportProfilingStates()
683
+ equal(first, second)
684
+
685
+ await first
686
+ await mock.waitForReports(2)
687
+
688
+ equal(mock.statesReports.length, 2)
689
+ equal(mock.getMaxConcurrentStatesRequests(), 1)
690
+ })
691
+
692
+ test('a timed-out worker is not queried again until it restarts', async (t) => {
693
+ setUpEnvironment()
694
+
695
+ const { statesReports, url, waitForReports } = await startScalerMock(t)
696
+ const app = createMockApp({
697
+ scalerUrl: url,
698
+ env: { PLT_FLAMEGRAPHS_STATE_QUERY_TIMEOUT: 100 },
699
+ sendCommandToApplication: async (workerId, command) => {
700
+ if (command === 'getProfilingState' && workerId === 'service-1:0') {
701
+ // A blocked worker event loop: the query never settles
702
+ return new Promise(() => {})
703
+ }
704
+ if (command === 'getProfilingState') {
705
+ return {
706
+ isCapturing: true,
707
+ isProfilerRunning: true,
708
+ isPaused: false
709
+ }
710
+ }
711
+ return {}
712
+ }
713
+ })
714
+
715
+ const queriesToBlockedWorker = () => app.commandCalls.filter(
716
+ (c) => c.command === 'getProfilingState' && c.workerId === 'service-1:0'
717
+ ).length
718
+
719
+ await flamegraphsPlugin(app)
720
+ await app.setupFlamegraphs()
721
+ t.after(() => app.cleanupFlamegraphs())
722
+
723
+ // The first report queries the blocked worker, times out, and suppresses it
724
+ await waitForReports(1)
725
+ equal(queriesToBlockedWorker(), 2)
726
+
727
+ // The next report does not query the suppressed worker but still reports
728
+ // it as unresponsive
729
+ await app.reportProfilingStates()
730
+ await waitForReports(2)
731
+ equal(queriesToBlockedWorker(), 2)
732
+ const suppressedReport = statesReports[statesReports.length - 1]
733
+ equal(suppressedReport.states.length, 4)
734
+ ok(suppressedReport.states
735
+ .filter((s) => s.workerId === 'service-1:0')
736
+ .every((s) => s.unresponsive === true))
737
+
738
+ // A worker restart clears the suppression and it is queried again
739
+ app.watt.runtime.emit('application:worker:started', {
740
+ application: 'service-1',
741
+ worker: 0
742
+ })
743
+ await app.reportProfilingStates()
744
+ await waitForReports(3)
745
+ equal(queriesToBlockedWorker(), 4)
746
+ })