@platformatic/watt-extra 1.6.3-alpha.5 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/settings.local.json +8 -5
- package/package.json +1 -1
- package/plugins/alerts.js +25 -1
- package/plugins/env.js +2 -1
- package/plugins/flamegraphs.js +210 -244
- package/plugins/health-signals.js +3 -5
- package/plugins/update.js +2 -2
- package/test/alerts.test.js +179 -7
- package/test/health-signals.test.js +5 -2
- package/test/helper.js +1 -0
- package/test/trigger-flamegraphs.test.js +439 -187
- package/test/profiler.test.js +0 -443
|
@@ -79,16 +79,13 @@ function createMockApp (port, includeScalerUrl = true, env = {}) {
|
|
|
79
79
|
}
|
|
80
80
|
}
|
|
81
81
|
|
|
82
|
-
const createLogger = () => ({
|
|
83
|
-
info: () => {},
|
|
84
|
-
error: () => {},
|
|
85
|
-
warn: () => {},
|
|
86
|
-
debug: () => {},
|
|
87
|
-
child: () => createLogger()
|
|
88
|
-
})
|
|
89
|
-
|
|
90
82
|
const app = {
|
|
91
|
-
log:
|
|
83
|
+
log: {
|
|
84
|
+
info: () => {},
|
|
85
|
+
error: () => {},
|
|
86
|
+
warn: () => {},
|
|
87
|
+
debug: () => {}
|
|
88
|
+
},
|
|
92
89
|
instanceConfig: {
|
|
93
90
|
applicationId: 'test-application-id'
|
|
94
91
|
},
|
|
@@ -129,19 +126,23 @@ test('setupFlamegraphs should pass sourceMaps from application config to startPr
|
|
|
129
126
|
const app = createMockApp(port)
|
|
130
127
|
const startProfilingCalls = []
|
|
131
128
|
|
|
132
|
-
// Mock getApplicationDetails to return config with sourceMaps
|
|
133
|
-
app.watt.runtime.getApplicationDetails = async (
|
|
134
|
-
if (
|
|
135
|
-
return { id:
|
|
136
|
-
} else if (
|
|
137
|
-
return { id:
|
|
129
|
+
// Mock getApplicationDetails to return config with sourceMaps for worker IDs
|
|
130
|
+
app.watt.runtime.getApplicationDetails = async (workerFullId) => {
|
|
131
|
+
if (workerFullId.startsWith('service-1')) {
|
|
132
|
+
return { id: workerFullId, sourceMaps: true }
|
|
133
|
+
} else if (workerFullId.startsWith('service-2')) {
|
|
134
|
+
return { id: workerFullId, sourceMaps: false }
|
|
138
135
|
}
|
|
139
|
-
return { id:
|
|
136
|
+
return { id: workerFullId, sourceMaps: false }
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
app.watt.runtime.getRuntimeConfig = async () => {
|
|
140
|
+
return {}
|
|
140
141
|
}
|
|
141
142
|
|
|
142
|
-
app.watt.runtime.sendCommandToApplication = async (
|
|
143
|
+
app.watt.runtime.sendCommandToApplication = async (workerFullId, command, options) => {
|
|
143
144
|
if (command === 'startProfiling') {
|
|
144
|
-
startProfilingCalls.push({
|
|
145
|
+
startProfilingCalls.push({ workerFullId, command, options })
|
|
145
146
|
return { success: true }
|
|
146
147
|
}
|
|
147
148
|
return { success: false }
|
|
@@ -150,24 +151,13 @@ test('setupFlamegraphs should pass sourceMaps from application config to startPr
|
|
|
150
151
|
await flamegraphsPlugin(app)
|
|
151
152
|
await app.setupFlamegraphs()
|
|
152
153
|
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
})
|
|
156
|
-
|
|
157
|
-
// Trigger profiling for both CPU and heap profiles
|
|
158
|
-
await app.requestFlamegraphs({ profileType: 'cpu' })
|
|
159
|
-
await app.requestFlamegraphs({ profileType: 'heap' })
|
|
160
|
-
|
|
161
|
-
// Wait for profiling to start
|
|
162
|
-
await sleep(100)
|
|
163
|
-
|
|
164
|
-
// Should call startProfiling 4 times: 2 services × 2 profile types (cpu + heap)
|
|
165
|
-
equal(startProfilingCalls.length, 4, 'Should call startProfiling for both services with cpu and heap')
|
|
154
|
+
// Should call startProfiling 4 times: 2 workers × 2 profile types (cpu + heap)
|
|
155
|
+
equal(startProfilingCalls.length, 4, 'Should call startProfiling for both workers with cpu and heap')
|
|
166
156
|
|
|
167
|
-
const service1CpuCall = startProfilingCalls.find(c => c.
|
|
168
|
-
const service1HeapCall = startProfilingCalls.find(c => c.
|
|
169
|
-
const service2CpuCall = startProfilingCalls.find(c => c.
|
|
170
|
-
const service2HeapCall = startProfilingCalls.find(c => c.
|
|
157
|
+
const service1CpuCall = startProfilingCalls.find(c => c.workerFullId === 'service-1:0' && c.options.type === 'cpu')
|
|
158
|
+
const service1HeapCall = startProfilingCalls.find(c => c.workerFullId === 'service-1:0' && c.options.type === 'heap')
|
|
159
|
+
const service2CpuCall = startProfilingCalls.find(c => c.workerFullId === 'service-2:0' && c.options.type === 'cpu')
|
|
160
|
+
const service2HeapCall = startProfilingCalls.find(c => c.workerFullId === 'service-2:0' && c.options.type === 'heap')
|
|
171
161
|
|
|
172
162
|
ok(service1CpuCall, 'Should have called startProfiling for service-1 CPU')
|
|
173
163
|
ok(service1HeapCall, 'Should have called startProfiling for service-1 heap')
|
|
@@ -187,13 +177,17 @@ test('setupFlamegraphs should handle missing sourceMaps in application config',
|
|
|
187
177
|
const startProfilingCalls = []
|
|
188
178
|
|
|
189
179
|
// Mock getApplicationDetails to return config without sourceMaps
|
|
190
|
-
app.watt.runtime.getApplicationDetails = async (
|
|
191
|
-
return { id:
|
|
180
|
+
app.watt.runtime.getApplicationDetails = async (workerFullId) => {
|
|
181
|
+
return { id: workerFullId }
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
app.watt.runtime.getRuntimeConfig = async () => {
|
|
185
|
+
return {}
|
|
192
186
|
}
|
|
193
187
|
|
|
194
|
-
app.watt.runtime.sendCommandToApplication = async (
|
|
188
|
+
app.watt.runtime.sendCommandToApplication = async (workerFullId, command, options) => {
|
|
195
189
|
if (command === 'startProfiling') {
|
|
196
|
-
startProfilingCalls.push({
|
|
190
|
+
startProfilingCalls.push({ workerFullId, command, options })
|
|
197
191
|
return { success: true }
|
|
198
192
|
}
|
|
199
193
|
return { success: false }
|
|
@@ -202,18 +196,7 @@ test('setupFlamegraphs should handle missing sourceMaps in application config',
|
|
|
202
196
|
await flamegraphsPlugin(app)
|
|
203
197
|
await app.setupFlamegraphs()
|
|
204
198
|
|
|
205
|
-
|
|
206
|
-
await app.cleanupFlamegraphs()
|
|
207
|
-
})
|
|
208
|
-
|
|
209
|
-
// Trigger profiling for both CPU and heap profiles
|
|
210
|
-
await app.requestFlamegraphs({ profileType: 'cpu' })
|
|
211
|
-
await app.requestFlamegraphs({ profileType: 'heap' })
|
|
212
|
-
|
|
213
|
-
// Wait for profiling to start
|
|
214
|
-
await sleep(100)
|
|
215
|
-
|
|
216
|
-
equal(startProfilingCalls.length, 4, 'Should call startProfiling for both services with cpu and heap')
|
|
199
|
+
equal(startProfilingCalls.length, 4, 'Should call startProfiling for both workers with cpu and heap')
|
|
217
200
|
|
|
218
201
|
for (const call of startProfilingCalls) {
|
|
219
202
|
equal(call.options.sourceMaps, false, 'sourceMaps should be false when not in config')
|
|
@@ -229,9 +212,9 @@ test('setupFlamegraphs should skip profiling when PLT_DISABLE_FLAMEGRAPHS is set
|
|
|
229
212
|
|
|
230
213
|
const startProfilingCalls = []
|
|
231
214
|
|
|
232
|
-
app.watt.runtime.sendCommandToApplication = async (
|
|
215
|
+
app.watt.runtime.sendCommandToApplication = async (serviceId, command, options) => {
|
|
233
216
|
if (command === 'startProfiling') {
|
|
234
|
-
startProfilingCalls.push({
|
|
217
|
+
startProfilingCalls.push({ serviceId, command, options })
|
|
235
218
|
return { success: true }
|
|
236
219
|
}
|
|
237
220
|
return { success: false }
|
|
@@ -240,16 +223,10 @@ test('setupFlamegraphs should skip profiling when PLT_DISABLE_FLAMEGRAPHS is set
|
|
|
240
223
|
await flamegraphsPlugin(app)
|
|
241
224
|
await app.setupFlamegraphs()
|
|
242
225
|
|
|
243
|
-
// Also try to send flamegraphs
|
|
244
|
-
await app.requestFlamegraphs()
|
|
245
|
-
|
|
246
|
-
// Wait to ensure no profiling starts
|
|
247
|
-
await sleep(100)
|
|
248
|
-
|
|
249
226
|
equal(startProfilingCalls.length, 0, 'Should not call startProfiling when disabled')
|
|
250
227
|
})
|
|
251
228
|
|
|
252
|
-
test('
|
|
229
|
+
test('setupFlamegraphs should handle errors when starting profiling', async (t) => {
|
|
253
230
|
setUpEnvironment()
|
|
254
231
|
|
|
255
232
|
const app = createMockApp(port)
|
|
@@ -259,13 +236,17 @@ test('requestFlamegraphs should handle errors when starting profiling', async (t
|
|
|
259
236
|
errors.push(result)
|
|
260
237
|
}
|
|
261
238
|
|
|
262
|
-
app.watt.runtime.getApplicationDetails = async (
|
|
263
|
-
return { id:
|
|
239
|
+
app.watt.runtime.getApplicationDetails = async (workerFullId) => {
|
|
240
|
+
return { id: workerFullId, sourceMaps: true }
|
|
264
241
|
}
|
|
265
242
|
|
|
266
|
-
app.watt.runtime.
|
|
243
|
+
app.watt.runtime.getRuntimeConfig = async () => {
|
|
244
|
+
return {}
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
app.watt.runtime.sendCommandToApplication = async (workerFullId, command, options) => {
|
|
267
248
|
if (command === 'startProfiling') {
|
|
268
|
-
throw new Error(`Failed to start profiling for ${
|
|
249
|
+
throw new Error(`Failed to start profiling for ${workerFullId}`)
|
|
269
250
|
}
|
|
270
251
|
return { success: false }
|
|
271
252
|
}
|
|
@@ -273,21 +254,11 @@ test('requestFlamegraphs should handle errors when starting profiling', async (t
|
|
|
273
254
|
await flamegraphsPlugin(app)
|
|
274
255
|
await app.setupFlamegraphs()
|
|
275
256
|
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
})
|
|
279
|
-
|
|
280
|
-
// Trigger profiling which will fail
|
|
281
|
-
await app.requestFlamegraphs()
|
|
282
|
-
|
|
283
|
-
// Wait for errors to be logged
|
|
284
|
-
await sleep(100)
|
|
285
|
-
|
|
286
|
-
// Should log 2 errors (1 per service in Profiler)
|
|
287
|
-
equal(errors.length, 2, 'Should log errors for both failed services')
|
|
257
|
+
// Should log 4 errors (2 workers, each logged twice: once in startProfilingOnWorker, once in setupFlamegraphs)
|
|
258
|
+
equal(errors.length, 4, 'Should log errors for both failed workers')
|
|
288
259
|
})
|
|
289
260
|
|
|
290
|
-
test('
|
|
261
|
+
test('sendFlamegraphs should upload flamegraphs from all services', async (t) => {
|
|
291
262
|
setUpEnvironment()
|
|
292
263
|
|
|
293
264
|
const uploadedFlamegraphs = []
|
|
@@ -297,13 +268,7 @@ test('requestFlamegraphs should upload flamegraphs from all services', async (t)
|
|
|
297
268
|
|
|
298
269
|
const mockProfile = new Uint8Array([1, 2, 3, 4, 5])
|
|
299
270
|
|
|
300
|
-
app.watt.runtime.sendCommandToApplication = async (
|
|
301
|
-
if (command === 'startProfiling') {
|
|
302
|
-
return { success: true }
|
|
303
|
-
}
|
|
304
|
-
if (command === 'getProfilingState') {
|
|
305
|
-
return { latestProfileTimestamp: Date.now() }
|
|
306
|
-
}
|
|
271
|
+
app.watt.runtime.sendCommandToApplication = async (serviceId, command) => {
|
|
307
272
|
if (command === 'getLastProfile') {
|
|
308
273
|
return mockProfile
|
|
309
274
|
}
|
|
@@ -323,7 +288,7 @@ test('requestFlamegraphs should upload flamegraphs from all services', async (t)
|
|
|
323
288
|
body: buffer
|
|
324
289
|
})
|
|
325
290
|
res.writeHead(200)
|
|
326
|
-
res.end(
|
|
291
|
+
res.end()
|
|
327
292
|
})
|
|
328
293
|
})
|
|
329
294
|
|
|
@@ -331,17 +296,7 @@ test('requestFlamegraphs should upload flamegraphs from all services', async (t)
|
|
|
331
296
|
t.after(() => server.close())
|
|
332
297
|
|
|
333
298
|
await flamegraphsPlugin(app)
|
|
334
|
-
await app.
|
|
335
|
-
|
|
336
|
-
t.after(async () => {
|
|
337
|
-
await app.cleanupFlamegraphs()
|
|
338
|
-
})
|
|
339
|
-
|
|
340
|
-
// Trigger profiling
|
|
341
|
-
await app.requestFlamegraphs()
|
|
342
|
-
|
|
343
|
-
// Wait for profile to be generated (duration is 1 second)
|
|
344
|
-
await sleep(1500)
|
|
299
|
+
await app.sendFlamegraphs()
|
|
345
300
|
|
|
346
301
|
equal(uploadedFlamegraphs.length, 2, 'Should upload flamegraphs for both services')
|
|
347
302
|
|
|
@@ -356,7 +311,7 @@ test('requestFlamegraphs should upload flamegraphs from all services', async (t)
|
|
|
356
311
|
deepEqual(service1Upload.body, Buffer.from(mockProfile))
|
|
357
312
|
})
|
|
358
313
|
|
|
359
|
-
test('
|
|
314
|
+
test('sendFlamegraphs should handle missing profile data', async (t) => {
|
|
360
315
|
setUpEnvironment()
|
|
361
316
|
|
|
362
317
|
const app = createMockApp(port + 11)
|
|
@@ -366,13 +321,7 @@ test('requestFlamegraphs should handle missing profile data', async (t) => {
|
|
|
366
321
|
errors.push(obj)
|
|
367
322
|
}
|
|
368
323
|
|
|
369
|
-
app.watt.runtime.sendCommandToApplication = async (
|
|
370
|
-
if (command === 'startProfiling') {
|
|
371
|
-
return { success: true }
|
|
372
|
-
}
|
|
373
|
-
if (command === 'getProfilingState') {
|
|
374
|
-
return { latestProfileTimestamp: Date.now() }
|
|
375
|
-
}
|
|
324
|
+
app.watt.runtime.sendCommandToApplication = async (serviceId, command) => {
|
|
376
325
|
if (command === 'getLastProfile') {
|
|
377
326
|
// Return invalid data (not Uint8Array)
|
|
378
327
|
return null
|
|
@@ -381,34 +330,18 @@ test('requestFlamegraphs should handle missing profile data', async (t) => {
|
|
|
381
330
|
}
|
|
382
331
|
|
|
383
332
|
await flamegraphsPlugin(app)
|
|
384
|
-
await app.
|
|
385
|
-
|
|
386
|
-
t.after(async () => {
|
|
387
|
-
await app.cleanupFlamegraphs()
|
|
388
|
-
})
|
|
389
|
-
|
|
390
|
-
// Trigger profiling
|
|
391
|
-
await app.requestFlamegraphs()
|
|
392
|
-
|
|
393
|
-
// Wait for profile to be generated (duration is 1 second)
|
|
394
|
-
await sleep(1500)
|
|
333
|
+
await app.sendFlamegraphs()
|
|
395
334
|
|
|
396
335
|
equal(errors.length, 2, 'Should log errors for both services with missing profiles')
|
|
397
336
|
})
|
|
398
337
|
|
|
399
|
-
test('
|
|
338
|
+
test('sendFlamegraphs should filter by workerIds when provided', async (t) => {
|
|
400
339
|
setUpEnvironment()
|
|
401
340
|
|
|
402
341
|
const app = createMockApp(port + 12)
|
|
403
342
|
const getProfileCalls = []
|
|
404
343
|
|
|
405
344
|
app.watt.runtime.sendCommandToApplication = async (workerId, command) => {
|
|
406
|
-
if (command === 'startProfiling') {
|
|
407
|
-
return { success: true }
|
|
408
|
-
}
|
|
409
|
-
if (command === 'getProfilingState') {
|
|
410
|
-
return { latestProfileTimestamp: Date.now() }
|
|
411
|
-
}
|
|
412
345
|
if (command === 'getLastProfile') {
|
|
413
346
|
getProfileCalls.push(workerId)
|
|
414
347
|
return new Uint8Array([1, 2, 3])
|
|
@@ -423,7 +356,7 @@ test('requestFlamegraphs should filter by workerIds when provided', async (t) =>
|
|
|
423
356
|
req.on('data', chunk => body.push(chunk))
|
|
424
357
|
req.on('end', () => {
|
|
425
358
|
res.writeHead(200)
|
|
426
|
-
res.end(
|
|
359
|
+
res.end()
|
|
427
360
|
})
|
|
428
361
|
})
|
|
429
362
|
|
|
@@ -431,23 +364,52 @@ test('requestFlamegraphs should filter by workerIds when provided', async (t) =>
|
|
|
431
364
|
t.after(() => server.close())
|
|
432
365
|
|
|
433
366
|
await flamegraphsPlugin(app)
|
|
434
|
-
await app.
|
|
367
|
+
await app.sendFlamegraphs({ workerIds: ['service-1:0'] })
|
|
435
368
|
|
|
436
|
-
|
|
437
|
-
|
|
369
|
+
equal(getProfileCalls.length, 1, 'Should only request profile for specified service')
|
|
370
|
+
equal(getProfileCalls[0], 'service-1:0', 'Should request profile for service-1')
|
|
371
|
+
})
|
|
372
|
+
|
|
373
|
+
test('sendFlamegraphs should try to get the profile from a service if worker is not available', async (t) => {
|
|
374
|
+
setUpEnvironment()
|
|
375
|
+
|
|
376
|
+
const app = createMockApp(port + 12)
|
|
377
|
+
const getProfileCalls = []
|
|
378
|
+
|
|
379
|
+
app.watt.runtime.sendCommandToApplication = async (workerId, command) => {
|
|
380
|
+
if (command === 'getLastProfile') {
|
|
381
|
+
getProfileCalls.push(workerId)
|
|
382
|
+
if (workerId === 'service-1:2') {
|
|
383
|
+
throw new Error('Worker not available')
|
|
384
|
+
}
|
|
385
|
+
return new Uint8Array([1, 2, 3])
|
|
386
|
+
}
|
|
387
|
+
return { success: false }
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
// Mock HTTP server
|
|
391
|
+
const { createServer } = await import('node:http')
|
|
392
|
+
const server = createServer((req, res) => {
|
|
393
|
+
const body = []
|
|
394
|
+
req.on('data', chunk => body.push(chunk))
|
|
395
|
+
req.on('end', () => {
|
|
396
|
+
res.writeHead(200)
|
|
397
|
+
res.end()
|
|
398
|
+
})
|
|
438
399
|
})
|
|
439
400
|
|
|
440
|
-
|
|
441
|
-
|
|
401
|
+
await new Promise(resolve => server.listen(port + 12, resolve))
|
|
402
|
+
t.after(() => server.close())
|
|
442
403
|
|
|
443
|
-
|
|
444
|
-
await
|
|
404
|
+
await flamegraphsPlugin(app)
|
|
405
|
+
await app.sendFlamegraphs({ workerIds: ['service-1:2'] })
|
|
445
406
|
|
|
446
|
-
equal(getProfileCalls.length,
|
|
447
|
-
equal(getProfileCalls[0], 'service-1:
|
|
407
|
+
equal(getProfileCalls.length, 2)
|
|
408
|
+
equal(getProfileCalls[0], 'service-1:2')
|
|
409
|
+
equal(getProfileCalls[1], 'service-1')
|
|
448
410
|
})
|
|
449
411
|
|
|
450
|
-
test('
|
|
412
|
+
test('sendFlamegraphs should skip when PLT_DISABLE_FLAMEGRAPHS is set', async (t) => {
|
|
451
413
|
setUpEnvironment()
|
|
452
414
|
|
|
453
415
|
const app = createMockApp(port + 13)
|
|
@@ -456,9 +418,6 @@ test('requestFlamegraphs should skip when PLT_DISABLE_FLAMEGRAPHS is set', async
|
|
|
456
418
|
const getProfileCalls = []
|
|
457
419
|
|
|
458
420
|
app.watt.runtime.sendCommandToApplication = async (workerId, command) => {
|
|
459
|
-
if (command === 'startProfiling') {
|
|
460
|
-
return { success: true }
|
|
461
|
-
}
|
|
462
421
|
if (command === 'getLastProfile') {
|
|
463
422
|
getProfileCalls.push(workerId)
|
|
464
423
|
return new Uint8Array([1, 2, 3])
|
|
@@ -467,15 +426,12 @@ test('requestFlamegraphs should skip when PLT_DISABLE_FLAMEGRAPHS is set', async
|
|
|
467
426
|
}
|
|
468
427
|
|
|
469
428
|
await flamegraphsPlugin(app)
|
|
470
|
-
await app.
|
|
471
|
-
|
|
472
|
-
// Wait to ensure no profiling starts
|
|
473
|
-
await sleep(100)
|
|
429
|
+
await app.sendFlamegraphs()
|
|
474
430
|
|
|
475
431
|
equal(getProfileCalls.length, 0, 'Should not request profiles when disabled')
|
|
476
432
|
})
|
|
477
433
|
|
|
478
|
-
test('
|
|
434
|
+
test('sendFlamegraphs should throw error when scaler URL is missing', async (t) => {
|
|
479
435
|
setUpEnvironment()
|
|
480
436
|
|
|
481
437
|
const app = createMockApp(port + 14, false) // Don't include scaler URL
|
|
@@ -484,7 +440,7 @@ test('requestFlamegraphs should throw error when scaler URL is missing', async (
|
|
|
484
440
|
|
|
485
441
|
let errorThrown = false
|
|
486
442
|
try {
|
|
487
|
-
await app.
|
|
443
|
+
await app.sendFlamegraphs()
|
|
488
444
|
} catch (err) {
|
|
489
445
|
errorThrown = true
|
|
490
446
|
ok(err.message.includes('No scaler URL'), 'Should throw error about missing scaler URL')
|
|
@@ -504,7 +460,7 @@ test('should handle trigger-flamegraph command and upload flamegraphs from servi
|
|
|
504
460
|
})
|
|
505
461
|
|
|
506
462
|
const wss = new WebSocketServer({ port: port + 15 })
|
|
507
|
-
t.after(() => wss.close())
|
|
463
|
+
t.after(async () => wss.close())
|
|
508
464
|
|
|
509
465
|
const { waitForClientSubscription, getWs } = setupMockIccServer(
|
|
510
466
|
wss,
|
|
@@ -515,17 +471,14 @@ test('should handle trigger-flamegraph command and upload flamegraphs from servi
|
|
|
515
471
|
const app = createMockApp(port + 15)
|
|
516
472
|
|
|
517
473
|
app.watt.runtime.sendCommandToApplication = async (
|
|
518
|
-
|
|
474
|
+
serviceId,
|
|
519
475
|
command
|
|
520
476
|
) => {
|
|
521
477
|
if (command === 'startProfiling') {
|
|
522
478
|
return { success: true }
|
|
523
479
|
}
|
|
524
|
-
if (command === 'getProfilingState') {
|
|
525
|
-
return { latestProfileTimestamp: Date.now() }
|
|
526
|
-
}
|
|
527
480
|
if (command === 'getLastProfile') {
|
|
528
|
-
getFlamegraphReqs.push({ serviceId
|
|
481
|
+
getFlamegraphReqs.push({ serviceId })
|
|
529
482
|
if (getFlamegraphReqs.length === 2) {
|
|
530
483
|
uploadResolve()
|
|
531
484
|
}
|
|
@@ -542,7 +495,7 @@ test('should handle trigger-flamegraph command and upload flamegraphs from servi
|
|
|
542
495
|
|
|
543
496
|
t.after(async () => {
|
|
544
497
|
if (app.cleanupFlamegraphs) {
|
|
545
|
-
|
|
498
|
+
app.cleanupFlamegraphs()
|
|
546
499
|
}
|
|
547
500
|
await app.closeUpdates()
|
|
548
501
|
})
|
|
@@ -560,14 +513,14 @@ test('should handle trigger-flamegraph command and upload flamegraphs from servi
|
|
|
560
513
|
equal(getFlamegraphReqs.length, 2)
|
|
561
514
|
|
|
562
515
|
const service1Req = getFlamegraphReqs.find(
|
|
563
|
-
(f) => f.serviceId === 'service-1
|
|
516
|
+
(f) => f.serviceId === 'service-1'
|
|
564
517
|
)
|
|
565
518
|
const service2Req = getFlamegraphReqs.find(
|
|
566
|
-
(f) => f.serviceId === 'service-2
|
|
519
|
+
(f) => f.serviceId === 'service-2'
|
|
567
520
|
)
|
|
568
521
|
|
|
569
|
-
equal(service1Req.serviceId, 'service-1
|
|
570
|
-
equal(service2Req.serviceId, 'service-2
|
|
522
|
+
equal(service1Req.serviceId, 'service-1')
|
|
523
|
+
equal(service2Req.serviceId, 'service-2')
|
|
571
524
|
})
|
|
572
525
|
|
|
573
526
|
test('should handle trigger-flamegraph when no runtime is available', async (t) => {
|
|
@@ -576,7 +529,7 @@ test('should handle trigger-flamegraph when no runtime is available', async (t)
|
|
|
576
529
|
const receivedMessages = []
|
|
577
530
|
|
|
578
531
|
const wss = new WebSocketServer({ port: port + 16 })
|
|
579
|
-
t.after(() => wss.close())
|
|
532
|
+
t.after(async () => wss.close())
|
|
580
533
|
|
|
581
534
|
const { waitForClientSubscription, getWs } = setupMockIccServer(
|
|
582
535
|
wss,
|
|
@@ -609,7 +562,7 @@ test('should handle trigger-flamegraph when flamegraph upload fails', async (t)
|
|
|
609
562
|
const receivedMessages = []
|
|
610
563
|
|
|
611
564
|
const wss = new WebSocketServer({ port: port + 17 })
|
|
612
|
-
t.after(() => wss.close())
|
|
565
|
+
t.after(async () => wss.close())
|
|
613
566
|
|
|
614
567
|
const { waitForClientSubscription, getWs } = setupMockIccServer(
|
|
615
568
|
wss,
|
|
@@ -663,8 +616,8 @@ test('should handle trigger-heapprofile command and upload heap profiles from se
|
|
|
663
616
|
uploadResolve = resolve
|
|
664
617
|
})
|
|
665
618
|
|
|
666
|
-
const wss = new WebSocketServer({ port: port +
|
|
667
|
-
t.after(() => wss.close())
|
|
619
|
+
const wss = new WebSocketServer({ port: port + 3 })
|
|
620
|
+
t.after(async () => wss.close())
|
|
668
621
|
|
|
669
622
|
const { waitForClientSubscription, getWs } = setupMockIccServer(
|
|
670
623
|
wss,
|
|
@@ -672,10 +625,10 @@ test('should handle trigger-heapprofile command and upload heap profiles from se
|
|
|
672
625
|
true
|
|
673
626
|
)
|
|
674
627
|
|
|
675
|
-
const app = createMockApp(port +
|
|
628
|
+
const app = createMockApp(port + 3)
|
|
676
629
|
|
|
677
630
|
app.watt.runtime.sendCommandToApplication = async (
|
|
678
|
-
|
|
631
|
+
serviceId,
|
|
679
632
|
command,
|
|
680
633
|
options
|
|
681
634
|
) => {
|
|
@@ -685,11 +638,8 @@ test('should handle trigger-heapprofile command and upload heap profiles from se
|
|
|
685
638
|
if (command === 'startProfiling') {
|
|
686
639
|
return { success: true }
|
|
687
640
|
}
|
|
688
|
-
if (command === 'getProfilingState') {
|
|
689
|
-
return { latestProfileTimestamp: Date.now() }
|
|
690
|
-
}
|
|
691
641
|
if (command === 'getLastProfile') {
|
|
692
|
-
getHeapProfileReqs.push({ serviceId
|
|
642
|
+
getHeapProfileReqs.push({ serviceId })
|
|
693
643
|
if (getHeapProfileReqs.length === 2) {
|
|
694
644
|
uploadResolve()
|
|
695
645
|
}
|
|
@@ -706,7 +656,7 @@ test('should handle trigger-heapprofile command and upload heap profiles from se
|
|
|
706
656
|
|
|
707
657
|
t.after(async () => {
|
|
708
658
|
if (app.cleanupFlamegraphs) {
|
|
709
|
-
|
|
659
|
+
app.cleanupFlamegraphs()
|
|
710
660
|
}
|
|
711
661
|
await app.closeUpdates()
|
|
712
662
|
})
|
|
@@ -724,30 +674,342 @@ test('should handle trigger-heapprofile command and upload heap profiles from se
|
|
|
724
674
|
equal(getHeapProfileReqs.length, 2)
|
|
725
675
|
|
|
726
676
|
const service1Req = getHeapProfileReqs.find(
|
|
727
|
-
(f) => f.serviceId === 'service-1
|
|
677
|
+
(f) => f.serviceId === 'service-1'
|
|
728
678
|
)
|
|
729
679
|
const service2Req = getHeapProfileReqs.find(
|
|
730
|
-
(f) => f.serviceId === 'service-2
|
|
680
|
+
(f) => f.serviceId === 'service-2'
|
|
731
681
|
)
|
|
732
682
|
|
|
733
|
-
equal(service1Req.serviceId, 'service-1
|
|
734
|
-
equal(service2Req.serviceId, 'service-2
|
|
683
|
+
equal(service1Req.serviceId, 'service-1')
|
|
684
|
+
equal(service2Req.serviceId, 'service-2')
|
|
735
685
|
})
|
|
736
686
|
|
|
737
|
-
test('
|
|
687
|
+
test('should handle PLT_PPROF_NO_PROFILE_AVAILABLE error with info log', async (t) => {
|
|
738
688
|
setUpEnvironment()
|
|
739
689
|
|
|
740
|
-
const
|
|
690
|
+
const receivedMessages = []
|
|
691
|
+
const infoLogs = []
|
|
741
692
|
|
|
742
|
-
const
|
|
693
|
+
const wss = new WebSocketServer({ port: port + 4 })
|
|
694
|
+
t.after(async () => wss.close())
|
|
743
695
|
|
|
744
|
-
|
|
696
|
+
const { waitForClientSubscription, getWs } = setupMockIccServer(
|
|
697
|
+
wss,
|
|
698
|
+
receivedMessages,
|
|
699
|
+
true
|
|
700
|
+
)
|
|
701
|
+
|
|
702
|
+
const app = createMockApp(port + 4, true, {
|
|
703
|
+
PLT_FLAMEGRAPHS_INTERVAL_SEC: 10,
|
|
704
|
+
PLT_FLAMEGRAPHS_ATTEMPT_TIMEOUT: 1000
|
|
705
|
+
})
|
|
706
|
+
|
|
707
|
+
const originalInfo = app.log.info
|
|
708
|
+
app.log.info = (...args) => {
|
|
709
|
+
originalInfo(...args)
|
|
710
|
+
infoLogs.push(args)
|
|
711
|
+
}
|
|
712
|
+
|
|
713
|
+
// Profile will be generated in 10s
|
|
714
|
+
const profileGenerationDate = Date.now() + 10000
|
|
715
|
+
const mockProfile = new Uint8Array([1, 2, 3, 4, 5])
|
|
716
|
+
|
|
717
|
+
app.watt.runtime.sendCommandToApplication = async (
|
|
718
|
+
serviceId,
|
|
719
|
+
command
|
|
720
|
+
) => {
|
|
745
721
|
if (command === 'startProfiling') {
|
|
746
722
|
return { success: true }
|
|
747
723
|
}
|
|
748
|
-
if (command === '
|
|
749
|
-
|
|
724
|
+
if (command === 'getLastProfile') {
|
|
725
|
+
const now = Date.now()
|
|
726
|
+
if (now < profileGenerationDate) {
|
|
727
|
+
const error = new Error('No profile available - wait for profiling to complete or trigger manual capture')
|
|
728
|
+
error.code = 'PLT_PPROF_NO_PROFILE_AVAILABLE'
|
|
729
|
+
throw error
|
|
730
|
+
}
|
|
731
|
+
return mockProfile
|
|
732
|
+
}
|
|
733
|
+
return { success: false }
|
|
734
|
+
}
|
|
735
|
+
|
|
736
|
+
await updatePlugin(app)
|
|
737
|
+
await flamegraphsPlugin(app)
|
|
738
|
+
|
|
739
|
+
await app.connectToUpdates()
|
|
740
|
+
await app.setupFlamegraphs()
|
|
741
|
+
|
|
742
|
+
t.after(async () => {
|
|
743
|
+
if (app.cleanupFlamegraphs) {
|
|
744
|
+
app.cleanupFlamegraphs()
|
|
745
|
+
}
|
|
746
|
+
await app.closeUpdates()
|
|
747
|
+
})
|
|
748
|
+
|
|
749
|
+
await waitForClientSubscription
|
|
750
|
+
|
|
751
|
+
const triggerFlamegraphMessage = {
|
|
752
|
+
command: 'trigger-flamegraph'
|
|
753
|
+
}
|
|
754
|
+
|
|
755
|
+
getWs().send(JSON.stringify(triggerFlamegraphMessage))
|
|
756
|
+
|
|
757
|
+
await sleep(15000)
|
|
758
|
+
|
|
759
|
+
const service1AttemptLogs = []
|
|
760
|
+
const service2AttemptLogs = []
|
|
761
|
+
const service1SuccessLogs = []
|
|
762
|
+
const service2SuccessLogs = []
|
|
763
|
+
|
|
764
|
+
for (const infoLog of infoLogs) {
|
|
765
|
+
if (infoLog.length !== 2) continue
|
|
766
|
+
const [options, message] = infoLog
|
|
767
|
+
|
|
768
|
+
if (message.includes('No profile available for the service')) {
|
|
769
|
+
const { workerId, attempt, maxAttempts, attemptTimeout } = options
|
|
770
|
+
|
|
771
|
+
equal(maxAttempts, 11)
|
|
772
|
+
equal(attemptTimeout, 1000)
|
|
773
|
+
|
|
774
|
+
if (workerId === 'service-1') {
|
|
775
|
+
service1AttemptLogs.push(infoLog)
|
|
776
|
+
equal(attempt, service1AttemptLogs.length)
|
|
777
|
+
}
|
|
778
|
+
if (workerId === 'service-2') {
|
|
779
|
+
service2AttemptLogs.push(infoLog)
|
|
780
|
+
equal(attempt, service2AttemptLogs.length)
|
|
781
|
+
}
|
|
782
|
+
continue
|
|
783
|
+
}
|
|
784
|
+
|
|
785
|
+
if (message.includes('Sending flamegraph')) {
|
|
786
|
+
if (options.serviceId === 'service-1') {
|
|
787
|
+
service1SuccessLogs.push(infoLog)
|
|
788
|
+
} else if (options.serviceId === 'service-2') {
|
|
789
|
+
service2SuccessLogs.push(infoLog)
|
|
790
|
+
}
|
|
791
|
+
}
|
|
792
|
+
}
|
|
793
|
+
|
|
794
|
+
equal(service1AttemptLogs.length, 10)
|
|
795
|
+
equal(service2AttemptLogs.length, 10)
|
|
796
|
+
equal(service1SuccessLogs.length, 1)
|
|
797
|
+
equal(service2SuccessLogs.length, 1)
|
|
798
|
+
})
|
|
799
|
+
|
|
800
|
+
test('should handle PLT_PPROF_NOT_ENOUGH_ELU error with info log', async (t) => {
|
|
801
|
+
setUpEnvironment()
|
|
802
|
+
|
|
803
|
+
const receivedMessages = []
|
|
804
|
+
const infoLogs = []
|
|
805
|
+
let errorCount = 0
|
|
806
|
+
let uploadResolve
|
|
807
|
+
const allUploadsComplete = new Promise((resolve) => {
|
|
808
|
+
uploadResolve = resolve
|
|
809
|
+
})
|
|
810
|
+
|
|
811
|
+
const wss = new WebSocketServer({ port: port + 5 })
|
|
812
|
+
t.after(async () => wss.close())
|
|
813
|
+
|
|
814
|
+
const { waitForClientSubscription, getWs } = setupMockIccServer(
|
|
815
|
+
wss,
|
|
816
|
+
receivedMessages,
|
|
817
|
+
true
|
|
818
|
+
)
|
|
819
|
+
|
|
820
|
+
const app = createMockApp(port + 5)
|
|
821
|
+
const originalInfo = app.log.info
|
|
822
|
+
app.log.info = (...args) => {
|
|
823
|
+
originalInfo(...args)
|
|
824
|
+
if (args[1] && args[1].includes('ELU low, CPU profiling not active')) {
|
|
825
|
+
infoLogs.push(args)
|
|
826
|
+
errorCount++
|
|
827
|
+
if (errorCount === 2) {
|
|
828
|
+
uploadResolve()
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
}
|
|
832
|
+
|
|
833
|
+
app.watt.runtime.sendCommandToApplication = async (
|
|
834
|
+
serviceId,
|
|
835
|
+
command
|
|
836
|
+
) => {
|
|
837
|
+
if (command === 'startProfiling') {
|
|
838
|
+
return { success: true }
|
|
750
839
|
}
|
|
840
|
+
if (command === 'getLastProfile') {
|
|
841
|
+
const error = new Error('No profile available - event loop utilization has been below threshold for too long')
|
|
842
|
+
error.code = 'PLT_PPROF_NOT_ENOUGH_ELU'
|
|
843
|
+
throw error
|
|
844
|
+
}
|
|
845
|
+
return { success: false }
|
|
846
|
+
}
|
|
847
|
+
|
|
848
|
+
await updatePlugin(app)
|
|
849
|
+
await flamegraphsPlugin(app)
|
|
850
|
+
|
|
851
|
+
await app.connectToUpdates()
|
|
852
|
+
await app.setupFlamegraphs()
|
|
853
|
+
|
|
854
|
+
t.after(async () => {
|
|
855
|
+
if (app.cleanupFlamegraphs) {
|
|
856
|
+
app.cleanupFlamegraphs()
|
|
857
|
+
}
|
|
858
|
+
await app.closeUpdates()
|
|
859
|
+
})
|
|
860
|
+
|
|
861
|
+
await waitForClientSubscription
|
|
862
|
+
|
|
863
|
+
const triggerFlamegraphMessage = {
|
|
864
|
+
command: 'trigger-flamegraph'
|
|
865
|
+
}
|
|
866
|
+
|
|
867
|
+
getWs().send(JSON.stringify(triggerFlamegraphMessage))
|
|
868
|
+
|
|
869
|
+
await allUploadsComplete
|
|
870
|
+
|
|
871
|
+
equal(infoLogs.length, 2)
|
|
872
|
+
equal(infoLogs[0][0].workerId, 'service-1')
|
|
873
|
+
equal(infoLogs[0][1], 'ELU low, CPU profiling not active')
|
|
874
|
+
})
|
|
875
|
+
|
|
876
|
+
test('should start profiling on new workers that start after initial setup', async (t) => {
|
|
877
|
+
setUpEnvironment()
|
|
878
|
+
|
|
879
|
+
const receivedMessages = []
|
|
880
|
+
const startProfilingCalls = []
|
|
881
|
+
|
|
882
|
+
const wss = new WebSocketServer({ port: port + 6 })
|
|
883
|
+
t.after(async () => wss.close())
|
|
884
|
+
|
|
885
|
+
const { waitForClientSubscription } = setupMockIccServer(
|
|
886
|
+
wss,
|
|
887
|
+
receivedMessages,
|
|
888
|
+
false
|
|
889
|
+
)
|
|
890
|
+
|
|
891
|
+
const app = createMockApp(port + 6)
|
|
892
|
+
|
|
893
|
+
app.watt.runtime.sendCommandToApplication = async (
|
|
894
|
+
serviceId,
|
|
895
|
+
command,
|
|
896
|
+
options
|
|
897
|
+
) => {
|
|
898
|
+
if (command === 'startProfiling') {
|
|
899
|
+
startProfilingCalls.push({ serviceId, options })
|
|
900
|
+
}
|
|
901
|
+
return { success: true }
|
|
902
|
+
}
|
|
903
|
+
|
|
904
|
+
await updatePlugin(app)
|
|
905
|
+
await flamegraphsPlugin(app)
|
|
906
|
+
|
|
907
|
+
await app.connectToUpdates()
|
|
908
|
+
await app.setupFlamegraphs()
|
|
909
|
+
|
|
910
|
+
t.after(async () => {
|
|
911
|
+
if (app.cleanupFlamegraphs) {
|
|
912
|
+
app.cleanupFlamegraphs()
|
|
913
|
+
}
|
|
914
|
+
await app.closeUpdates()
|
|
915
|
+
})
|
|
916
|
+
|
|
917
|
+
await waitForClientSubscription
|
|
918
|
+
|
|
919
|
+
equal(startProfilingCalls.length, 4)
|
|
920
|
+
equal(startProfilingCalls[0].serviceId, 'service-1:0')
|
|
921
|
+
equal(startProfilingCalls[0].options.type, 'cpu')
|
|
922
|
+
equal(startProfilingCalls[1].serviceId, 'service-1:0')
|
|
923
|
+
equal(startProfilingCalls[1].options.type, 'heap')
|
|
924
|
+
equal(startProfilingCalls[2].serviceId, 'service-2:0')
|
|
925
|
+
equal(startProfilingCalls[2].options.type, 'cpu')
|
|
926
|
+
equal(startProfilingCalls[3].serviceId, 'service-2:0')
|
|
927
|
+
equal(startProfilingCalls[3].options.type, 'heap')
|
|
928
|
+
|
|
929
|
+
app.watt.runtime.emit('application:worker:started', {
|
|
930
|
+
application: 'service-1',
|
|
931
|
+
worker: 1,
|
|
932
|
+
workersCount: 2
|
|
933
|
+
})
|
|
934
|
+
|
|
935
|
+
await sleep(10)
|
|
936
|
+
|
|
937
|
+
equal(startProfilingCalls.length, 6)
|
|
938
|
+
equal(startProfilingCalls[4].serviceId, 'service-1:1')
|
|
939
|
+
equal(startProfilingCalls[4].options.durationMillis, 1000)
|
|
940
|
+
equal(startProfilingCalls[4].options.eluThreshold, 0)
|
|
941
|
+
equal(startProfilingCalls[4].options.type, 'cpu')
|
|
942
|
+
equal(startProfilingCalls[5].serviceId, 'service-1:1')
|
|
943
|
+
equal(startProfilingCalls[5].options.durationMillis, 1000)
|
|
944
|
+
equal(startProfilingCalls[5].options.eluThreshold, 0)
|
|
945
|
+
equal(startProfilingCalls[5].options.type, 'heap')
|
|
946
|
+
})
|
|
947
|
+
|
|
948
|
+
test('should not start profiling on new workers when flamegraphs are disabled', async (t) => {
|
|
949
|
+
setUpEnvironment()
|
|
950
|
+
|
|
951
|
+
const receivedMessages = []
|
|
952
|
+
const startProfilingCalls = []
|
|
953
|
+
|
|
954
|
+
const wss = new WebSocketServer({ port: port + 7 })
|
|
955
|
+
t.after(async () => wss.close())
|
|
956
|
+
|
|
957
|
+
const { waitForClientSubscription } = setupMockIccServer(
|
|
958
|
+
wss,
|
|
959
|
+
receivedMessages,
|
|
960
|
+
false
|
|
961
|
+
)
|
|
962
|
+
|
|
963
|
+
const app = createMockApp(port + 7)
|
|
964
|
+
app.env.PLT_DISABLE_FLAMEGRAPHS = true
|
|
965
|
+
|
|
966
|
+
app.watt.runtime.sendCommandToApplication = async (
|
|
967
|
+
serviceId,
|
|
968
|
+
command,
|
|
969
|
+
options
|
|
970
|
+
) => {
|
|
971
|
+
if (command === 'startProfiling') {
|
|
972
|
+
startProfilingCalls.push({ serviceId, options })
|
|
973
|
+
}
|
|
974
|
+
return { success: true }
|
|
975
|
+
}
|
|
976
|
+
|
|
977
|
+
await updatePlugin(app)
|
|
978
|
+
await flamegraphsPlugin(app)
|
|
979
|
+
|
|
980
|
+
await app.connectToUpdates()
|
|
981
|
+
await app.setupFlamegraphs()
|
|
982
|
+
|
|
983
|
+
t.after(async () => {
|
|
984
|
+
if (app.cleanupFlamegraphs) {
|
|
985
|
+
app.cleanupFlamegraphs()
|
|
986
|
+
}
|
|
987
|
+
await app.closeUpdates()
|
|
988
|
+
})
|
|
989
|
+
|
|
990
|
+
await waitForClientSubscription
|
|
991
|
+
|
|
992
|
+
equal(startProfilingCalls.length, 0)
|
|
993
|
+
|
|
994
|
+
app.watt.runtime.emit('application:worker:started', {
|
|
995
|
+
application: 'service-1',
|
|
996
|
+
worker: 1,
|
|
997
|
+
workersCount: 2
|
|
998
|
+
})
|
|
999
|
+
|
|
1000
|
+
await sleep(10)
|
|
1001
|
+
|
|
1002
|
+
equal(startProfilingCalls.length, 0)
|
|
1003
|
+
})
|
|
1004
|
+
|
|
1005
|
+
test('sendFlamegraphs should include alertId in query params when provided', async (t) => {
|
|
1006
|
+
setUpEnvironment()
|
|
1007
|
+
|
|
1008
|
+
const uploadedRequests = []
|
|
1009
|
+
|
|
1010
|
+
const app = createMockApp(port + 18)
|
|
1011
|
+
|
|
1012
|
+
app.watt.runtime.sendCommandToApplication = async (serviceId, command) => {
|
|
751
1013
|
if (command === 'getLastProfile') {
|
|
752
1014
|
return new Uint8Array([1, 2, 3])
|
|
753
1015
|
}
|
|
@@ -765,25 +1027,15 @@ test('requestFlamegraphs should include alertId in query params when provided',
|
|
|
765
1027
|
method: req.method
|
|
766
1028
|
})
|
|
767
1029
|
res.writeHead(200)
|
|
768
|
-
res.end(
|
|
1030
|
+
res.end()
|
|
769
1031
|
})
|
|
770
1032
|
})
|
|
771
1033
|
|
|
772
|
-
await new Promise(resolve => server.listen(port +
|
|
1034
|
+
await new Promise(resolve => server.listen(port + 18, resolve))
|
|
773
1035
|
t.after(() => server.close())
|
|
774
1036
|
|
|
775
1037
|
await flamegraphsPlugin(app)
|
|
776
|
-
await app.
|
|
777
|
-
|
|
778
|
-
t.after(async () => {
|
|
779
|
-
await app.cleanupFlamegraphs()
|
|
780
|
-
})
|
|
781
|
-
|
|
782
|
-
// Trigger profiling with alertId
|
|
783
|
-
await app.requestFlamegraphs({ alertId: 'test-alert-123' })
|
|
784
|
-
|
|
785
|
-
// Wait for profile to be generated (duration is 1 second)
|
|
786
|
-
await sleep(1500)
|
|
1038
|
+
await app.sendFlamegraphs({ alertId: 'test-alert-123' })
|
|
787
1039
|
|
|
788
1040
|
equal(uploadedRequests.length, 2, 'Should upload flamegraphs for both services')
|
|
789
1041
|
|