@platformatic/watt-extra 1.18.0-alpha.1 → 1.18.0-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/plugins/flamegraphs.js +50 -1
- package/test/profiling-toggle.test.js +60 -0
package/package.json
CHANGED
package/plugins/flamegraphs.js
CHANGED
|
@@ -42,7 +42,38 @@ async function flamegraphs (app, _opts) {
|
|
|
42
42
|
const enabledTypes = new Set(isFlamegraphsDisabled ? [] : PROFILE_TYPES)
|
|
43
43
|
|
|
44
44
|
let workerStartedListener = null
|
|
45
|
+
let healthListener = null
|
|
45
46
|
let stateReportTimer = null
|
|
47
|
+
|
|
48
|
+
// Last observed event-loop utilization per worker, from the same health
|
|
49
|
+
// events that drive the runtime's profiling ELU gate. It is attached to
|
|
50
|
+
// the profiling state report so ICC can tell an idle pause (ELU below the
|
|
51
|
+
// profiling threshold) from an overload pause (ELU above the cutoff): the
|
|
52
|
+
// profiler state itself does not carry the pause reason. The field is
|
|
53
|
+
// optional on both sides, so any watt-extra/ICC version mix keeps working.
|
|
54
|
+
const ELU_SAMPLE_MAX_AGE = 60 * 1000
|
|
55
|
+
const lastWorkerELU = new Map()
|
|
56
|
+
|
|
57
|
+
function setupHealthTracking (runtime) {
|
|
58
|
+
const healthEventName = app.watt.runtimeSupportsNewHealthMetrics?.()
|
|
59
|
+
? 'application:worker:health:metrics'
|
|
60
|
+
: 'application:worker:health'
|
|
61
|
+
|
|
62
|
+
// Remove the old listener if it exists (for ICC recovery scenario)
|
|
63
|
+
if (healthListener) {
|
|
64
|
+
runtime.removeListener(healthEventName, healthListener)
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
healthListener = (healthInfo) => {
|
|
68
|
+
const workerId = healthInfo?.id
|
|
69
|
+
const elu = healthInfo?.currentHealth?.elu
|
|
70
|
+
if (workerId == null || typeof elu !== 'number') {
|
|
71
|
+
return
|
|
72
|
+
}
|
|
73
|
+
lastWorkerELU.set(workerId, { elu, at: Date.now() })
|
|
74
|
+
}
|
|
75
|
+
runtime.on(healthEventName, healthListener)
|
|
76
|
+
}
|
|
46
77
|
// Set to false when the ICC scaler does not expose the profiling states
|
|
47
78
|
// API (older ICC): reporting is disabled for the lifetime of the pod.
|
|
48
79
|
let stateReportingSupported = true
|
|
@@ -123,6 +154,8 @@ async function flamegraphs (app, _opts) {
|
|
|
123
154
|
}
|
|
124
155
|
}
|
|
125
156
|
|
|
157
|
+
setupHealthTracking(runtime)
|
|
158
|
+
|
|
126
159
|
// Remove old listener if it exists (for ICC recovery scenario)
|
|
127
160
|
if (workerStartedListener) {
|
|
128
161
|
runtime.removeListener('application:worker:started', workerStartedListener)
|
|
@@ -173,6 +206,15 @@ async function flamegraphs (app, _opts) {
|
|
|
173
206
|
stateReportTimer = null
|
|
174
207
|
}
|
|
175
208
|
|
|
209
|
+
if (healthListener && app.watt?.runtime) {
|
|
210
|
+
const healthEventName = app.watt.runtimeSupportsNewHealthMetrics?.()
|
|
211
|
+
? 'application:worker:health:metrics'
|
|
212
|
+
: 'application:worker:health'
|
|
213
|
+
app.watt.runtime.removeListener(healthEventName, healthListener)
|
|
214
|
+
healthListener = null
|
|
215
|
+
}
|
|
216
|
+
lastWorkerELU.clear()
|
|
217
|
+
|
|
176
218
|
// Explicitly stop all active profiling sessions to avoid memory
|
|
177
219
|
// corruption. Profiling may be active even when PLT_DISABLE_FLAMEGRAPHS
|
|
178
220
|
// is set (runtime activation), so always attempt the stop: it is
|
|
@@ -662,13 +704,20 @@ async function flamegraphs (app, _opts) {
|
|
|
662
704
|
}
|
|
663
705
|
}
|
|
664
706
|
|
|
665
|
-
|
|
707
|
+
const entry = {
|
|
666
708
|
serviceId,
|
|
667
709
|
workerId: workerFullId,
|
|
668
710
|
type,
|
|
669
711
|
enabled: enabledTypes.has(type),
|
|
670
712
|
...state
|
|
671
713
|
}
|
|
714
|
+
|
|
715
|
+
const eluSample = lastWorkerELU.get(workerFullId) ?? lastWorkerELU.get(serviceId)
|
|
716
|
+
if (eluSample && Date.now() - eluSample.at <= ELU_SAMPLE_MAX_AGE) {
|
|
717
|
+
entry.lastELU = eluSample.elu
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
return entry
|
|
672
721
|
}
|
|
673
722
|
|
|
674
723
|
async function sendProfilingStates (scalerUrl, applicationId, states) {
|
|
@@ -744,3 +744,63 @@ test('a timed-out worker is not queried again until it restarts', async (t) => {
|
|
|
744
744
|
await waitForReports(3)
|
|
745
745
|
equal(queriesToBlockedWorker(), 4)
|
|
746
746
|
})
|
|
747
|
+
|
|
748
|
+
test('attaches the last observed worker ELU to the state report', async (t) => {
|
|
749
|
+
setUpEnvironment()
|
|
750
|
+
|
|
751
|
+
const { statesReports, url, waitForReports } = await startScalerMock(t)
|
|
752
|
+
const app = createMockApp({ scalerUrl: url })
|
|
753
|
+
|
|
754
|
+
await flamegraphsPlugin(app)
|
|
755
|
+
await app.setupFlamegraphs()
|
|
756
|
+
t.after(() => app.cleanupFlamegraphs())
|
|
757
|
+
await waitForReports(1)
|
|
758
|
+
|
|
759
|
+
// The runtime health events carry the same ELU the profiling gate uses
|
|
760
|
+
app.watt.runtime.emit('application:worker:health', {
|
|
761
|
+
id: 'service-1:0',
|
|
762
|
+
application: 'service-1',
|
|
763
|
+
currentHealth: { elu: 0.97, heapUsed: 1, heapTotal: 2 }
|
|
764
|
+
})
|
|
765
|
+
|
|
766
|
+
await app.reportProfilingStates()
|
|
767
|
+
await waitForReports(2)
|
|
768
|
+
|
|
769
|
+
const report = statesReports[statesReports.length - 1]
|
|
770
|
+
const sampled = report.states.filter((s) => s.workerId === 'service-1:0')
|
|
771
|
+
ok(sampled.length > 0)
|
|
772
|
+
ok(sampled.every((s) => s.lastELU === 0.97))
|
|
773
|
+
|
|
774
|
+
// Workers without a sample simply omit the field (older runtimes, quiet
|
|
775
|
+
// workers): the consumer must not require it
|
|
776
|
+
const unsampled = report.states.filter((s) => s.workerId === 'service-2:0')
|
|
777
|
+
ok(unsampled.every((s) => s.lastELU === undefined))
|
|
778
|
+
})
|
|
779
|
+
|
|
780
|
+
test('a stale ELU sample is not attached to the state report', async (t) => {
|
|
781
|
+
setUpEnvironment()
|
|
782
|
+
|
|
783
|
+
const { statesReports, url, waitForReports } = await startScalerMock(t)
|
|
784
|
+
const app = createMockApp({ scalerUrl: url })
|
|
785
|
+
|
|
786
|
+
await flamegraphsPlugin(app)
|
|
787
|
+
await app.setupFlamegraphs()
|
|
788
|
+
t.after(() => app.cleanupFlamegraphs())
|
|
789
|
+
await waitForReports(1)
|
|
790
|
+
|
|
791
|
+
app.watt.runtime.emit('application:worker:health', {
|
|
792
|
+
id: 'service-1:0',
|
|
793
|
+
application: 'service-1',
|
|
794
|
+
currentHealth: { elu: 0.97 }
|
|
795
|
+
})
|
|
796
|
+
|
|
797
|
+
// Jump past the sample max age
|
|
798
|
+
const realNow = Date.now()
|
|
799
|
+
t.mock.method(Date, 'now', () => realNow + 61 * 1000)
|
|
800
|
+
|
|
801
|
+
await app.reportProfilingStates()
|
|
802
|
+
await waitForReports(2)
|
|
803
|
+
|
|
804
|
+
const report = statesReports[statesReports.length - 1]
|
|
805
|
+
ok(report.states.every((s) => s.lastELU === undefined))
|
|
806
|
+
})
|