@platformatic/watt-extra 1.17.0 → 1.18.0-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/plugins/env.js +2 -0
- package/plugins/flamegraphs.js +50 -1
- package/plugins/update.js +54 -2
- package/test/profiling-toggle.test.js +60 -0
- package/test/update.test.js +69 -0
package/package.json
CHANGED
package/plugins/env.js
CHANGED
|
@@ -27,6 +27,8 @@ const schema = {
|
|
|
27
27
|
PLT_FLAMEGRAPHS_STATE_QUERY_TIMEOUT: { type: 'number', default: 5000 },
|
|
28
28
|
PLT_JWT_EXPIRATION_OFFSET_SEC: { type: 'number', default: 60 },
|
|
29
29
|
PLT_UPDATES_RECONNECT_INTERVAL_SEC: { type: 'number', default: 1 },
|
|
30
|
+
PLT_UPDATES_HEARTBEAT_INTERVAL_SEC: { type: 'number', default: 30 },
|
|
31
|
+
PLT_UPDATES_ACK_TIMEOUT_SEC: { type: 'number', default: 10 },
|
|
30
32
|
PLT_ELU_HEALTH_SIGNAL_THRESHOLD: { type: 'number', default: 0.7 },
|
|
31
33
|
PLT_HEAP_HEALTH_SIGNAL_THRESHOLD: { type: ['number', 'string'], default: '400MB' },
|
|
32
34
|
PLT_ALERTS_GRACE_PERIOD_SEC: { type: 'number', default: 30 },
|
package/plugins/flamegraphs.js
CHANGED
|
@@ -42,7 +42,38 @@ async function flamegraphs (app, _opts) {
|
|
|
42
42
|
const enabledTypes = new Set(isFlamegraphsDisabled ? [] : PROFILE_TYPES)
|
|
43
43
|
|
|
44
44
|
let workerStartedListener = null
|
|
45
|
+
let healthListener = null
|
|
45
46
|
let stateReportTimer = null
|
|
47
|
+
|
|
48
|
+
// Last observed event-loop utilization per worker, from the same health
|
|
49
|
+
// events that drive the runtime's profiling ELU gate. It is attached to
|
|
50
|
+
// the profiling state report so ICC can tell an idle pause (ELU below the
|
|
51
|
+
// profiling threshold) from an overload pause (ELU above the cutoff): the
|
|
52
|
+
// profiler state itself does not carry the pause reason. The field is
|
|
53
|
+
// optional on both sides, so any watt-extra/ICC version mix keeps working.
|
|
54
|
+
const ELU_SAMPLE_MAX_AGE = 60 * 1000
|
|
55
|
+
const lastWorkerELU = new Map()
|
|
56
|
+
|
|
57
|
+
function setupHealthTracking (runtime) {
|
|
58
|
+
const healthEventName = app.watt.runtimeSupportsNewHealthMetrics?.()
|
|
59
|
+
? 'application:worker:health:metrics'
|
|
60
|
+
: 'application:worker:health'
|
|
61
|
+
|
|
62
|
+
// Remove the old listener if it exists (for ICC recovery scenario)
|
|
63
|
+
if (healthListener) {
|
|
64
|
+
runtime.removeListener(healthEventName, healthListener)
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
healthListener = (healthInfo) => {
|
|
68
|
+
const workerId = healthInfo?.id
|
|
69
|
+
const elu = healthInfo?.currentHealth?.elu
|
|
70
|
+
if (workerId == null || typeof elu !== 'number') {
|
|
71
|
+
return
|
|
72
|
+
}
|
|
73
|
+
lastWorkerELU.set(workerId, { elu, at: Date.now() })
|
|
74
|
+
}
|
|
75
|
+
runtime.on(healthEventName, healthListener)
|
|
76
|
+
}
|
|
46
77
|
// Set to false when the ICC scaler does not expose the profiling states
|
|
47
78
|
// API (older ICC): reporting is disabled for the lifetime of the pod.
|
|
48
79
|
let stateReportingSupported = true
|
|
@@ -123,6 +154,8 @@ async function flamegraphs (app, _opts) {
|
|
|
123
154
|
}
|
|
124
155
|
}
|
|
125
156
|
|
|
157
|
+
setupHealthTracking(runtime)
|
|
158
|
+
|
|
126
159
|
// Remove old listener if it exists (for ICC recovery scenario)
|
|
127
160
|
if (workerStartedListener) {
|
|
128
161
|
runtime.removeListener('application:worker:started', workerStartedListener)
|
|
@@ -173,6 +206,15 @@ async function flamegraphs (app, _opts) {
|
|
|
173
206
|
stateReportTimer = null
|
|
174
207
|
}
|
|
175
208
|
|
|
209
|
+
if (healthListener && app.watt?.runtime) {
|
|
210
|
+
const healthEventName = app.watt.runtimeSupportsNewHealthMetrics?.()
|
|
211
|
+
? 'application:worker:health:metrics'
|
|
212
|
+
: 'application:worker:health'
|
|
213
|
+
app.watt.runtime.removeListener(healthEventName, healthListener)
|
|
214
|
+
healthListener = null
|
|
215
|
+
}
|
|
216
|
+
lastWorkerELU.clear()
|
|
217
|
+
|
|
176
218
|
// Explicitly stop all active profiling sessions to avoid memory
|
|
177
219
|
// corruption. Profiling may be active even when PLT_DISABLE_FLAMEGRAPHS
|
|
178
220
|
// is set (runtime activation), so always attempt the stop: it is
|
|
@@ -662,13 +704,20 @@ async function flamegraphs (app, _opts) {
|
|
|
662
704
|
}
|
|
663
705
|
}
|
|
664
706
|
|
|
665
|
-
|
|
707
|
+
const entry = {
|
|
666
708
|
serviceId,
|
|
667
709
|
workerId: workerFullId,
|
|
668
710
|
type,
|
|
669
711
|
enabled: enabledTypes.has(type),
|
|
670
712
|
...state
|
|
671
713
|
}
|
|
714
|
+
|
|
715
|
+
const eluSample = lastWorkerELU.get(workerFullId) ?? lastWorkerELU.get(serviceId)
|
|
716
|
+
if (eluSample && Date.now() - eluSample.at <= ELU_SAMPLE_MAX_AGE) {
|
|
717
|
+
entry.lastELU = eluSample.elu
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
return entry
|
|
672
721
|
}
|
|
673
722
|
|
|
674
723
|
async function sendProfilingStates (scalerUrl, applicationId, states) {
|
package/plugins/update.js
CHANGED
|
@@ -15,9 +15,44 @@ function createWebSocketUrl (httpUrl, path, queryParams = {}) {
|
|
|
15
15
|
|
|
16
16
|
async function updatePlugin (app) {
|
|
17
17
|
const reconnectInterval = app.env.PLT_UPDATES_RECONNECT_INTERVAL_SEC * 1000
|
|
18
|
+
const heartbeatInterval = (app.env.PLT_UPDATES_HEARTBEAT_INTERVAL_SEC ?? 30) * 1000
|
|
19
|
+
const ackTimeout = (app.env.PLT_UPDATES_ACK_TIMEOUT_SEC ?? 10) * 1000
|
|
20
|
+
const kAckTimeout = Symbol('kAckTimeout')
|
|
18
21
|
|
|
19
22
|
let socket = null
|
|
20
23
|
|
|
24
|
+
// A half-open connection (e.g. the ICC endpoint died without a TCP reset
|
|
25
|
+
// reaching the pod) fires no 'close' event: without a heartbeat the pod
|
|
26
|
+
// believes it is connected forever and becomes unreachable for ICC
|
|
27
|
+
// commands, while its outgoing HTTP calls keep working. Ping the server
|
|
28
|
+
// and terminate the socket when the pong does not come back, so the close
|
|
29
|
+
// handler runs and the reconnect loop takes over. The server side of ws
|
|
30
|
+
// answers pings automatically, so this works against any ICC version.
|
|
31
|
+
function startHeartbeat (ws) {
|
|
32
|
+
let alive = true
|
|
33
|
+
ws.on('pong', () => {
|
|
34
|
+
alive = true
|
|
35
|
+
})
|
|
36
|
+
|
|
37
|
+
const timer = setInterval(() => {
|
|
38
|
+
if (ws.readyState !== WebSocket.OPEN) {
|
|
39
|
+
return
|
|
40
|
+
}
|
|
41
|
+
if (!alive) {
|
|
42
|
+
app.log.warn('Updates websocket heartbeat timed out, terminating the connection')
|
|
43
|
+
ws.terminate()
|
|
44
|
+
return
|
|
45
|
+
}
|
|
46
|
+
alive = false
|
|
47
|
+
ws.ping()
|
|
48
|
+
}, heartbeatInterval)
|
|
49
|
+
timer.unref()
|
|
50
|
+
|
|
51
|
+
ws.on('close', () => {
|
|
52
|
+
clearInterval(timer)
|
|
53
|
+
})
|
|
54
|
+
}
|
|
55
|
+
|
|
21
56
|
async function processMessage (data) {
|
|
22
57
|
try {
|
|
23
58
|
const message = JSON.parse(data)
|
|
@@ -115,7 +150,9 @@ async function updatePlugin (app) {
|
|
|
115
150
|
try {
|
|
116
151
|
const headers = await app.getAuthorizationHeaders()
|
|
117
152
|
|
|
118
|
-
|
|
153
|
+
// handshakeTimeout turns a hung upgrade into an 'error', so the retry
|
|
154
|
+
// loop is never stuck waiting for an 'open' that cannot arrive
|
|
155
|
+
socket = new WebSocket(wsUrl, { headers, handshakeTimeout: ackTimeout })
|
|
119
156
|
await once(socket, 'open')
|
|
120
157
|
|
|
121
158
|
app.log.info('Connected to updates websocket')
|
|
@@ -123,7 +160,20 @@ async function updatePlugin (app) {
|
|
|
123
160
|
const subscribeMsg = JSON.stringify({ command: 'subscribe', topic: '/config' })
|
|
124
161
|
socket.send(subscribeMsg)
|
|
125
162
|
|
|
126
|
-
|
|
163
|
+
// A connection which closes cleanly between the subscribe and the ack
|
|
164
|
+
// emits no 'error', so an unbounded wait here would hang this loop
|
|
165
|
+
// forever: bound it and retry
|
|
166
|
+
const ackPromise = once(socket, 'message')
|
|
167
|
+
ackPromise.catch(() => {})
|
|
168
|
+
const command = await Promise.race([
|
|
169
|
+
ackPromise,
|
|
170
|
+
sleep(ackTimeout, kAckTimeout, { ref: false })
|
|
171
|
+
])
|
|
172
|
+
if (command === kAckTimeout) {
|
|
173
|
+
socket.terminate()
|
|
174
|
+
throw new Error('Timed out waiting for the updates subscription acknowledgment')
|
|
175
|
+
}
|
|
176
|
+
|
|
127
177
|
const message = JSON.parse(command[0])
|
|
128
178
|
if (message?.command !== 'ack') {
|
|
129
179
|
app.log.error({ message }, 'Subscription updates failed')
|
|
@@ -131,6 +181,8 @@ async function updatePlugin (app) {
|
|
|
131
181
|
}
|
|
132
182
|
app.log.info('Received subscription acknowledgment from updates websocket')
|
|
133
183
|
|
|
184
|
+
startHeartbeat(socket)
|
|
185
|
+
|
|
134
186
|
// listen for subsequent messages
|
|
135
187
|
socket.on('message', processMessage)
|
|
136
188
|
|
|
@@ -744,3 +744,63 @@ test('a timed-out worker is not queried again until it restarts', async (t) => {
|
|
|
744
744
|
await waitForReports(3)
|
|
745
745
|
equal(queriesToBlockedWorker(), 4)
|
|
746
746
|
})
|
|
747
|
+
|
|
748
|
+
test('attaches the last observed worker ELU to the state report', async (t) => {
|
|
749
|
+
setUpEnvironment()
|
|
750
|
+
|
|
751
|
+
const { statesReports, url, waitForReports } = await startScalerMock(t)
|
|
752
|
+
const app = createMockApp({ scalerUrl: url })
|
|
753
|
+
|
|
754
|
+
await flamegraphsPlugin(app)
|
|
755
|
+
await app.setupFlamegraphs()
|
|
756
|
+
t.after(() => app.cleanupFlamegraphs())
|
|
757
|
+
await waitForReports(1)
|
|
758
|
+
|
|
759
|
+
// The runtime health events carry the same ELU the profiling gate uses
|
|
760
|
+
app.watt.runtime.emit('application:worker:health', {
|
|
761
|
+
id: 'service-1:0',
|
|
762
|
+
application: 'service-1',
|
|
763
|
+
currentHealth: { elu: 0.97, heapUsed: 1, heapTotal: 2 }
|
|
764
|
+
})
|
|
765
|
+
|
|
766
|
+
await app.reportProfilingStates()
|
|
767
|
+
await waitForReports(2)
|
|
768
|
+
|
|
769
|
+
const report = statesReports[statesReports.length - 1]
|
|
770
|
+
const sampled = report.states.filter((s) => s.workerId === 'service-1:0')
|
|
771
|
+
ok(sampled.length > 0)
|
|
772
|
+
ok(sampled.every((s) => s.lastELU === 0.97))
|
|
773
|
+
|
|
774
|
+
// Workers without a sample simply omit the field (older runtimes, quiet
|
|
775
|
+
// workers): the consumer must not require it
|
|
776
|
+
const unsampled = report.states.filter((s) => s.workerId === 'service-2:0')
|
|
777
|
+
ok(unsampled.every((s) => s.lastELU === undefined))
|
|
778
|
+
})
|
|
779
|
+
|
|
780
|
+
test('a stale ELU sample is not attached to the state report', async (t) => {
|
|
781
|
+
setUpEnvironment()
|
|
782
|
+
|
|
783
|
+
const { statesReports, url, waitForReports } = await startScalerMock(t)
|
|
784
|
+
const app = createMockApp({ scalerUrl: url })
|
|
785
|
+
|
|
786
|
+
await flamegraphsPlugin(app)
|
|
787
|
+
await app.setupFlamegraphs()
|
|
788
|
+
t.after(() => app.cleanupFlamegraphs())
|
|
789
|
+
await waitForReports(1)
|
|
790
|
+
|
|
791
|
+
app.watt.runtime.emit('application:worker:health', {
|
|
792
|
+
id: 'service-1:0',
|
|
793
|
+
application: 'service-1',
|
|
794
|
+
currentHealth: { elu: 0.97 }
|
|
795
|
+
})
|
|
796
|
+
|
|
797
|
+
// Jump past the sample max age
|
|
798
|
+
const realNow = Date.now()
|
|
799
|
+
t.mock.method(Date, 'now', () => realNow + 61 * 1000)
|
|
800
|
+
|
|
801
|
+
await app.reportProfilingStates()
|
|
802
|
+
await waitForReports(2)
|
|
803
|
+
|
|
804
|
+
const report = statesReports[statesReports.length - 1]
|
|
805
|
+
ok(report.states.every((s) => s.lastELU === undefined))
|
|
806
|
+
})
|
package/test/update.test.js
CHANGED
|
@@ -564,3 +564,72 @@ test('should reconnect to updates if connection closes', async (t) => {
|
|
|
564
564
|
deepEqual(processedMessages, [testMessage1, testMessage2])
|
|
565
565
|
wss2.close()
|
|
566
566
|
})
|
|
567
|
+
|
|
568
|
+
test('terminates a half-open connection via heartbeat and reconnects', async (t) => {
|
|
569
|
+
setUpEnvironment()
|
|
570
|
+
|
|
571
|
+
// autoPong: false simulates a peer that no longer answers: the client's
|
|
572
|
+
// pings get no pong back, exactly like a half-open TCP connection
|
|
573
|
+
const wss = new WebSocketServer({ port: port + 20, autoPong: false })
|
|
574
|
+
t.after(async () => wss.close())
|
|
575
|
+
|
|
576
|
+
const ee = new EventEmitter()
|
|
577
|
+
let connections = 0
|
|
578
|
+
wss.on('connection', (ws) => {
|
|
579
|
+
connections++
|
|
580
|
+
if (connections >= 2) {
|
|
581
|
+
ee.emit('reconnected')
|
|
582
|
+
}
|
|
583
|
+
ws.on('message', (data) => {
|
|
584
|
+
const message = JSON.parse(data.toString())
|
|
585
|
+
if (message.command === 'subscribe' && message.topic === '/config') {
|
|
586
|
+
ws.send(JSON.stringify({ command: 'ack' }))
|
|
587
|
+
}
|
|
588
|
+
})
|
|
589
|
+
})
|
|
590
|
+
|
|
591
|
+
const app = createMockApp(port + 20)
|
|
592
|
+
app.env.PLT_UPDATES_HEARTBEAT_INTERVAL_SEC = 0.1
|
|
593
|
+
app.env.PLT_UPDATES_RECONNECT_INTERVAL_SEC = 0.05
|
|
594
|
+
t.after(() => app.closeUpdates())
|
|
595
|
+
|
|
596
|
+
await updatePlugin(app)
|
|
597
|
+
await app.connectToUpdates()
|
|
598
|
+
|
|
599
|
+
// First heartbeat miss terminates the connection, the reconnect loop
|
|
600
|
+
// opens a new one, which times out again, and so on
|
|
601
|
+
const reconnected = once(ee, 'reconnected')
|
|
602
|
+
await reconnected
|
|
603
|
+
|
|
604
|
+
equal(connections >= 2, true, 'Should reconnect after the heartbeat timeout')
|
|
605
|
+
})
|
|
606
|
+
|
|
607
|
+
test('retries when the subscription acknowledgment never arrives', async (t) => {
|
|
608
|
+
setUpEnvironment()
|
|
609
|
+
|
|
610
|
+
const wss = new WebSocketServer({ port: port + 21 })
|
|
611
|
+
t.after(async () => wss.close())
|
|
612
|
+
|
|
613
|
+
const ee = new EventEmitter()
|
|
614
|
+
let connections = 0
|
|
615
|
+
wss.on('connection', () => {
|
|
616
|
+
// Accept the connection and the subscribe message, never send the ack
|
|
617
|
+
connections++
|
|
618
|
+
if (connections >= 2) {
|
|
619
|
+
ee.emit('reconnected')
|
|
620
|
+
}
|
|
621
|
+
})
|
|
622
|
+
|
|
623
|
+
const app = createMockApp(port + 21)
|
|
624
|
+
app.env.PLT_UPDATES_ACK_TIMEOUT_SEC = 0.2
|
|
625
|
+
app.env.PLT_UPDATES_RECONNECT_INTERVAL_SEC = 0.05
|
|
626
|
+
t.after(() => app.closeUpdates())
|
|
627
|
+
|
|
628
|
+
await updatePlugin(app)
|
|
629
|
+
await app.connectToUpdates()
|
|
630
|
+
|
|
631
|
+
const reconnected = once(ee, 'reconnected')
|
|
632
|
+
await reconnected
|
|
633
|
+
|
|
634
|
+
equal(connections >= 2, true, 'Should retry after the ack timeout')
|
|
635
|
+
})
|