@platformatic/watt-extra 1.17.0 → 1.18.0-alpha.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@platformatic/watt-extra",
3
- "version": "1.17.0",
3
+ "version": "1.18.0-alpha.2",
4
4
  "description": "The Platformatic runtime manager",
5
5
  "type": "module",
6
6
  "scripts": {
package/plugins/env.js CHANGED
@@ -27,6 +27,8 @@ const schema = {
27
27
  PLT_FLAMEGRAPHS_STATE_QUERY_TIMEOUT: { type: 'number', default: 5000 },
28
28
  PLT_JWT_EXPIRATION_OFFSET_SEC: { type: 'number', default: 60 },
29
29
  PLT_UPDATES_RECONNECT_INTERVAL_SEC: { type: 'number', default: 1 },
30
+ PLT_UPDATES_HEARTBEAT_INTERVAL_SEC: { type: 'number', default: 30 },
31
+ PLT_UPDATES_ACK_TIMEOUT_SEC: { type: 'number', default: 10 },
30
32
  PLT_ELU_HEALTH_SIGNAL_THRESHOLD: { type: 'number', default: 0.7 },
31
33
  PLT_HEAP_HEALTH_SIGNAL_THRESHOLD: { type: ['number', 'string'], default: '400MB' },
32
34
  PLT_ALERTS_GRACE_PERIOD_SEC: { type: 'number', default: 30 },
@@ -42,7 +42,38 @@ async function flamegraphs (app, _opts) {
42
42
  const enabledTypes = new Set(isFlamegraphsDisabled ? [] : PROFILE_TYPES)
43
43
 
44
44
  let workerStartedListener = null
45
+ let healthListener = null
45
46
  let stateReportTimer = null
47
+
48
+ // Last observed event-loop utilization per worker, from the same health
49
+ // events that drive the runtime's profiling ELU gate. It is attached to
50
+ // the profiling state report so ICC can tell an idle pause (ELU below the
51
+ // profiling threshold) from an overload pause (ELU above the cutoff): the
52
+ // profiler state itself does not carry the pause reason. The field is
53
+ // optional on both sides, so any watt-extra/ICC version mix keeps working.
54
+ const ELU_SAMPLE_MAX_AGE = 60 * 1000
55
+ const lastWorkerELU = new Map()
56
+
57
+ function setupHealthTracking (runtime) {
58
+ const healthEventName = app.watt.runtimeSupportsNewHealthMetrics?.()
59
+ ? 'application:worker:health:metrics'
60
+ : 'application:worker:health'
61
+
62
+ // Remove the old listener if it exists (for ICC recovery scenario)
63
+ if (healthListener) {
64
+ runtime.removeListener(healthEventName, healthListener)
65
+ }
66
+
67
+ healthListener = (healthInfo) => {
68
+ const workerId = healthInfo?.id
69
+ const elu = healthInfo?.currentHealth?.elu
70
+ if (workerId == null || typeof elu !== 'number') {
71
+ return
72
+ }
73
+ lastWorkerELU.set(workerId, { elu, at: Date.now() })
74
+ }
75
+ runtime.on(healthEventName, healthListener)
76
+ }
46
77
  // Set to false when the ICC scaler does not expose the profiling states
47
78
  // API (older ICC): reporting is disabled for the lifetime of the pod.
48
79
  let stateReportingSupported = true
@@ -123,6 +154,8 @@ async function flamegraphs (app, _opts) {
123
154
  }
124
155
  }
125
156
 
157
+ setupHealthTracking(runtime)
158
+
126
159
  // Remove old listener if it exists (for ICC recovery scenario)
127
160
  if (workerStartedListener) {
128
161
  runtime.removeListener('application:worker:started', workerStartedListener)
@@ -173,6 +206,15 @@ async function flamegraphs (app, _opts) {
173
206
  stateReportTimer = null
174
207
  }
175
208
 
209
+ if (healthListener && app.watt?.runtime) {
210
+ const healthEventName = app.watt.runtimeSupportsNewHealthMetrics?.()
211
+ ? 'application:worker:health:metrics'
212
+ : 'application:worker:health'
213
+ app.watt.runtime.removeListener(healthEventName, healthListener)
214
+ healthListener = null
215
+ }
216
+ lastWorkerELU.clear()
217
+
176
218
  // Explicitly stop all active profiling sessions to avoid memory
177
219
  // corruption. Profiling may be active even when PLT_DISABLE_FLAMEGRAPHS
178
220
  // is set (runtime activation), so always attempt the stop: it is
@@ -662,13 +704,20 @@ async function flamegraphs (app, _opts) {
662
704
  }
663
705
  }
664
706
 
665
- return {
707
+ const entry = {
666
708
  serviceId,
667
709
  workerId: workerFullId,
668
710
  type,
669
711
  enabled: enabledTypes.has(type),
670
712
  ...state
671
713
  }
714
+
715
+ const eluSample = lastWorkerELU.get(workerFullId) ?? lastWorkerELU.get(serviceId)
716
+ if (eluSample && Date.now() - eluSample.at <= ELU_SAMPLE_MAX_AGE) {
717
+ entry.lastELU = eluSample.elu
718
+ }
719
+
720
+ return entry
672
721
  }
673
722
 
674
723
  async function sendProfilingStates (scalerUrl, applicationId, states) {
package/plugins/update.js CHANGED
@@ -15,9 +15,44 @@ function createWebSocketUrl (httpUrl, path, queryParams = {}) {
15
15
 
16
16
  async function updatePlugin (app) {
17
17
  const reconnectInterval = app.env.PLT_UPDATES_RECONNECT_INTERVAL_SEC * 1000
18
+ const heartbeatInterval = (app.env.PLT_UPDATES_HEARTBEAT_INTERVAL_SEC ?? 30) * 1000
19
+ const ackTimeout = (app.env.PLT_UPDATES_ACK_TIMEOUT_SEC ?? 10) * 1000
20
+ const kAckTimeout = Symbol('kAckTimeout')
18
21
 
19
22
  let socket = null
20
23
 
24
+ // A half-open connection (e.g. the ICC endpoint died without a TCP reset
25
+ // reaching the pod) fires no 'close' event: without a heartbeat the pod
26
+ // believes it is connected forever and becomes unreachable for ICC
27
+ // commands, while its outgoing HTTP calls keep working. Ping the server
28
+ // and terminate the socket when the pong does not come back, so the close
29
+ // handler runs and the reconnect loop takes over. The server side of ws
30
+ // answers pings automatically, so this works against any ICC version.
31
+ function startHeartbeat (ws) {
32
+ let alive = true
33
+ ws.on('pong', () => {
34
+ alive = true
35
+ })
36
+
37
+ const timer = setInterval(() => {
38
+ if (ws.readyState !== WebSocket.OPEN) {
39
+ return
40
+ }
41
+ if (!alive) {
42
+ app.log.warn('Updates websocket heartbeat timed out, terminating the connection')
43
+ ws.terminate()
44
+ return
45
+ }
46
+ alive = false
47
+ ws.ping()
48
+ }, heartbeatInterval)
49
+ timer.unref()
50
+
51
+ ws.on('close', () => {
52
+ clearInterval(timer)
53
+ })
54
+ }
55
+
21
56
  async function processMessage (data) {
22
57
  try {
23
58
  const message = JSON.parse(data)
@@ -115,7 +150,9 @@ async function updatePlugin (app) {
115
150
  try {
116
151
  const headers = await app.getAuthorizationHeaders()
117
152
 
118
- socket = new WebSocket(wsUrl, { headers })
153
+ // handshakeTimeout turns a hung upgrade into an 'error', so the retry
154
+ // loop is never stuck waiting for an 'open' that cannot arrive
155
+ socket = new WebSocket(wsUrl, { headers, handshakeTimeout: ackTimeout })
119
156
  await once(socket, 'open')
120
157
 
121
158
  app.log.info('Connected to updates websocket')
@@ -123,7 +160,20 @@ async function updatePlugin (app) {
123
160
  const subscribeMsg = JSON.stringify({ command: 'subscribe', topic: '/config' })
124
161
  socket.send(subscribeMsg)
125
162
 
126
- const command = await once(socket, 'message')
163
+ // A connection which closes cleanly between the subscribe and the ack
164
+ // emits no 'error', so an unbounded wait here would hang this loop
165
+ // forever: bound it and retry
166
+ const ackPromise = once(socket, 'message')
167
+ ackPromise.catch(() => {})
168
+ const command = await Promise.race([
169
+ ackPromise,
170
+ sleep(ackTimeout, kAckTimeout, { ref: false })
171
+ ])
172
+ if (command === kAckTimeout) {
173
+ socket.terminate()
174
+ throw new Error('Timed out waiting for the updates subscription acknowledgment')
175
+ }
176
+
127
177
  const message = JSON.parse(command[0])
128
178
  if (message?.command !== 'ack') {
129
179
  app.log.error({ message }, 'Subscription updates failed')
@@ -131,6 +181,8 @@ async function updatePlugin (app) {
131
181
  }
132
182
  app.log.info('Received subscription acknowledgment from updates websocket')
133
183
 
184
+ startHeartbeat(socket)
185
+
134
186
  // listen for subsequent messages
135
187
  socket.on('message', processMessage)
136
188
 
@@ -744,3 +744,63 @@ test('a timed-out worker is not queried again until it restarts', async (t) => {
744
744
  await waitForReports(3)
745
745
  equal(queriesToBlockedWorker(), 4)
746
746
  })
747
+
748
+ test('attaches the last observed worker ELU to the state report', async (t) => {
749
+ setUpEnvironment()
750
+
751
+ const { statesReports, url, waitForReports } = await startScalerMock(t)
752
+ const app = createMockApp({ scalerUrl: url })
753
+
754
+ await flamegraphsPlugin(app)
755
+ await app.setupFlamegraphs()
756
+ t.after(() => app.cleanupFlamegraphs())
757
+ await waitForReports(1)
758
+
759
+ // The runtime health events carry the same ELU the profiling gate uses
760
+ app.watt.runtime.emit('application:worker:health', {
761
+ id: 'service-1:0',
762
+ application: 'service-1',
763
+ currentHealth: { elu: 0.97, heapUsed: 1, heapTotal: 2 }
764
+ })
765
+
766
+ await app.reportProfilingStates()
767
+ await waitForReports(2)
768
+
769
+ const report = statesReports[statesReports.length - 1]
770
+ const sampled = report.states.filter((s) => s.workerId === 'service-1:0')
771
+ ok(sampled.length > 0)
772
+ ok(sampled.every((s) => s.lastELU === 0.97))
773
+
774
+ // Workers without a sample simply omit the field (older runtimes, quiet
775
+ // workers): the consumer must not require it
776
+ const unsampled = report.states.filter((s) => s.workerId === 'service-2:0')
777
+ ok(unsampled.every((s) => s.lastELU === undefined))
778
+ })
779
+
780
+ test('a stale ELU sample is not attached to the state report', async (t) => {
781
+ setUpEnvironment()
782
+
783
+ const { statesReports, url, waitForReports } = await startScalerMock(t)
784
+ const app = createMockApp({ scalerUrl: url })
785
+
786
+ await flamegraphsPlugin(app)
787
+ await app.setupFlamegraphs()
788
+ t.after(() => app.cleanupFlamegraphs())
789
+ await waitForReports(1)
790
+
791
+ app.watt.runtime.emit('application:worker:health', {
792
+ id: 'service-1:0',
793
+ application: 'service-1',
794
+ currentHealth: { elu: 0.97 }
795
+ })
796
+
797
+ // Jump past the sample max age
798
+ const realNow = Date.now()
799
+ t.mock.method(Date, 'now', () => realNow + 61 * 1000)
800
+
801
+ await app.reportProfilingStates()
802
+ await waitForReports(2)
803
+
804
+ const report = statesReports[statesReports.length - 1]
805
+ ok(report.states.every((s) => s.lastELU === undefined))
806
+ })
@@ -564,3 +564,72 @@ test('should reconnect to updates if connection closes', async (t) => {
564
564
  deepEqual(processedMessages, [testMessage1, testMessage2])
565
565
  wss2.close()
566
566
  })
567
+
568
+ test('terminates a half-open connection via heartbeat and reconnects', async (t) => {
569
+ setUpEnvironment()
570
+
571
+ // autoPong: false simulates a peer that no longer answers: the client's
572
+ // pings get no pong back, exactly like a half-open TCP connection
573
+ const wss = new WebSocketServer({ port: port + 20, autoPong: false })
574
+ t.after(async () => wss.close())
575
+
576
+ const ee = new EventEmitter()
577
+ let connections = 0
578
+ wss.on('connection', (ws) => {
579
+ connections++
580
+ if (connections >= 2) {
581
+ ee.emit('reconnected')
582
+ }
583
+ ws.on('message', (data) => {
584
+ const message = JSON.parse(data.toString())
585
+ if (message.command === 'subscribe' && message.topic === '/config') {
586
+ ws.send(JSON.stringify({ command: 'ack' }))
587
+ }
588
+ })
589
+ })
590
+
591
+ const app = createMockApp(port + 20)
592
+ app.env.PLT_UPDATES_HEARTBEAT_INTERVAL_SEC = 0.1
593
+ app.env.PLT_UPDATES_RECONNECT_INTERVAL_SEC = 0.05
594
+ t.after(() => app.closeUpdates())
595
+
596
+ await updatePlugin(app)
597
+ await app.connectToUpdates()
598
+
599
+ // First heartbeat miss terminates the connection, the reconnect loop
600
+ // opens a new one, which times out again, and so on
601
+ const reconnected = once(ee, 'reconnected')
602
+ await reconnected
603
+
604
+ equal(connections >= 2, true, 'Should reconnect after the heartbeat timeout')
605
+ })
606
+
607
+ test('retries when the subscription acknowledgment never arrives', async (t) => {
608
+ setUpEnvironment()
609
+
610
+ const wss = new WebSocketServer({ port: port + 21 })
611
+ t.after(async () => wss.close())
612
+
613
+ const ee = new EventEmitter()
614
+ let connections = 0
615
+ wss.on('connection', () => {
616
+ // Accept the connection and the subscribe message, never send the ack
617
+ connections++
618
+ if (connections >= 2) {
619
+ ee.emit('reconnected')
620
+ }
621
+ })
622
+
623
+ const app = createMockApp(port + 21)
624
+ app.env.PLT_UPDATES_ACK_TIMEOUT_SEC = 0.2
625
+ app.env.PLT_UPDATES_RECONNECT_INTERVAL_SEC = 0.05
626
+ t.after(() => app.closeUpdates())
627
+
628
+ await updatePlugin(app)
629
+ await app.connectToUpdates()
630
+
631
+ const reconnected = once(ee, 'reconnected')
632
+ await reconnected
633
+
634
+ equal(connections >= 2, true, 'Should retry after the ack timeout')
635
+ })