@platformatic/watt-extra 1.17.0 → 1.18.0-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@platformatic/watt-extra",
3
- "version": "1.17.0",
3
+ "version": "1.18.0-alpha.1",
4
4
  "description": "The Platformatic runtime manager",
5
5
  "type": "module",
6
6
  "scripts": {
package/plugins/env.js CHANGED
@@ -27,6 +27,8 @@ const schema = {
27
27
  PLT_FLAMEGRAPHS_STATE_QUERY_TIMEOUT: { type: 'number', default: 5000 },
28
28
  PLT_JWT_EXPIRATION_OFFSET_SEC: { type: 'number', default: 60 },
29
29
  PLT_UPDATES_RECONNECT_INTERVAL_SEC: { type: 'number', default: 1 },
30
+ PLT_UPDATES_HEARTBEAT_INTERVAL_SEC: { type: 'number', default: 30 },
31
+ PLT_UPDATES_ACK_TIMEOUT_SEC: { type: 'number', default: 10 },
30
32
  PLT_ELU_HEALTH_SIGNAL_THRESHOLD: { type: 'number', default: 0.7 },
31
33
  PLT_HEAP_HEALTH_SIGNAL_THRESHOLD: { type: ['number', 'string'], default: '400MB' },
32
34
  PLT_ALERTS_GRACE_PERIOD_SEC: { type: 'number', default: 30 },
package/plugins/update.js CHANGED
@@ -15,9 +15,44 @@ function createWebSocketUrl (httpUrl, path, queryParams = {}) {
15
15
 
16
16
  async function updatePlugin (app) {
17
17
  const reconnectInterval = app.env.PLT_UPDATES_RECONNECT_INTERVAL_SEC * 1000
18
+ const heartbeatInterval = (app.env.PLT_UPDATES_HEARTBEAT_INTERVAL_SEC ?? 30) * 1000
19
+ const ackTimeout = (app.env.PLT_UPDATES_ACK_TIMEOUT_SEC ?? 10) * 1000
20
+ const kAckTimeout = Symbol('kAckTimeout')
18
21
 
19
22
  let socket = null
20
23
 
24
+ // A half-open connection (e.g. the ICC endpoint died without a TCP reset
25
+ // reaching the pod) fires no 'close' event: without a heartbeat the pod
26
+ // believes it is connected forever and becomes unreachable for ICC
27
+ // commands, while its outgoing HTTP calls keep working. Ping the server
28
+ // and terminate the socket when the pong does not come back, so the close
29
+ // handler runs and the reconnect loop takes over. The server side of ws
30
+ // answers pings automatically, so this works against any ICC version.
31
+ function startHeartbeat (ws) {
32
+ let alive = true
33
+ ws.on('pong', () => {
34
+ alive = true
35
+ })
36
+
37
+ const timer = setInterval(() => {
38
+ if (ws.readyState !== WebSocket.OPEN) {
39
+ return
40
+ }
41
+ if (!alive) {
42
+ app.log.warn('Updates websocket heartbeat timed out, terminating the connection')
43
+ ws.terminate()
44
+ return
45
+ }
46
+ alive = false
47
+ ws.ping()
48
+ }, heartbeatInterval)
49
+ timer.unref()
50
+
51
+ ws.on('close', () => {
52
+ clearInterval(timer)
53
+ })
54
+ }
55
+
21
56
  async function processMessage (data) {
22
57
  try {
23
58
  const message = JSON.parse(data)
@@ -115,7 +150,9 @@ async function updatePlugin (app) {
115
150
  try {
116
151
  const headers = await app.getAuthorizationHeaders()
117
152
 
118
- socket = new WebSocket(wsUrl, { headers })
153
+ // handshakeTimeout turns a hung upgrade into an 'error', so the retry
154
+ // loop is never stuck waiting for an 'open' that cannot arrive
155
+ socket = new WebSocket(wsUrl, { headers, handshakeTimeout: ackTimeout })
119
156
  await once(socket, 'open')
120
157
 
121
158
  app.log.info('Connected to updates websocket')
@@ -123,7 +160,20 @@ async function updatePlugin (app) {
123
160
  const subscribeMsg = JSON.stringify({ command: 'subscribe', topic: '/config' })
124
161
  socket.send(subscribeMsg)
125
162
 
126
- const command = await once(socket, 'message')
163
+ // A connection which closes cleanly between the subscribe and the ack
164
+ // emits no 'error', so an unbounded wait here would hang this loop
165
+ // forever: bound it and retry
166
+ const ackPromise = once(socket, 'message')
167
+ ackPromise.catch(() => {})
168
+ const command = await Promise.race([
169
+ ackPromise,
170
+ sleep(ackTimeout, kAckTimeout, { ref: false })
171
+ ])
172
+ if (command === kAckTimeout) {
173
+ socket.terminate()
174
+ throw new Error('Timed out waiting for the updates subscription acknowledgment')
175
+ }
176
+
127
177
  const message = JSON.parse(command[0])
128
178
  if (message?.command !== 'ack') {
129
179
  app.log.error({ message }, 'Subscription updates failed')
@@ -131,6 +181,8 @@ async function updatePlugin (app) {
131
181
  }
132
182
  app.log.info('Received subscription acknowledgment from updates websocket')
133
183
 
184
+ startHeartbeat(socket)
185
+
134
186
  // listen for subsequent messages
135
187
  socket.on('message', processMessage)
136
188
 
@@ -564,3 +564,72 @@ test('should reconnect to updates if connection closes', async (t) => {
564
564
  deepEqual(processedMessages, [testMessage1, testMessage2])
565
565
  wss2.close()
566
566
  })
567
+
568
+ test('terminates a half-open connection via heartbeat and reconnects', async (t) => {
569
+ setUpEnvironment()
570
+
571
+ // autoPong: false simulates a peer that no longer answers: the client's
572
+ // pings get no pong back, exactly like a half-open TCP connection
573
+ const wss = new WebSocketServer({ port: port + 20, autoPong: false })
574
+ t.after(async () => wss.close())
575
+
576
+ const ee = new EventEmitter()
577
+ let connections = 0
578
+ wss.on('connection', (ws) => {
579
+ connections++
580
+ if (connections >= 2) {
581
+ ee.emit('reconnected')
582
+ }
583
+ ws.on('message', (data) => {
584
+ const message = JSON.parse(data.toString())
585
+ if (message.command === 'subscribe' && message.topic === '/config') {
586
+ ws.send(JSON.stringify({ command: 'ack' }))
587
+ }
588
+ })
589
+ })
590
+
591
+ const app = createMockApp(port + 20)
592
+ app.env.PLT_UPDATES_HEARTBEAT_INTERVAL_SEC = 0.1
593
+ app.env.PLT_UPDATES_RECONNECT_INTERVAL_SEC = 0.05
594
+ t.after(() => app.closeUpdates())
595
+
596
+ await updatePlugin(app)
597
+ await app.connectToUpdates()
598
+
599
+ // First heartbeat miss terminates the connection, the reconnect loop
600
+ // opens a new one, which times out again, and so on
601
+ const reconnected = once(ee, 'reconnected')
602
+ await reconnected
603
+
604
+ equal(connections >= 2, true, 'Should reconnect after the heartbeat timeout')
605
+ })
606
+
607
+ test('retries when the subscription acknowledgment never arrives', async (t) => {
608
+ setUpEnvironment()
609
+
610
+ const wss = new WebSocketServer({ port: port + 21 })
611
+ t.after(async () => wss.close())
612
+
613
+ const ee = new EventEmitter()
614
+ let connections = 0
615
+ wss.on('connection', () => {
616
+ // Accept the connection and the subscribe message, never send the ack
617
+ connections++
618
+ if (connections >= 2) {
619
+ ee.emit('reconnected')
620
+ }
621
+ })
622
+
623
+ const app = createMockApp(port + 21)
624
+ app.env.PLT_UPDATES_ACK_TIMEOUT_SEC = 0.2
625
+ app.env.PLT_UPDATES_RECONNECT_INTERVAL_SEC = 0.05
626
+ t.after(() => app.closeUpdates())
627
+
628
+ await updatePlugin(app)
629
+ await app.connectToUpdates()
630
+
631
+ const reconnected = once(ee, 'reconnected')
632
+ await reconnected
633
+
634
+ equal(connections >= 2, true, 'Should retry after the ack timeout')
635
+ })