@platformatic/watt-extra 1.17.0-alpha.1 → 1.18.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/test.yml +2 -2
- package/package.json +3 -3
- package/plugins/env.js +3 -1
- package/plugins/update.js +54 -2
- package/test/update.test.js +69 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@platformatic/watt-extra",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.18.0-alpha.1",
|
|
4
4
|
"description": "The Platformatic runtime manager",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"scripts": {
|
|
@@ -43,13 +43,13 @@
|
|
|
43
43
|
"avvio": "^9.1.0",
|
|
44
44
|
"chalk": "^4.1.2",
|
|
45
45
|
"commist": "^3.2.0",
|
|
46
|
-
"env-schema": "^
|
|
46
|
+
"env-schema": "^8.0.0",
|
|
47
47
|
"help-me": "^5.0.0",
|
|
48
48
|
"minimist": "^1.2.8",
|
|
49
49
|
"pino": "^10.0.0",
|
|
50
50
|
"pino-pretty": "^13.0.0",
|
|
51
51
|
"semver": "^7.7.3",
|
|
52
|
-
"undici": "^
|
|
52
|
+
"undici": "^8.0.0",
|
|
53
53
|
"undici-cache-redis": "^1.0.0",
|
|
54
54
|
"undici-slicer-interceptor": "^0.4.1",
|
|
55
55
|
"undici-traffic-interceptor": "^0.1.4",
|
package/plugins/env.js
CHANGED
|
@@ -17,7 +17,7 @@ const schema = {
|
|
|
17
17
|
PLT_APP_INTERNAL_SUB_DOMAIN: { type: 'string', default: 'plt.local' },
|
|
18
18
|
PLT_DEFAULT_CACHE_TAGS_HEADER: { type: 'string', default: 'x-plt-cache-tags' },
|
|
19
19
|
PLT_CACHE_CONFIG: { type: 'string' },
|
|
20
|
-
PLT_DISABLE_FLAMEGRAPHS: { type: 'boolean', default:
|
|
20
|
+
PLT_DISABLE_FLAMEGRAPHS: { type: 'boolean', default: true },
|
|
21
21
|
PLT_FLAMEGRAPHS_INTERVAL_SEC: { type: 'number', default: 60 },
|
|
22
22
|
PLT_FLAMEGRAPHS_ELU_THRESHOLD: { type: 'number', default: 0.4 },
|
|
23
23
|
PLT_FLAMEGRAPHS_GRACE_PERIOD: { type: 'number', default: 3000 },
|
|
@@ -27,6 +27,8 @@ const schema = {
|
|
|
27
27
|
PLT_FLAMEGRAPHS_STATE_QUERY_TIMEOUT: { type: 'number', default: 5000 },
|
|
28
28
|
PLT_JWT_EXPIRATION_OFFSET_SEC: { type: 'number', default: 60 },
|
|
29
29
|
PLT_UPDATES_RECONNECT_INTERVAL_SEC: { type: 'number', default: 1 },
|
|
30
|
+
PLT_UPDATES_HEARTBEAT_INTERVAL_SEC: { type: 'number', default: 30 },
|
|
31
|
+
PLT_UPDATES_ACK_TIMEOUT_SEC: { type: 'number', default: 10 },
|
|
30
32
|
PLT_ELU_HEALTH_SIGNAL_THRESHOLD: { type: 'number', default: 0.7 },
|
|
31
33
|
PLT_HEAP_HEALTH_SIGNAL_THRESHOLD: { type: ['number', 'string'], default: '400MB' },
|
|
32
34
|
PLT_ALERTS_GRACE_PERIOD_SEC: { type: 'number', default: 30 },
|
package/plugins/update.js
CHANGED
|
@@ -15,9 +15,44 @@ function createWebSocketUrl (httpUrl, path, queryParams = {}) {
|
|
|
15
15
|
|
|
16
16
|
async function updatePlugin (app) {
|
|
17
17
|
const reconnectInterval = app.env.PLT_UPDATES_RECONNECT_INTERVAL_SEC * 1000
|
|
18
|
+
const heartbeatInterval = (app.env.PLT_UPDATES_HEARTBEAT_INTERVAL_SEC ?? 30) * 1000
|
|
19
|
+
const ackTimeout = (app.env.PLT_UPDATES_ACK_TIMEOUT_SEC ?? 10) * 1000
|
|
20
|
+
const kAckTimeout = Symbol('kAckTimeout')
|
|
18
21
|
|
|
19
22
|
let socket = null
|
|
20
23
|
|
|
24
|
+
// A half-open connection (e.g. the ICC endpoint died without a TCP reset
|
|
25
|
+
// reaching the pod) fires no 'close' event: without a heartbeat the pod
|
|
26
|
+
// believes it is connected forever and becomes unreachable for ICC
|
|
27
|
+
// commands, while its outgoing HTTP calls keep working. Ping the server
|
|
28
|
+
// and terminate the socket when the pong does not come back, so the close
|
|
29
|
+
// handler runs and the reconnect loop takes over. The server side of ws
|
|
30
|
+
// answers pings automatically, so this works against any ICC version.
|
|
31
|
+
function startHeartbeat (ws) {
|
|
32
|
+
let alive = true
|
|
33
|
+
ws.on('pong', () => {
|
|
34
|
+
alive = true
|
|
35
|
+
})
|
|
36
|
+
|
|
37
|
+
const timer = setInterval(() => {
|
|
38
|
+
if (ws.readyState !== WebSocket.OPEN) {
|
|
39
|
+
return
|
|
40
|
+
}
|
|
41
|
+
if (!alive) {
|
|
42
|
+
app.log.warn('Updates websocket heartbeat timed out, terminating the connection')
|
|
43
|
+
ws.terminate()
|
|
44
|
+
return
|
|
45
|
+
}
|
|
46
|
+
alive = false
|
|
47
|
+
ws.ping()
|
|
48
|
+
}, heartbeatInterval)
|
|
49
|
+
timer.unref()
|
|
50
|
+
|
|
51
|
+
ws.on('close', () => {
|
|
52
|
+
clearInterval(timer)
|
|
53
|
+
})
|
|
54
|
+
}
|
|
55
|
+
|
|
21
56
|
async function processMessage (data) {
|
|
22
57
|
try {
|
|
23
58
|
const message = JSON.parse(data)
|
|
@@ -115,7 +150,9 @@ async function updatePlugin (app) {
|
|
|
115
150
|
try {
|
|
116
151
|
const headers = await app.getAuthorizationHeaders()
|
|
117
152
|
|
|
118
|
-
|
|
153
|
+
// handshakeTimeout turns a hung upgrade into an 'error', so the retry
|
|
154
|
+
// loop is never stuck waiting for an 'open' that cannot arrive
|
|
155
|
+
socket = new WebSocket(wsUrl, { headers, handshakeTimeout: ackTimeout })
|
|
119
156
|
await once(socket, 'open')
|
|
120
157
|
|
|
121
158
|
app.log.info('Connected to updates websocket')
|
|
@@ -123,7 +160,20 @@ async function updatePlugin (app) {
|
|
|
123
160
|
const subscribeMsg = JSON.stringify({ command: 'subscribe', topic: '/config' })
|
|
124
161
|
socket.send(subscribeMsg)
|
|
125
162
|
|
|
126
|
-
|
|
163
|
+
// A connection which closes cleanly between the subscribe and the ack
|
|
164
|
+
// emits no 'error', so an unbounded wait here would hang this loop
|
|
165
|
+
// forever: bound it and retry
|
|
166
|
+
const ackPromise = once(socket, 'message')
|
|
167
|
+
ackPromise.catch(() => {})
|
|
168
|
+
const command = await Promise.race([
|
|
169
|
+
ackPromise,
|
|
170
|
+
sleep(ackTimeout, kAckTimeout, { ref: false })
|
|
171
|
+
])
|
|
172
|
+
if (command === kAckTimeout) {
|
|
173
|
+
socket.terminate()
|
|
174
|
+
throw new Error('Timed out waiting for the updates subscription acknowledgment')
|
|
175
|
+
}
|
|
176
|
+
|
|
127
177
|
const message = JSON.parse(command[0])
|
|
128
178
|
if (message?.command !== 'ack') {
|
|
129
179
|
app.log.error({ message }, 'Subscription updates failed')
|
|
@@ -131,6 +181,8 @@ async function updatePlugin (app) {
|
|
|
131
181
|
}
|
|
132
182
|
app.log.info('Received subscription acknowledgment from updates websocket')
|
|
133
183
|
|
|
184
|
+
startHeartbeat(socket)
|
|
185
|
+
|
|
134
186
|
// listen for subsequent messages
|
|
135
187
|
socket.on('message', processMessage)
|
|
136
188
|
|
package/test/update.test.js
CHANGED
|
@@ -564,3 +564,72 @@ test('should reconnect to updates if connection closes', async (t) => {
|
|
|
564
564
|
deepEqual(processedMessages, [testMessage1, testMessage2])
|
|
565
565
|
wss2.close()
|
|
566
566
|
})
|
|
567
|
+
|
|
568
|
+
test('terminates a half-open connection via heartbeat and reconnects', async (t) => {
|
|
569
|
+
setUpEnvironment()
|
|
570
|
+
|
|
571
|
+
// autoPong: false simulates a peer that no longer answers: the client's
|
|
572
|
+
// pings get no pong back, exactly like a half-open TCP connection
|
|
573
|
+
const wss = new WebSocketServer({ port: port + 20, autoPong: false })
|
|
574
|
+
t.after(async () => wss.close())
|
|
575
|
+
|
|
576
|
+
const ee = new EventEmitter()
|
|
577
|
+
let connections = 0
|
|
578
|
+
wss.on('connection', (ws) => {
|
|
579
|
+
connections++
|
|
580
|
+
if (connections >= 2) {
|
|
581
|
+
ee.emit('reconnected')
|
|
582
|
+
}
|
|
583
|
+
ws.on('message', (data) => {
|
|
584
|
+
const message = JSON.parse(data.toString())
|
|
585
|
+
if (message.command === 'subscribe' && message.topic === '/config') {
|
|
586
|
+
ws.send(JSON.stringify({ command: 'ack' }))
|
|
587
|
+
}
|
|
588
|
+
})
|
|
589
|
+
})
|
|
590
|
+
|
|
591
|
+
const app = createMockApp(port + 20)
|
|
592
|
+
app.env.PLT_UPDATES_HEARTBEAT_INTERVAL_SEC = 0.1
|
|
593
|
+
app.env.PLT_UPDATES_RECONNECT_INTERVAL_SEC = 0.05
|
|
594
|
+
t.after(() => app.closeUpdates())
|
|
595
|
+
|
|
596
|
+
await updatePlugin(app)
|
|
597
|
+
await app.connectToUpdates()
|
|
598
|
+
|
|
599
|
+
// First heartbeat miss terminates the connection, the reconnect loop
|
|
600
|
+
// opens a new one, which times out again, and so on
|
|
601
|
+
const reconnected = once(ee, 'reconnected')
|
|
602
|
+
await reconnected
|
|
603
|
+
|
|
604
|
+
equal(connections >= 2, true, 'Should reconnect after the heartbeat timeout')
|
|
605
|
+
})
|
|
606
|
+
|
|
607
|
+
test('retries when the subscription acknowledgment never arrives', async (t) => {
|
|
608
|
+
setUpEnvironment()
|
|
609
|
+
|
|
610
|
+
const wss = new WebSocketServer({ port: port + 21 })
|
|
611
|
+
t.after(async () => wss.close())
|
|
612
|
+
|
|
613
|
+
const ee = new EventEmitter()
|
|
614
|
+
let connections = 0
|
|
615
|
+
wss.on('connection', () => {
|
|
616
|
+
// Accept the connection and the subscribe message, never send the ack
|
|
617
|
+
connections++
|
|
618
|
+
if (connections >= 2) {
|
|
619
|
+
ee.emit('reconnected')
|
|
620
|
+
}
|
|
621
|
+
})
|
|
622
|
+
|
|
623
|
+
const app = createMockApp(port + 21)
|
|
624
|
+
app.env.PLT_UPDATES_ACK_TIMEOUT_SEC = 0.2
|
|
625
|
+
app.env.PLT_UPDATES_RECONNECT_INTERVAL_SEC = 0.05
|
|
626
|
+
t.after(() => app.closeUpdates())
|
|
627
|
+
|
|
628
|
+
await updatePlugin(app)
|
|
629
|
+
await app.connectToUpdates()
|
|
630
|
+
|
|
631
|
+
const reconnected = once(ee, 'reconnected')
|
|
632
|
+
await reconnected
|
|
633
|
+
|
|
634
|
+
equal(connections >= 2, true, 'Should retry after the ack timeout')
|
|
635
|
+
})
|