@platformatic/watt-extra 1.16.0-alpha.1 → 1.17.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/test.yml +1 -1
- package/lib/errors.js +7 -1
- package/lib/utils.js +1 -10
- package/lib/watt.js +17 -2
- package/package.json +12 -13
- package/plugins/env.js +2 -0
- package/plugins/flamegraphs.js +386 -59
- package/plugins/update.js +15 -0
- package/test/profiling-toggle.test.js +746 -0
- package/test/scheduler-coordination.test.js +7 -2
- package/test/scheduler.test.js +15 -11
package/lib/errors.js
CHANGED
|
@@ -19,6 +19,10 @@ const MetadataRuntimeNotStartedError = createError('PLT_METADATA_RUNTIME_NOT_STA
|
|
|
19
19
|
const StatusSendError = createError('PLT_STATUS_SEND_ERROR', 'Failed to send application status')
|
|
20
20
|
const StatusAppIdError = createError('PLT_STATUS_APPID_ERROR', 'Cannot send application status: no applicationId available')
|
|
21
21
|
|
|
22
|
+
// Profiling errors
|
|
23
|
+
const ProfilingRuntimeNotAvailableError = createError('PLT_PROFILING_RUNTIME_NOT_AVAILABLE_ERROR', 'Runtime not available, cannot toggle profiling')
|
|
24
|
+
const ProfilingStatesSendError = createError('PLT_PROFILING_STATES_SEND_ERROR', 'Failed to send profiling states: %s')
|
|
25
|
+
|
|
22
26
|
export {
|
|
23
27
|
CompliancyMetadataError,
|
|
24
28
|
CompliancyStatusError,
|
|
@@ -30,5 +34,7 @@ export {
|
|
|
30
34
|
MetadataAppIdError,
|
|
31
35
|
MetadataRuntimeNotStartedError,
|
|
32
36
|
StatusSendError,
|
|
33
|
-
StatusAppIdError
|
|
37
|
+
StatusAppIdError,
|
|
38
|
+
ProfilingRuntimeNotAvailableError,
|
|
39
|
+
ProfilingStatesSendError
|
|
34
40
|
}
|
package/lib/utils.js
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { access, rm, readdir } from 'node:fs/promises'
|
|
2
|
-
import { execa } from 'execa'
|
|
3
2
|
|
|
4
3
|
async function fileExists (path) {
|
|
5
4
|
try {
|
|
@@ -23,12 +22,4 @@ async function dirExists (dir) {
|
|
|
23
22
|
}
|
|
24
23
|
}
|
|
25
24
|
|
|
26
|
-
|
|
27
|
-
const child = execa('tar', ['-xf', tarballPath, '-C', dir])
|
|
28
|
-
child.stdout.pipe(process.stdout)
|
|
29
|
-
child.stderr.pipe(process.stderr)
|
|
30
|
-
|
|
31
|
-
await child
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
export { fileExists, dirExists, extractTarball }
|
|
25
|
+
export { fileExists, dirExists }
|
package/lib/watt.js
CHANGED
|
@@ -245,11 +245,26 @@ class Watt {
|
|
|
245
245
|
]
|
|
246
246
|
}
|
|
247
247
|
|
|
248
|
-
|
|
248
|
+
// Always load the pprof capture preload: PLT_DISABLE_FLAMEGRAPHS only
|
|
249
|
+
// controls whether profiling starts at boot, and ICC can activate it at
|
|
250
|
+
// runtime on this pod with the start-profiling command. The preload is
|
|
251
|
+
// passive until startProfiling is invoked.
|
|
252
|
+
// The module is resolved from watt-extra's own dependencies. If it is
|
|
253
|
+
// missing anyway (e.g. a stripped install), the pod boots without it:
|
|
254
|
+
// profiling is unavailable and the workers are reported as unsupported.
|
|
255
|
+
let pprofPath = null
|
|
256
|
+
try {
|
|
257
|
+
pprofPath = require.resolve('@platformatic/wattpm-pprof-capture')
|
|
258
|
+
} catch {
|
|
259
|
+
this.#logger.warn(
|
|
260
|
+
'@platformatic/wattpm-pprof-capture is not available, profiling is disabled on this pod'
|
|
261
|
+
)
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
if (pprofPath !== null) {
|
|
249
265
|
if (config.preload === undefined) {
|
|
250
266
|
config.preload = []
|
|
251
267
|
}
|
|
252
|
-
const pprofPath = require.resolve('@platformatic/wattpm-pprof-capture')
|
|
253
268
|
config.preload.push(pprofPath)
|
|
254
269
|
}
|
|
255
270
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@platformatic/watt-extra",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.17.0-alpha.1",
|
|
4
4
|
"description": "The Platformatic runtime manager",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"scripts": {
|
|
@@ -19,32 +19,31 @@
|
|
|
19
19
|
},
|
|
20
20
|
"devDependencies": {
|
|
21
21
|
"@fastify/websocket": "^11.1.0",
|
|
22
|
-
"@platformatic/composer": "^3.
|
|
23
|
-
"@platformatic/next": "^3.
|
|
24
|
-
"@platformatic/node": "^3.
|
|
25
|
-
"@platformatic/service": "^3.
|
|
22
|
+
"@platformatic/composer": "^3.68.0",
|
|
23
|
+
"@platformatic/next": "^3.68.0",
|
|
24
|
+
"@platformatic/node": "^3.68.0",
|
|
25
|
+
"@platformatic/service": "^3.68.0",
|
|
26
26
|
"atomic-sleep": "^1.0.0",
|
|
27
27
|
"borp": "^1.0.0",
|
|
28
|
-
"eslint": "
|
|
28
|
+
"eslint": "10",
|
|
29
29
|
"fastify": "^5.10.0",
|
|
30
|
-
"fastify-plugin": "^
|
|
31
|
-
"neostandard": "
|
|
30
|
+
"fastify-plugin": "^6.0.0",
|
|
31
|
+
"neostandard": "0.14.0-next.1",
|
|
32
32
|
"next": "^16.2.10",
|
|
33
|
-
"platformatic": "^3.
|
|
33
|
+
"platformatic": "^3.68.0",
|
|
34
34
|
"pprof-format": "^2.1.0",
|
|
35
35
|
"why-is-node-running": "^2.3.0"
|
|
36
36
|
},
|
|
37
37
|
"dependencies": {
|
|
38
38
|
"@datadog/pprof": "^5.9.0",
|
|
39
39
|
"@fastify/error": "^4.2.0",
|
|
40
|
-
"@platformatic/foundation": "^3.
|
|
41
|
-
"@platformatic/runtime": "^3.
|
|
42
|
-
"@platformatic/wattpm-pprof-capture": "^3.
|
|
40
|
+
"@platformatic/foundation": "^3.68.0",
|
|
41
|
+
"@platformatic/runtime": "^3.68.0",
|
|
42
|
+
"@platformatic/wattpm-pprof-capture": "^3.68.0",
|
|
43
43
|
"avvio": "^9.1.0",
|
|
44
44
|
"chalk": "^4.1.2",
|
|
45
45
|
"commist": "^3.2.0",
|
|
46
46
|
"env-schema": "^7.0.0",
|
|
47
|
-
"execa": "^9.6.0",
|
|
48
47
|
"help-me": "^5.0.0",
|
|
49
48
|
"minimist": "^1.2.8",
|
|
50
49
|
"pino": "^10.0.0",
|
package/plugins/env.js
CHANGED
|
@@ -23,6 +23,8 @@ const schema = {
|
|
|
23
23
|
PLT_FLAMEGRAPHS_GRACE_PERIOD: { type: 'number', default: 3000 },
|
|
24
24
|
PLT_FLAMEGRAPHS_ATTEMPT_TIMEOUT: { type: 'number', default: 10000 },
|
|
25
25
|
PLT_FLAMEGRAPHS_CACHE_CLEANUP_INTERVAL: { type: 'number', default: 120000 },
|
|
26
|
+
PLT_FLAMEGRAPHS_STATE_REPORT_INTERVAL: { type: 'number', default: 30000 },
|
|
27
|
+
PLT_FLAMEGRAPHS_STATE_QUERY_TIMEOUT: { type: 'number', default: 5000 },
|
|
26
28
|
PLT_JWT_EXPIRATION_OFFSET_SEC: { type: 'number', default: 60 },
|
|
27
29
|
PLT_UPDATES_RECONNECT_INTERVAL_SEC: { type: 'number', default: 1 },
|
|
28
30
|
PLT_ELU_HEALTH_SIGNAL_THRESHOLD: { type: 'number', default: 0.7 },
|
package/plugins/flamegraphs.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import { setTimeout as sleep } from 'node:timers/promises'
|
|
4
4
|
import { request } from 'undici'
|
|
5
|
+
import { ProfilingRuntimeNotAvailableError, ProfilingStatesSendError } from '../lib/errors.js'
|
|
5
6
|
|
|
6
7
|
async function flamegraphs (app, _opts) {
|
|
7
8
|
const isFlamegraphsDisabled = app.env.PLT_DISABLE_FLAMEGRAPHS
|
|
@@ -17,59 +18,108 @@ async function flamegraphs (app, _opts) {
|
|
|
17
18
|
const attemptTimeout = Math.min(parseInt(flamegraphsAttemptTimeout), durationMillis)
|
|
18
19
|
const maxAttempts = Math.ceil(durationMillis / attemptTimeout) + 1
|
|
19
20
|
const cacheCleanupInterval = parseInt(flamegraphsCacheCleanupInterval)
|
|
21
|
+
const stateReportInterval = parseInt(app.env.PLT_FLAMEGRAPHS_STATE_REPORT_INTERVAL ?? 30000)
|
|
22
|
+
const stateQueryTimeout = parseInt(app.env.PLT_FLAMEGRAPHS_STATE_QUERY_TIMEOUT ?? 5000)
|
|
23
|
+
|
|
24
|
+
const PROFILE_TYPES = ['cpu', 'heap']
|
|
25
|
+
const kStateQueryTimeout = Symbol('kStateQueryTimeout')
|
|
26
|
+
|
|
27
|
+
// Workers whose event loop did not answer a state query in time. The
|
|
28
|
+
// timeout only bounds the report: the underlying ITC request cannot be
|
|
29
|
+
// cancelled and stays pending in the runtime until the worker answers, so
|
|
30
|
+
// querying a blocked worker on every interval would leak one pending
|
|
31
|
+
// request per query. Further queries are suppressed with a bounded
|
|
32
|
+
// backoff; the suppression is cleared when the worker restarts or answers.
|
|
33
|
+
const STATE_QUERY_BACKOFF_MAX = 5 * 60 * 1000
|
|
34
|
+
const unresponsiveWorkers = new Map()
|
|
35
|
+
|
|
36
|
+
// Desired profiling state for this pod. It can be changed at runtime by the
|
|
37
|
+
// start-profiling/stop-profiling ICC commands and it gates the
|
|
38
|
+
// worker-started listener, so a deactivation survives worker restarts.
|
|
39
|
+
// It does not survive a pod restart: a new pod boots with the default.
|
|
40
|
+
// PLT_DISABLE_FLAMEGRAPHS only sets the boot default to off: ICC can still
|
|
41
|
+
// activate profiling on this pod at runtime.
|
|
42
|
+
const enabledTypes = new Set(isFlamegraphsDisabled ? [] : PROFILE_TYPES)
|
|
20
43
|
|
|
21
44
|
let workerStartedListener = null
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
45
|
+
let stateReportTimer = null
|
|
46
|
+
// Set to false when the ICC scaler does not expose the profiling states
|
|
47
|
+
// API (older ICC): reporting is disabled for the lifetime of the pod.
|
|
48
|
+
let stateReportingSupported = true
|
|
49
|
+
|
|
50
|
+
const startProfilingOnWorker = async (runtime, workerFullId, types, logContext = {}, { grace = true } = {}) => {
|
|
51
|
+
if (grace) {
|
|
52
|
+
await sleep(gracePeriod)
|
|
53
|
+
}
|
|
25
54
|
|
|
26
55
|
// Get application details to read service-level sourceMaps setting
|
|
27
56
|
const appDetails = await runtime.getApplicationDetails(workerFullId)
|
|
28
57
|
const sourceMaps = appDetails.sourceMaps ?? false
|
|
29
58
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
59
|
+
for (const type of types) {
|
|
60
|
+
try {
|
|
61
|
+
await runtime.sendCommandToApplication(
|
|
62
|
+
workerFullId,
|
|
63
|
+
'startProfiling',
|
|
64
|
+
{ durationMillis, eluThreshold, type, sourceMaps }
|
|
65
|
+
)
|
|
66
|
+
} catch (err) {
|
|
67
|
+
// A worker which is already being profiled is considered covered
|
|
68
|
+
if (err.code === 'PLT_PPROF_PROFILING_ALREADY_STARTED') {
|
|
69
|
+
continue
|
|
70
|
+
}
|
|
71
|
+
app.log.error({ err, type, ...logContext }, 'Failed to start profiling')
|
|
72
|
+
throw err
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
}
|
|
37
76
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
77
|
+
const stopProfilingOnWorker = async (runtime, workerFullId, types, logContext = {}) => {
|
|
78
|
+
for (const type of types) {
|
|
79
|
+
try {
|
|
80
|
+
await runtime.sendCommandToApplication(
|
|
81
|
+
workerFullId,
|
|
82
|
+
'stopProfiling',
|
|
83
|
+
{ type }
|
|
84
|
+
)
|
|
85
|
+
} catch (err) {
|
|
86
|
+
// A worker which is not being profiled is already in the target state
|
|
87
|
+
if (err.code === 'PLT_PPROF_PROFILING_NOT_STARTED') {
|
|
88
|
+
continue
|
|
89
|
+
}
|
|
90
|
+
app.log.warn({ err, type, ...logContext }, 'Failed to stop profiling')
|
|
91
|
+
throw err
|
|
92
|
+
}
|
|
47
93
|
}
|
|
48
94
|
}
|
|
49
95
|
|
|
50
96
|
app.setupFlamegraphs = async () => {
|
|
51
97
|
if (isFlamegraphsDisabled) {
|
|
52
|
-
app.log.info('PLT_DISABLE_FLAMEGRAPHS is set,
|
|
53
|
-
|
|
98
|
+
app.log.info('PLT_DISABLE_FLAMEGRAPHS is set, profiling starts deactivated on this pod')
|
|
99
|
+
} else {
|
|
100
|
+
app.log.info('Start profiling services')
|
|
54
101
|
}
|
|
55
102
|
|
|
56
|
-
app.log.info('Start profiling services')
|
|
57
|
-
|
|
58
103
|
const runtime = app.watt.runtime
|
|
59
|
-
const workers = await runtime.getWorkers()
|
|
60
104
|
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
105
|
+
// Respect the runtime toggle on an ICC recovery re-setup: an operator
|
|
106
|
+
// who deactivated profiling on this pod must not get it back silently.
|
|
107
|
+
if (enabledTypes.size > 0) {
|
|
108
|
+
const workers = await runtime.getWorkers()
|
|
109
|
+
|
|
110
|
+
const promises = []
|
|
111
|
+
for (const [workerFullId, workerInfo] of Object.entries(workers)) {
|
|
112
|
+
if (workerInfo.status === 'started') {
|
|
113
|
+
const promise = startProfilingOnWorker(runtime, workerFullId, [...enabledTypes], { workerFullId })
|
|
114
|
+
promises.push(promise)
|
|
115
|
+
}
|
|
66
116
|
}
|
|
67
|
-
}
|
|
68
117
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
118
|
+
const results = await Promise.allSettled(promises)
|
|
119
|
+
for (const result of results) {
|
|
120
|
+
if (result.status === 'rejected') {
|
|
121
|
+
app.log.error({ result }, 'Failed to start profiling')
|
|
122
|
+
}
|
|
73
123
|
}
|
|
74
124
|
}
|
|
75
125
|
|
|
@@ -80,20 +130,36 @@ async function flamegraphs (app, _opts) {
|
|
|
80
130
|
|
|
81
131
|
// Listen for new workers starting and start profiling on them
|
|
82
132
|
workerStartedListener = ({ application, worker }) => {
|
|
83
|
-
|
|
133
|
+
const workerFullId = [application, worker].join(':')
|
|
134
|
+
|
|
135
|
+
// A restarted worker has a fresh event loop: query its state again
|
|
136
|
+
unresponsiveWorkers.delete(workerFullId)
|
|
137
|
+
|
|
138
|
+
if (enabledTypes.size === 0) {
|
|
84
139
|
return
|
|
85
140
|
}
|
|
86
|
-
|
|
87
|
-
const workerFullId = [application, worker].join(':')
|
|
88
141
|
app.log.info({ application, worker }, 'Starting profiling on new worker')
|
|
89
142
|
|
|
90
|
-
startProfilingOnWorker(runtime, workerFullId, { application, worker }).catch(() => {
|
|
143
|
+
startProfilingOnWorker(runtime, workerFullId, [...enabledTypes], { application, worker }).catch(() => {
|
|
91
144
|
// Error already logged in startProfilingOnWorker
|
|
92
145
|
})
|
|
93
146
|
}
|
|
94
147
|
runtime.on('application:worker:started', workerStartedListener)
|
|
95
148
|
|
|
96
149
|
setInterval(cleanupFlamegraphsCache, cacheCleanupInterval).unref()
|
|
150
|
+
|
|
151
|
+
if (stateReportTimer === null) {
|
|
152
|
+
stateReportTimer = setInterval(() => {
|
|
153
|
+
reportProfilingStates().catch((err) => {
|
|
154
|
+
app.log.error({ err }, 'Failed to report profiling states')
|
|
155
|
+
})
|
|
156
|
+
}, stateReportInterval)
|
|
157
|
+
stateReportTimer.unref()
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
reportProfilingStates().catch((err) => {
|
|
161
|
+
app.log.error({ err }, 'Failed to report profiling states')
|
|
162
|
+
})
|
|
97
163
|
}
|
|
98
164
|
|
|
99
165
|
app.cleanupFlamegraphs = async () => {
|
|
@@ -102,29 +168,25 @@ async function flamegraphs (app, _opts) {
|
|
|
102
168
|
workerStartedListener = null
|
|
103
169
|
}
|
|
104
170
|
|
|
105
|
-
|
|
106
|
-
|
|
171
|
+
if (stateReportTimer) {
|
|
172
|
+
clearInterval(stateReportTimer)
|
|
173
|
+
stateReportTimer = null
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
// Explicitly stop all active profiling sessions to avoid memory
|
|
177
|
+
// corruption. Profiling may be active even when PLT_DISABLE_FLAMEGRAPHS
|
|
178
|
+
// is set (runtime activation), so always attempt the stop: it is
|
|
179
|
+
// idempotent.
|
|
180
|
+
if (app.watt?.runtime) {
|
|
107
181
|
try {
|
|
108
182
|
const workers = await app.watt.runtime.getWorkers()
|
|
109
183
|
const stopPromises = []
|
|
110
184
|
for (const workerFullId of Object.keys(workers)) {
|
|
111
185
|
// Stop both CPU and heap profiling on each worker
|
|
112
186
|
stopPromises.push(
|
|
113
|
-
app.watt.runtime
|
|
114
|
-
.catch(
|
|
115
|
-
//
|
|
116
|
-
if (err.code !== 'PLT_PPROF_PROFILING_NOT_STARTED') {
|
|
117
|
-
app.log.warn({ err, workerFullId }, 'Failed to stop CPU profiling')
|
|
118
|
-
}
|
|
119
|
-
})
|
|
120
|
-
)
|
|
121
|
-
stopPromises.push(
|
|
122
|
-
app.watt.runtime.sendCommandToApplication(workerFullId, 'stopProfiling', { type: 'heap' })
|
|
123
|
-
.catch(err => {
|
|
124
|
-
// Ignore errors if profiling wasn't running
|
|
125
|
-
if (err.code !== 'PLT_PPROF_PROFILING_NOT_STARTED') {
|
|
126
|
-
app.log.warn({ err, workerFullId }, 'Failed to stop heap profiling')
|
|
127
|
-
}
|
|
187
|
+
stopProfilingOnWorker(app.watt.runtime, workerFullId, PROFILE_TYPES, { workerFullId })
|
|
188
|
+
.catch(() => {
|
|
189
|
+
// Error already logged in stopProfilingOnWorker
|
|
128
190
|
})
|
|
129
191
|
)
|
|
130
192
|
}
|
|
@@ -137,16 +199,79 @@ async function flamegraphs (app, _opts) {
|
|
|
137
199
|
}
|
|
138
200
|
}
|
|
139
201
|
|
|
202
|
+
// Activates or deactivates continuous profiling on this pod at runtime,
|
|
203
|
+
// driven by the start-profiling/stop-profiling ICC commands. It also works
|
|
204
|
+
// on a pod started with PLT_DISABLE_FLAMEGRAPHS: the variable only sets
|
|
205
|
+
// the boot default, the commands override it for this pod's lifetime.
|
|
206
|
+
app.setProfilingEnabled = async (enabled, types = PROFILE_TYPES) => {
|
|
207
|
+
if (isFlamegraphsDisabled && enabled) {
|
|
208
|
+
app.log.info('Activating profiling on a pod started with PLT_DISABLE_FLAMEGRAPHS')
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
const runtime = app.watt?.runtime
|
|
212
|
+
if (!runtime) {
|
|
213
|
+
throw new ProfilingRuntimeNotAvailableError()
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
const validTypes = types.filter((type) => PROFILE_TYPES.includes(type))
|
|
217
|
+
for (const type of validTypes) {
|
|
218
|
+
if (enabled) {
|
|
219
|
+
enabledTypes.add(type)
|
|
220
|
+
} else {
|
|
221
|
+
enabledTypes.delete(type)
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
const workers = await runtime.getWorkers()
|
|
226
|
+
const promises = []
|
|
227
|
+
for (const [workerFullId, workerInfo] of Object.entries(workers)) {
|
|
228
|
+
if (workerInfo.status !== 'started') {
|
|
229
|
+
continue
|
|
230
|
+
}
|
|
231
|
+
const promise = enabled
|
|
232
|
+
? startProfilingOnWorker(runtime, workerFullId, validTypes, { workerFullId }, { grace: false })
|
|
233
|
+
: stopProfilingOnWorker(runtime, workerFullId, validTypes, { workerFullId })
|
|
234
|
+
promises.push(promise)
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
const results = await Promise.allSettled(promises)
|
|
238
|
+
let failures = 0
|
|
239
|
+
for (const result of results) {
|
|
240
|
+
if (result.status === 'rejected') {
|
|
241
|
+
failures++
|
|
242
|
+
app.log.error({ result, enabled }, 'Failed to toggle profiling on a worker')
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
app.log.info({ enabled, types: validTypes, failures }, 'Profiling toggled')
|
|
247
|
+
|
|
248
|
+
// Report the new state right away so the UI converges quickly. The
|
|
249
|
+
// report carries the real per-worker profiler state (isCapturing), so a
|
|
250
|
+
// worker which failed to toggle is reconciled by ICC from that, not from
|
|
251
|
+
// the enabledTypes intent.
|
|
252
|
+
reportProfilingStates().catch((err) => {
|
|
253
|
+
app.log.error({ err }, 'Failed to report profiling states')
|
|
254
|
+
})
|
|
255
|
+
|
|
256
|
+
return { success: failures === 0, enabled, types: [...enabledTypes], failures }
|
|
257
|
+
}
|
|
258
|
+
|
|
140
259
|
const profilesByWorkerId = {}
|
|
141
260
|
|
|
142
261
|
app.sendFlamegraphs = async (options = {}) => {
|
|
143
|
-
|
|
144
|
-
|
|
262
|
+
let { workerIds, alertId, profileType = 'cpu' } = options
|
|
263
|
+
|
|
264
|
+
// The runtime toggle is the single gate: profiling may be deactivated on
|
|
265
|
+
// a pod booted with it on, or activated on a pod booted with
|
|
266
|
+
// PLT_DISABLE_FLAMEGRAPHS
|
|
267
|
+
if (!enabledTypes.has(profileType)) {
|
|
268
|
+
app.log.warn(
|
|
269
|
+
{ profileType },
|
|
270
|
+
'Profiling is deactivated on this pod, cannot collect the profile'
|
|
271
|
+
)
|
|
145
272
|
return
|
|
146
273
|
}
|
|
147
274
|
|
|
148
|
-
let { workerIds, alertId, profileType = 'cpu' } = options
|
|
149
|
-
|
|
150
275
|
const scalerUrl = app.instanceConfig?.iccServices?.scaler?.url
|
|
151
276
|
if (!scalerUrl) {
|
|
152
277
|
app.log.error('No scaler URL found in ICC services, cannot send flamegraph')
|
|
@@ -388,6 +513,208 @@ async function flamegraphs (app, _opts) {
|
|
|
388
513
|
}
|
|
389
514
|
}
|
|
390
515
|
|
|
516
|
+
// Collects the real profiling state of every worker and reports it to the
|
|
517
|
+
// scaler together with the pod's desired state. The report expires in
|
|
518
|
+
// Valkey, so a pod which stops reporting naturally disappears from the UI.
|
|
519
|
+
//
|
|
520
|
+
// Concurrent invocations are coalesced: while a report is running, a new
|
|
521
|
+
// request queues at most one re-run after it, so a slow scaler or a
|
|
522
|
+
// blocked worker cannot pile up overlapping reports.
|
|
523
|
+
let reportInFlight = null
|
|
524
|
+
let reportQueued = false
|
|
525
|
+
|
|
526
|
+
app.reportProfilingStates = reportProfilingStates
|
|
527
|
+
function reportProfilingStates () {
|
|
528
|
+
if (reportInFlight) {
|
|
529
|
+
reportQueued = true
|
|
530
|
+
return reportInFlight
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
reportInFlight = collectAndSendProfilingStates().finally(() => {
|
|
534
|
+
reportInFlight = null
|
|
535
|
+
if (reportQueued) {
|
|
536
|
+
reportQueued = false
|
|
537
|
+
reportProfilingStates().catch((err) => {
|
|
538
|
+
app.log.error({ err }, 'Failed to report profiling states')
|
|
539
|
+
})
|
|
540
|
+
}
|
|
541
|
+
})
|
|
542
|
+
return reportInFlight
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
async function collectAndSendProfilingStates () {
|
|
546
|
+
if (!stateReportingSupported) {
|
|
547
|
+
return
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
const runtime = app.watt?.runtime
|
|
551
|
+
if (!runtime) {
|
|
552
|
+
return
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
const scalerUrl = app.instanceConfig?.iccServices?.scaler?.url
|
|
556
|
+
const applicationId = app.instanceConfig?.applicationId
|
|
557
|
+
if (!scalerUrl || !applicationId) {
|
|
558
|
+
return
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
const workers = await runtime.getWorkers()
|
|
562
|
+
|
|
563
|
+
// One blocked worker must not prevent the others from being reported:
|
|
564
|
+
// the queries run in parallel and each is bounded by a timeout
|
|
565
|
+
const queries = []
|
|
566
|
+
for (const [workerFullId, workerInfo] of Object.entries(workers)) {
|
|
567
|
+
if (workerInfo.status !== 'started') {
|
|
568
|
+
continue
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
const serviceId = workerFullId.split(':')[0]
|
|
572
|
+
|
|
573
|
+
// A suppressed worker is still part of the report: an explicit
|
|
574
|
+
// unresponsive entry keeps ICC from treating the remaining healthy
|
|
575
|
+
// workers as the whole pod
|
|
576
|
+
const suppression = unresponsiveWorkers.get(workerFullId)
|
|
577
|
+
if (suppression && Date.now() < suppression.skipUntil) {
|
|
578
|
+
app.log.debug({ workerFullId }, 'Skipping the state query of an unresponsive worker')
|
|
579
|
+
for (const type of PROFILE_TYPES) {
|
|
580
|
+
queries.push(Promise.resolve(unresponsiveState(workerFullId, serviceId, type)))
|
|
581
|
+
}
|
|
582
|
+
continue
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
for (const type of PROFILE_TYPES) {
|
|
586
|
+
queries.push(getWorkerProfilingState(runtime, workerFullId, serviceId, type))
|
|
587
|
+
}
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
const states = await Promise.all(queries)
|
|
591
|
+
|
|
592
|
+
await sendProfilingStates(scalerUrl, applicationId, states)
|
|
593
|
+
}
|
|
594
|
+
|
|
595
|
+
// A worker whose profiling state cannot be read still appears in the
|
|
596
|
+
// report: ICC must know the state is unknown rather than assume the
|
|
597
|
+
// reported workers are the whole pod
|
|
598
|
+
function unresponsiveState (workerFullId, serviceId, type) {
|
|
599
|
+
return {
|
|
600
|
+
serviceId,
|
|
601
|
+
workerId: workerFullId,
|
|
602
|
+
type,
|
|
603
|
+
enabled: enabledTypes.has(type),
|
|
604
|
+
unresponsive: true
|
|
605
|
+
}
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
// Queries one worker for its profiling state of one type. Reports the
|
|
609
|
+
// worker as unresponsive when it cannot answer: the query has no timeout
|
|
610
|
+
// of its own and hangs if the worker event loop is blocked, which is
|
|
611
|
+
// likely exactly when profiling is in use, so it is bounded here.
|
|
612
|
+
async function getWorkerProfilingState (runtime, workerFullId, serviceId, type) {
|
|
613
|
+
let state = null
|
|
614
|
+
try {
|
|
615
|
+
const query = runtime.sendCommandToApplication(
|
|
616
|
+
workerFullId,
|
|
617
|
+
'getProfilingState',
|
|
618
|
+
{ type }
|
|
619
|
+
)
|
|
620
|
+
// A settlement after the timeout must not surface as an unhandled
|
|
621
|
+
// rejection
|
|
622
|
+
query.catch(() => {})
|
|
623
|
+
|
|
624
|
+
state = await Promise.race([
|
|
625
|
+
query,
|
|
626
|
+
sleep(stateQueryTimeout, kStateQueryTimeout, { ref: false })
|
|
627
|
+
])
|
|
628
|
+
|
|
629
|
+
if (state === kStateQueryTimeout) {
|
|
630
|
+
// Both types of the same worker time out together: escalate the
|
|
631
|
+
// backoff only once per cycle
|
|
632
|
+
const existing = unresponsiveWorkers.get(workerFullId)
|
|
633
|
+
if (!existing || Date.now() >= existing.skipUntil) {
|
|
634
|
+
const backoff = Math.min(
|
|
635
|
+
existing ? existing.backoff * 2 : stateReportInterval,
|
|
636
|
+
STATE_QUERY_BACKOFF_MAX
|
|
637
|
+
)
|
|
638
|
+
unresponsiveWorkers.set(workerFullId, { skipUntil: Date.now() + backoff, backoff })
|
|
639
|
+
}
|
|
640
|
+
app.log.warn(
|
|
641
|
+
{ workerFullId, type, stateQueryTimeout },
|
|
642
|
+
'Timed out getting the profiling state from a worker, suppressing its state queries'
|
|
643
|
+
)
|
|
644
|
+
return unresponsiveState(workerFullId, serviceId, type)
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
unresponsiveWorkers.delete(workerFullId)
|
|
648
|
+
} catch (err) {
|
|
649
|
+
app.log.warn({ err, workerFullId, type }, 'Failed to get profiling state from worker')
|
|
650
|
+
return unresponsiveState(workerFullId, serviceId, type)
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
// A runtime without the pprof capture command answers with an
|
|
654
|
+
// empty object
|
|
655
|
+
if (!state || state.isCapturing === undefined) {
|
|
656
|
+
return {
|
|
657
|
+
serviceId,
|
|
658
|
+
workerId: workerFullId,
|
|
659
|
+
type,
|
|
660
|
+
enabled: enabledTypes.has(type),
|
|
661
|
+
unsupported: true
|
|
662
|
+
}
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
return {
|
|
666
|
+
serviceId,
|
|
667
|
+
workerId: workerFullId,
|
|
668
|
+
type,
|
|
669
|
+
enabled: enabledTypes.has(type),
|
|
670
|
+
...state
|
|
671
|
+
}
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
async function sendProfilingStates (scalerUrl, applicationId, states) {
|
|
675
|
+
const podId = app.instanceId
|
|
676
|
+
const url = `${scalerUrl}/flamegraphs/states`
|
|
677
|
+
|
|
678
|
+
const authHeaders = await app.getAuthorizationHeaders()
|
|
679
|
+
const { statusCode, body } = await request(url, {
|
|
680
|
+
method: 'POST',
|
|
681
|
+
headers: {
|
|
682
|
+
'Content-Type': 'application/json',
|
|
683
|
+
...authHeaders
|
|
684
|
+
},
|
|
685
|
+
body: JSON.stringify({
|
|
686
|
+
applicationId,
|
|
687
|
+
podId,
|
|
688
|
+
// Three report intervals, so one missed report does not blank the UI
|
|
689
|
+
expiresIn: stateReportInterval * 3,
|
|
690
|
+
states
|
|
691
|
+
})
|
|
692
|
+
})
|
|
693
|
+
|
|
694
|
+
if (statusCode === 404) {
|
|
695
|
+
// An ICC without the profiling states API: disable reporting for the
|
|
696
|
+
// lifetime of this pod
|
|
697
|
+
await body.dump()
|
|
698
|
+
stateReportingSupported = false
|
|
699
|
+
if (stateReportTimer) {
|
|
700
|
+
clearInterval(stateReportTimer)
|
|
701
|
+
stateReportTimer = null
|
|
702
|
+
}
|
|
703
|
+
app.log.warn(
|
|
704
|
+
'The scaler does not support profiling state reports.' +
|
|
705
|
+
' Please upgrade to the latest ICC version to use this feature.'
|
|
706
|
+
)
|
|
707
|
+
return
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
if (statusCode !== 200) {
|
|
711
|
+
const error = await body.text()
|
|
712
|
+
throw new ProfilingStatesSendError(error)
|
|
713
|
+
}
|
|
714
|
+
|
|
715
|
+
await body.dump()
|
|
716
|
+
}
|
|
717
|
+
|
|
391
718
|
function cleanupFlamegraphsCache () {
|
|
392
719
|
const now = Date.now()
|
|
393
720
|
|