@platformatic/watt-extra 1.16.0-alpha.1 → 1.17.0-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -36,7 +36,7 @@ jobs:
36
36
 
37
37
  steps:
38
38
  - uses: actions/checkout@v7
39
- - uses: pnpm/action-setup@v4.2.0
39
+ - uses: pnpm/action-setup@v4.4.0
40
40
  with:
41
41
  version: 10
42
42
 
package/lib/errors.js CHANGED
@@ -19,6 +19,10 @@ const MetadataRuntimeNotStartedError = createError('PLT_METADATA_RUNTIME_NOT_STA
19
19
  const StatusSendError = createError('PLT_STATUS_SEND_ERROR', 'Failed to send application status')
20
20
  const StatusAppIdError = createError('PLT_STATUS_APPID_ERROR', 'Cannot send application status: no applicationId available')
21
21
 
22
+ // Profiling errors
23
+ const ProfilingRuntimeNotAvailableError = createError('PLT_PROFILING_RUNTIME_NOT_AVAILABLE_ERROR', 'Runtime not available, cannot toggle profiling')
24
+ const ProfilingStatesSendError = createError('PLT_PROFILING_STATES_SEND_ERROR', 'Failed to send profiling states: %s')
25
+
22
26
  export {
23
27
  CompliancyMetadataError,
24
28
  CompliancyStatusError,
@@ -30,5 +34,7 @@ export {
30
34
  MetadataAppIdError,
31
35
  MetadataRuntimeNotStartedError,
32
36
  StatusSendError,
33
- StatusAppIdError
37
+ StatusAppIdError,
38
+ ProfilingRuntimeNotAvailableError,
39
+ ProfilingStatesSendError
34
40
  }
package/lib/utils.js CHANGED
@@ -1,5 +1,4 @@
1
1
  import { access, rm, readdir } from 'node:fs/promises'
2
- import { execa } from 'execa'
3
2
 
4
3
  async function fileExists (path) {
5
4
  try {
@@ -23,12 +22,4 @@ async function dirExists (dir) {
23
22
  }
24
23
  }
25
24
 
26
- async function extractTarball (tarballPath, dir) {
27
- const child = execa('tar', ['-xf', tarballPath, '-C', dir])
28
- child.stdout.pipe(process.stdout)
29
- child.stderr.pipe(process.stderr)
30
-
31
- await child
32
- }
33
-
34
- export { fileExists, dirExists, extractTarball }
25
+ export { fileExists, dirExists }
package/lib/watt.js CHANGED
@@ -245,11 +245,26 @@ class Watt {
245
245
  ]
246
246
  }
247
247
 
248
- if (this.#env.PLT_DISABLE_FLAMEGRAPHS !== true) {
248
+ // Always load the pprof capture preload: PLT_DISABLE_FLAMEGRAPHS only
249
+ // controls whether profiling starts at boot, and ICC can activate it at
250
+ // runtime on this pod with the start-profiling command. The preload is
251
+ // passive until startProfiling is invoked.
252
+ // The module is resolved from watt-extra's own dependencies. If it is
253
+ // missing anyway (e.g. a stripped install), the pod boots without it:
254
+ // profiling is unavailable and the workers are reported as unsupported.
255
+ let pprofPath = null
256
+ try {
257
+ pprofPath = require.resolve('@platformatic/wattpm-pprof-capture')
258
+ } catch {
259
+ this.#logger.warn(
260
+ '@platformatic/wattpm-pprof-capture is not available, profiling is disabled on this pod'
261
+ )
262
+ }
263
+
264
+ if (pprofPath !== null) {
249
265
  if (config.preload === undefined) {
250
266
  config.preload = []
251
267
  }
252
- const pprofPath = require.resolve('@platformatic/wattpm-pprof-capture')
253
268
  config.preload.push(pprofPath)
254
269
  }
255
270
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@platformatic/watt-extra",
3
- "version": "1.16.0-alpha.1",
3
+ "version": "1.17.0-alpha.1",
4
4
  "description": "The Platformatic runtime manager",
5
5
  "type": "module",
6
6
  "scripts": {
@@ -19,32 +19,31 @@
19
19
  },
20
20
  "devDependencies": {
21
21
  "@fastify/websocket": "^11.1.0",
22
- "@platformatic/composer": "^3.62.2",
23
- "@platformatic/next": "^3.62.2",
24
- "@platformatic/node": "^3.62.2",
25
- "@platformatic/service": "^3.62.2",
22
+ "@platformatic/composer": "^3.68.0",
23
+ "@platformatic/next": "^3.68.0",
24
+ "@platformatic/node": "^3.68.0",
25
+ "@platformatic/service": "^3.68.0",
26
26
  "atomic-sleep": "^1.0.0",
27
27
  "borp": "^1.0.0",
28
- "eslint": "9",
28
+ "eslint": "10",
29
29
  "fastify": "^5.10.0",
30
- "fastify-plugin": "^5.0.1",
31
- "neostandard": "^0.12.0",
30
+ "fastify-plugin": "^6.0.0",
31
+ "neostandard": "0.14.0-next.1",
32
32
  "next": "^16.2.10",
33
- "platformatic": "^3.62.2",
33
+ "platformatic": "^3.68.0",
34
34
  "pprof-format": "^2.1.0",
35
35
  "why-is-node-running": "^2.3.0"
36
36
  },
37
37
  "dependencies": {
38
38
  "@datadog/pprof": "^5.9.0",
39
39
  "@fastify/error": "^4.2.0",
40
- "@platformatic/foundation": "^3.62.2",
41
- "@platformatic/runtime": "^3.62.2",
42
- "@platformatic/wattpm-pprof-capture": "^3.62.2",
40
+ "@platformatic/foundation": "^3.68.0",
41
+ "@platformatic/runtime": "^3.68.0",
42
+ "@platformatic/wattpm-pprof-capture": "^3.68.0",
43
43
  "avvio": "^9.1.0",
44
44
  "chalk": "^4.1.2",
45
45
  "commist": "^3.2.0",
46
46
  "env-schema": "^7.0.0",
47
- "execa": "^9.6.0",
48
47
  "help-me": "^5.0.0",
49
48
  "minimist": "^1.2.8",
50
49
  "pino": "^10.0.0",
package/plugins/env.js CHANGED
@@ -23,6 +23,8 @@ const schema = {
23
23
  PLT_FLAMEGRAPHS_GRACE_PERIOD: { type: 'number', default: 3000 },
24
24
  PLT_FLAMEGRAPHS_ATTEMPT_TIMEOUT: { type: 'number', default: 10000 },
25
25
  PLT_FLAMEGRAPHS_CACHE_CLEANUP_INTERVAL: { type: 'number', default: 120000 },
26
+ PLT_FLAMEGRAPHS_STATE_REPORT_INTERVAL: { type: 'number', default: 30000 },
27
+ PLT_FLAMEGRAPHS_STATE_QUERY_TIMEOUT: { type: 'number', default: 5000 },
26
28
  PLT_JWT_EXPIRATION_OFFSET_SEC: { type: 'number', default: 60 },
27
29
  PLT_UPDATES_RECONNECT_INTERVAL_SEC: { type: 'number', default: 1 },
28
30
  PLT_ELU_HEALTH_SIGNAL_THRESHOLD: { type: 'number', default: 0.7 },
@@ -2,6 +2,7 @@
2
2
 
3
3
  import { setTimeout as sleep } from 'node:timers/promises'
4
4
  import { request } from 'undici'
5
+ import { ProfilingRuntimeNotAvailableError, ProfilingStatesSendError } from '../lib/errors.js'
5
6
 
6
7
  async function flamegraphs (app, _opts) {
7
8
  const isFlamegraphsDisabled = app.env.PLT_DISABLE_FLAMEGRAPHS
@@ -17,59 +18,108 @@ async function flamegraphs (app, _opts) {
17
18
  const attemptTimeout = Math.min(parseInt(flamegraphsAttemptTimeout), durationMillis)
18
19
  const maxAttempts = Math.ceil(durationMillis / attemptTimeout) + 1
19
20
  const cacheCleanupInterval = parseInt(flamegraphsCacheCleanupInterval)
21
+ const stateReportInterval = parseInt(app.env.PLT_FLAMEGRAPHS_STATE_REPORT_INTERVAL ?? 30000)
22
+ const stateQueryTimeout = parseInt(app.env.PLT_FLAMEGRAPHS_STATE_QUERY_TIMEOUT ?? 5000)
23
+
24
+ const PROFILE_TYPES = ['cpu', 'heap']
25
+ const kStateQueryTimeout = Symbol('kStateQueryTimeout')
26
+
27
+ // Workers whose event loop did not answer a state query in time. The
28
+ // timeout only bounds the report: the underlying ITC request cannot be
29
+ // cancelled and stays pending in the runtime until the worker answers, so
30
+ // querying a blocked worker on every interval would leak one pending
31
+ // request per query. Further queries are suppressed with a bounded
32
+ // backoff; the suppression is cleared when the worker restarts or answers.
33
+ const STATE_QUERY_BACKOFF_MAX = 5 * 60 * 1000
34
+ const unresponsiveWorkers = new Map()
35
+
36
+ // Desired profiling state for this pod. It can be changed at runtime by the
37
+ // start-profiling/stop-profiling ICC commands and it gates the
38
+ // worker-started listener, so a deactivation survives worker restarts.
39
+ // It does not survive a pod restart: a new pod boots with the default.
40
+ // PLT_DISABLE_FLAMEGRAPHS only sets the boot default to off: ICC can still
41
+ // activate profiling on this pod at runtime.
42
+ const enabledTypes = new Set(isFlamegraphsDisabled ? [] : PROFILE_TYPES)
20
43
 
21
44
  let workerStartedListener = null
22
-
23
- const startProfilingOnWorker = async (runtime, workerFullId, logContext = {}) => {
24
- await sleep(gracePeriod)
45
+ let stateReportTimer = null
46
+ // Set to false when the ICC scaler does not expose the profiling states
47
+ // API (older ICC): reporting is disabled for the lifetime of the pod.
48
+ let stateReportingSupported = true
49
+
50
+ const startProfilingOnWorker = async (runtime, workerFullId, types, logContext = {}, { grace = true } = {}) => {
51
+ if (grace) {
52
+ await sleep(gracePeriod)
53
+ }
25
54
 
26
55
  // Get application details to read service-level sourceMaps setting
27
56
  const appDetails = await runtime.getApplicationDetails(workerFullId)
28
57
  const sourceMaps = appDetails.sourceMaps ?? false
29
58
 
30
- try {
31
- // Start CPU profiling
32
- await runtime.sendCommandToApplication(
33
- workerFullId,
34
- 'startProfiling',
35
- { durationMillis, eluThreshold, type: 'cpu', sourceMaps }
36
- )
59
+ for (const type of types) {
60
+ try {
61
+ await runtime.sendCommandToApplication(
62
+ workerFullId,
63
+ 'startProfiling',
64
+ { durationMillis, eluThreshold, type, sourceMaps }
65
+ )
66
+ } catch (err) {
67
+ // A worker which is already being profiled is considered covered
68
+ if (err.code === 'PLT_PPROF_PROFILING_ALREADY_STARTED') {
69
+ continue
70
+ }
71
+ app.log.error({ err, type, ...logContext }, 'Failed to start profiling')
72
+ throw err
73
+ }
74
+ }
75
+ }
37
76
 
38
- // Start HEAP profiling
39
- await runtime.sendCommandToApplication(
40
- workerFullId,
41
- 'startProfiling',
42
- { durationMillis, eluThreshold, type: 'heap', sourceMaps }
43
- )
44
- } catch (err) {
45
- app.log.error({ err, ...logContext }, 'Failed to start profiling')
46
- throw err
77
+ const stopProfilingOnWorker = async (runtime, workerFullId, types, logContext = {}) => {
78
+ for (const type of types) {
79
+ try {
80
+ await runtime.sendCommandToApplication(
81
+ workerFullId,
82
+ 'stopProfiling',
83
+ { type }
84
+ )
85
+ } catch (err) {
86
+ // A worker which is not being profiled is already in the target state
87
+ if (err.code === 'PLT_PPROF_PROFILING_NOT_STARTED') {
88
+ continue
89
+ }
90
+ app.log.warn({ err, type, ...logContext }, 'Failed to stop profiling')
91
+ throw err
92
+ }
47
93
  }
48
94
  }
49
95
 
50
96
  app.setupFlamegraphs = async () => {
51
97
  if (isFlamegraphsDisabled) {
52
- app.log.info('PLT_DISABLE_FLAMEGRAPHS is set, skipping profiling')
53
- return
98
+ app.log.info('PLT_DISABLE_FLAMEGRAPHS is set, profiling starts deactivated on this pod')
99
+ } else {
100
+ app.log.info('Start profiling services')
54
101
  }
55
102
 
56
- app.log.info('Start profiling services')
57
-
58
103
  const runtime = app.watt.runtime
59
- const workers = await runtime.getWorkers()
60
104
 
61
- const promises = []
62
- for (const [workerFullId, workerInfo] of Object.entries(workers)) {
63
- if (workerInfo.status === 'started') {
64
- const promise = startProfilingOnWorker(runtime, workerFullId, { workerFullId })
65
- promises.push(promise)
105
+ // Respect the runtime toggle on an ICC recovery re-setup: an operator
106
+ // who deactivated profiling on this pod must not get it back silently.
107
+ if (enabledTypes.size > 0) {
108
+ const workers = await runtime.getWorkers()
109
+
110
+ const promises = []
111
+ for (const [workerFullId, workerInfo] of Object.entries(workers)) {
112
+ if (workerInfo.status === 'started') {
113
+ const promise = startProfilingOnWorker(runtime, workerFullId, [...enabledTypes], { workerFullId })
114
+ promises.push(promise)
115
+ }
66
116
  }
67
- }
68
117
 
69
- const results = await Promise.allSettled(promises)
70
- for (const result of results) {
71
- if (result.status === 'rejected') {
72
- app.log.error({ result }, 'Failed to start profiling')
118
+ const results = await Promise.allSettled(promises)
119
+ for (const result of results) {
120
+ if (result.status === 'rejected') {
121
+ app.log.error({ result }, 'Failed to start profiling')
122
+ }
73
123
  }
74
124
  }
75
125
 
@@ -80,20 +130,36 @@ async function flamegraphs (app, _opts) {
80
130
 
81
131
  // Listen for new workers starting and start profiling on them
82
132
  workerStartedListener = ({ application, worker }) => {
83
- if (isFlamegraphsDisabled) {
133
+ const workerFullId = [application, worker].join(':')
134
+
135
+ // A restarted worker has a fresh event loop: query its state again
136
+ unresponsiveWorkers.delete(workerFullId)
137
+
138
+ if (enabledTypes.size === 0) {
84
139
  return
85
140
  }
86
-
87
- const workerFullId = [application, worker].join(':')
88
141
  app.log.info({ application, worker }, 'Starting profiling on new worker')
89
142
 
90
- startProfilingOnWorker(runtime, workerFullId, { application, worker }).catch(() => {
143
+ startProfilingOnWorker(runtime, workerFullId, [...enabledTypes], { application, worker }).catch(() => {
91
144
  // Error already logged in startProfilingOnWorker
92
145
  })
93
146
  }
94
147
  runtime.on('application:worker:started', workerStartedListener)
95
148
 
96
149
  setInterval(cleanupFlamegraphsCache, cacheCleanupInterval).unref()
150
+
151
+ if (stateReportTimer === null) {
152
+ stateReportTimer = setInterval(() => {
153
+ reportProfilingStates().catch((err) => {
154
+ app.log.error({ err }, 'Failed to report profiling states')
155
+ })
156
+ }, stateReportInterval)
157
+ stateReportTimer.unref()
158
+ }
159
+
160
+ reportProfilingStates().catch((err) => {
161
+ app.log.error({ err }, 'Failed to report profiling states')
162
+ })
97
163
  }
98
164
 
99
165
  app.cleanupFlamegraphs = async () => {
@@ -102,29 +168,25 @@ async function flamegraphs (app, _opts) {
102
168
  workerStartedListener = null
103
169
  }
104
170
 
105
- // Explicitly stop all active profiling sessions to avoid memory corruption
106
- if (!isFlamegraphsDisabled && app.watt?.runtime) {
171
+ if (stateReportTimer) {
172
+ clearInterval(stateReportTimer)
173
+ stateReportTimer = null
174
+ }
175
+
176
+ // Explicitly stop all active profiling sessions to avoid memory
177
+ // corruption. Profiling may be active even when PLT_DISABLE_FLAMEGRAPHS
178
+ // is set (runtime activation), so always attempt the stop: it is
179
+ // idempotent.
180
+ if (app.watt?.runtime) {
107
181
  try {
108
182
  const workers = await app.watt.runtime.getWorkers()
109
183
  const stopPromises = []
110
184
  for (const workerFullId of Object.keys(workers)) {
111
185
  // Stop both CPU and heap profiling on each worker
112
186
  stopPromises.push(
113
- app.watt.runtime.sendCommandToApplication(workerFullId, 'stopProfiling', { type: 'cpu' })
114
- .catch(err => {
115
- // Ignore errors if profiling wasn't running
116
- if (err.code !== 'PLT_PPROF_PROFILING_NOT_STARTED') {
117
- app.log.warn({ err, workerFullId }, 'Failed to stop CPU profiling')
118
- }
119
- })
120
- )
121
- stopPromises.push(
122
- app.watt.runtime.sendCommandToApplication(workerFullId, 'stopProfiling', { type: 'heap' })
123
- .catch(err => {
124
- // Ignore errors if profiling wasn't running
125
- if (err.code !== 'PLT_PPROF_PROFILING_NOT_STARTED') {
126
- app.log.warn({ err, workerFullId }, 'Failed to stop heap profiling')
127
- }
187
+ stopProfilingOnWorker(app.watt.runtime, workerFullId, PROFILE_TYPES, { workerFullId })
188
+ .catch(() => {
189
+ // Error already logged in stopProfilingOnWorker
128
190
  })
129
191
  )
130
192
  }
@@ -137,16 +199,79 @@ async function flamegraphs (app, _opts) {
137
199
  }
138
200
  }
139
201
 
202
+ // Activates or deactivates continuous profiling on this pod at runtime,
203
+ // driven by the start-profiling/stop-profiling ICC commands. It also works
204
+ // on a pod started with PLT_DISABLE_FLAMEGRAPHS: the variable only sets
205
+ // the boot default, the commands override it for this pod's lifetime.
206
+ app.setProfilingEnabled = async (enabled, types = PROFILE_TYPES) => {
207
+ if (isFlamegraphsDisabled && enabled) {
208
+ app.log.info('Activating profiling on a pod started with PLT_DISABLE_FLAMEGRAPHS')
209
+ }
210
+
211
+ const runtime = app.watt?.runtime
212
+ if (!runtime) {
213
+ throw new ProfilingRuntimeNotAvailableError()
214
+ }
215
+
216
+ const validTypes = types.filter((type) => PROFILE_TYPES.includes(type))
217
+ for (const type of validTypes) {
218
+ if (enabled) {
219
+ enabledTypes.add(type)
220
+ } else {
221
+ enabledTypes.delete(type)
222
+ }
223
+ }
224
+
225
+ const workers = await runtime.getWorkers()
226
+ const promises = []
227
+ for (const [workerFullId, workerInfo] of Object.entries(workers)) {
228
+ if (workerInfo.status !== 'started') {
229
+ continue
230
+ }
231
+ const promise = enabled
232
+ ? startProfilingOnWorker(runtime, workerFullId, validTypes, { workerFullId }, { grace: false })
233
+ : stopProfilingOnWorker(runtime, workerFullId, validTypes, { workerFullId })
234
+ promises.push(promise)
235
+ }
236
+
237
+ const results = await Promise.allSettled(promises)
238
+ let failures = 0
239
+ for (const result of results) {
240
+ if (result.status === 'rejected') {
241
+ failures++
242
+ app.log.error({ result, enabled }, 'Failed to toggle profiling on a worker')
243
+ }
244
+ }
245
+
246
+ app.log.info({ enabled, types: validTypes, failures }, 'Profiling toggled')
247
+
248
+ // Report the new state right away so the UI converges quickly. The
249
+ // report carries the real per-worker profiler state (isCapturing), so a
250
+ // worker which failed to toggle is reconciled by ICC from that, not from
251
+ // the enabledTypes intent.
252
+ reportProfilingStates().catch((err) => {
253
+ app.log.error({ err }, 'Failed to report profiling states')
254
+ })
255
+
256
+ return { success: failures === 0, enabled, types: [...enabledTypes], failures }
257
+ }
258
+
140
259
  const profilesByWorkerId = {}
141
260
 
142
261
  app.sendFlamegraphs = async (options = {}) => {
143
- if (isFlamegraphsDisabled) {
144
- app.log.info('PLT_DISABLE_FLAMEGRAPHS is set, flamegraphs are disabled')
262
+ let { workerIds, alertId, profileType = 'cpu' } = options
263
+
264
+ // The runtime toggle is the single gate: profiling may be deactivated on
265
+ // a pod booted with it on, or activated on a pod booted with
266
+ // PLT_DISABLE_FLAMEGRAPHS
267
+ if (!enabledTypes.has(profileType)) {
268
+ app.log.warn(
269
+ { profileType },
270
+ 'Profiling is deactivated on this pod, cannot collect the profile'
271
+ )
145
272
  return
146
273
  }
147
274
 
148
- let { workerIds, alertId, profileType = 'cpu' } = options
149
-
150
275
  const scalerUrl = app.instanceConfig?.iccServices?.scaler?.url
151
276
  if (!scalerUrl) {
152
277
  app.log.error('No scaler URL found in ICC services, cannot send flamegraph')
@@ -388,6 +513,208 @@ async function flamegraphs (app, _opts) {
388
513
  }
389
514
  }
390
515
 
516
+ // Collects the real profiling state of every worker and reports it to the
517
+ // scaler together with the pod's desired state. The report expires in
518
+ // Valkey, so a pod which stops reporting naturally disappears from the UI.
519
+ //
520
+ // Concurrent invocations are coalesced: while a report is running, a new
521
+ // request queues at most one re-run after it, so a slow scaler or a
522
+ // blocked worker cannot pile up overlapping reports.
523
+ let reportInFlight = null
524
+ let reportQueued = false
525
+
526
+ app.reportProfilingStates = reportProfilingStates
527
+ function reportProfilingStates () {
528
+ if (reportInFlight) {
529
+ reportQueued = true
530
+ return reportInFlight
531
+ }
532
+
533
+ reportInFlight = collectAndSendProfilingStates().finally(() => {
534
+ reportInFlight = null
535
+ if (reportQueued) {
536
+ reportQueued = false
537
+ reportProfilingStates().catch((err) => {
538
+ app.log.error({ err }, 'Failed to report profiling states')
539
+ })
540
+ }
541
+ })
542
+ return reportInFlight
543
+ }
544
+
545
+ async function collectAndSendProfilingStates () {
546
+ if (!stateReportingSupported) {
547
+ return
548
+ }
549
+
550
+ const runtime = app.watt?.runtime
551
+ if (!runtime) {
552
+ return
553
+ }
554
+
555
+ const scalerUrl = app.instanceConfig?.iccServices?.scaler?.url
556
+ const applicationId = app.instanceConfig?.applicationId
557
+ if (!scalerUrl || !applicationId) {
558
+ return
559
+ }
560
+
561
+ const workers = await runtime.getWorkers()
562
+
563
+ // One blocked worker must not prevent the others from being reported:
564
+ // the queries run in parallel and each is bounded by a timeout
565
+ const queries = []
566
+ for (const [workerFullId, workerInfo] of Object.entries(workers)) {
567
+ if (workerInfo.status !== 'started') {
568
+ continue
569
+ }
570
+
571
+ const serviceId = workerFullId.split(':')[0]
572
+
573
+ // A suppressed worker is still part of the report: an explicit
574
+ // unresponsive entry keeps ICC from treating the remaining healthy
575
+ // workers as the whole pod
576
+ const suppression = unresponsiveWorkers.get(workerFullId)
577
+ if (suppression && Date.now() < suppression.skipUntil) {
578
+ app.log.debug({ workerFullId }, 'Skipping the state query of an unresponsive worker')
579
+ for (const type of PROFILE_TYPES) {
580
+ queries.push(Promise.resolve(unresponsiveState(workerFullId, serviceId, type)))
581
+ }
582
+ continue
583
+ }
584
+
585
+ for (const type of PROFILE_TYPES) {
586
+ queries.push(getWorkerProfilingState(runtime, workerFullId, serviceId, type))
587
+ }
588
+ }
589
+
590
+ const states = await Promise.all(queries)
591
+
592
+ await sendProfilingStates(scalerUrl, applicationId, states)
593
+ }
594
+
595
+ // A worker whose profiling state cannot be read still appears in the
596
+ // report: ICC must know the state is unknown rather than assume the
597
+ // reported workers are the whole pod
598
+ function unresponsiveState (workerFullId, serviceId, type) {
599
+ return {
600
+ serviceId,
601
+ workerId: workerFullId,
602
+ type,
603
+ enabled: enabledTypes.has(type),
604
+ unresponsive: true
605
+ }
606
+ }
607
+
608
+ // Queries one worker for its profiling state of one type. Reports the
609
+ // worker as unresponsive when it cannot answer: the query has no timeout
610
+ // of its own and hangs if the worker event loop is blocked, which is
611
+ // likely exactly when profiling is in use, so it is bounded here.
612
+ async function getWorkerProfilingState (runtime, workerFullId, serviceId, type) {
613
+ let state = null
614
+ try {
615
+ const query = runtime.sendCommandToApplication(
616
+ workerFullId,
617
+ 'getProfilingState',
618
+ { type }
619
+ )
620
+ // A settlement after the timeout must not surface as an unhandled
621
+ // rejection
622
+ query.catch(() => {})
623
+
624
+ state = await Promise.race([
625
+ query,
626
+ sleep(stateQueryTimeout, kStateQueryTimeout, { ref: false })
627
+ ])
628
+
629
+ if (state === kStateQueryTimeout) {
630
+ // Both types of the same worker time out together: escalate the
631
+ // backoff only once per cycle
632
+ const existing = unresponsiveWorkers.get(workerFullId)
633
+ if (!existing || Date.now() >= existing.skipUntil) {
634
+ const backoff = Math.min(
635
+ existing ? existing.backoff * 2 : stateReportInterval,
636
+ STATE_QUERY_BACKOFF_MAX
637
+ )
638
+ unresponsiveWorkers.set(workerFullId, { skipUntil: Date.now() + backoff, backoff })
639
+ }
640
+ app.log.warn(
641
+ { workerFullId, type, stateQueryTimeout },
642
+ 'Timed out getting the profiling state from a worker, suppressing its state queries'
643
+ )
644
+ return unresponsiveState(workerFullId, serviceId, type)
645
+ }
646
+
647
+ unresponsiveWorkers.delete(workerFullId)
648
+ } catch (err) {
649
+ app.log.warn({ err, workerFullId, type }, 'Failed to get profiling state from worker')
650
+ return unresponsiveState(workerFullId, serviceId, type)
651
+ }
652
+
653
+ // A runtime without the pprof capture command answers with an
654
+ // empty object
655
+ if (!state || state.isCapturing === undefined) {
656
+ return {
657
+ serviceId,
658
+ workerId: workerFullId,
659
+ type,
660
+ enabled: enabledTypes.has(type),
661
+ unsupported: true
662
+ }
663
+ }
664
+
665
+ return {
666
+ serviceId,
667
+ workerId: workerFullId,
668
+ type,
669
+ enabled: enabledTypes.has(type),
670
+ ...state
671
+ }
672
+ }
673
+
674
+ async function sendProfilingStates (scalerUrl, applicationId, states) {
675
+ const podId = app.instanceId
676
+ const url = `${scalerUrl}/flamegraphs/states`
677
+
678
+ const authHeaders = await app.getAuthorizationHeaders()
679
+ const { statusCode, body } = await request(url, {
680
+ method: 'POST',
681
+ headers: {
682
+ 'Content-Type': 'application/json',
683
+ ...authHeaders
684
+ },
685
+ body: JSON.stringify({
686
+ applicationId,
687
+ podId,
688
+ // Three report intervals, so one missed report does not blank the UI
689
+ expiresIn: stateReportInterval * 3,
690
+ states
691
+ })
692
+ })
693
+
694
+ if (statusCode === 404) {
695
+ // An ICC without the profiling states API: disable reporting for the
696
+ // lifetime of this pod
697
+ await body.dump()
698
+ stateReportingSupported = false
699
+ if (stateReportTimer) {
700
+ clearInterval(stateReportTimer)
701
+ stateReportTimer = null
702
+ }
703
+ app.log.warn(
704
+ 'The scaler does not support profiling state reports.' +
705
+ ' Please upgrade to the latest ICC version to use this feature.'
706
+ )
707
+ return
708
+ }
709
+
710
+ if (statusCode !== 200) {
711
+ const error = await body.text()
712
+ throw new ProfilingStatesSendError(error)
713
+ }
714
+
715
+ await body.dump()
716
+ }
717
+
391
718
  function cleanupFlamegraphsCache () {
392
719
  const now = Date.now()
393
720