@platformatic/watt-extra 1.2.1-alpha.4 → 1.3.0

package/.claude/settings.local.json

@@ -0,0 +1,9 @@
+{
+  "permissions": {
+    "allow": [
+      "Read(//work/workspaces/workspace-platformatic/platformatic/**)"
+    ],
+    "deny": [],
+    "ask": []
+  }
+}

package/.github/workflows/test.yml

@@ -41,7 +41,7 @@ jobs:
           version: 10
 
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v5
+        uses: actions/setup-node@v6
         with:
           node-version: ${{ matrix.node-version }}
           # https://github.com/actions/setup-node/blob/main/docs/advanced-usage.md#use-private-packages

package/lib/watt.js

@@ -63,16 +63,6 @@ class Watt {
       )
       throw err
     }
-
-    const { eventLoopUtilization } = require('node:perf_hooks').performance
-    // Print eventloop utilization every second
-    let elu1 = eventLoopUtilization()
-    setInterval(() => {
-      const elu2 = eventLoopUtilization()
-      const current = eventLoopUtilization(elu1)
-      console.log('Watt-extra event loop utilization:', current)
-      elu1 = elu2
-    }, 1000)
   }
 
   async close () {
@@ -161,25 +151,22 @@ class Watt {
     config.server = {
       ...serverConfig,
       hostname: this.#env.PLT_APP_HOSTNAME || serverConfig.hostname,
-      port: this.#env.PLT_APP_PORT || serverConfig.port
+      port: this.#env.PLT_APP_PORT || serverConfig.port,
     }
 
     config.hotReload = false
     config.restartOnError = 1000
-    config.metrics = {
-      enabled: false
-    }
     config.metrics = {
       server: 'hide',
       defaultMetrics: {
-        enabled: true
+        enabled: true,
       },
       hostname: this.#env.PLT_APP_HOSTNAME || '0.0.0.0',
       port: this.#env.PLT_METRICS_PORT || 9090,
       labels: {
         serviceId: 'main',
         applicationId: this.#instanceConfig?.applicationId,
-        instanceId: this.#instanceId
+        instanceId: this.#instanceId,
       },
       applicationLabel: this.#instanceConfig?.applicationMetricsLabel ?? 'serviceId'
     }
@@ -193,7 +180,7 @@ class Watt {
     }
 
     this.#configureUndici(config)
-    config.managementApi = false
+    config.managementApi = true
   }
 
   #getUndiciConfig () {
@@ -244,20 +231,20 @@ class Watt {
       ),
       options: {
         labels: {
-          applicationId: this.#instanceConfig.applicationId
+          applicationId: this.#instanceConfig.applicationId,
         },
         bloomFilter: {
           size: 100000,
-          errorRate: 0.01
+          errorRate: 0.01,
         },
         maxResponseSize: 5 * 1024 * 1024, // 5MB
         trafficInspectorOptions: {
           url: trafficInspectorOrigin,
           pathSendBody: join(trafficInspectorPath, '/requests'),
-          pathSendMeta: join(trafficInspectorPath, '/requests/hash')
+          pathSendMeta: join(trafficInspectorPath, '/requests/hash'),
         },
-        matchingDomains: [this.#env.PLT_APP_INTERNAL_SUB_DOMAIN]
-      }
+        matchingDomains: [this.#env.PLT_APP_INTERNAL_SUB_DOMAIN],
+      },
     }
   }
 
@@ -268,9 +255,9 @@ class Watt {
       rules: [
         {
           routeToMatch: 'http://plt.slicer.default/',
-          headers: {}
-        }
-      ]
+          headers: {},
+        },
+      ],
     }
 
     // This is the cache config from ICC
@@ -322,7 +309,7 @@ class Watt {
 
     return {
       module: require.resolve('undici-slicer-interceptor'),
-      options: cacheConfig
+      options: cacheConfig,
     }
   }
 
@@ -354,7 +341,7 @@ class Watt {
       applicationName: `${this.#applicationName}`,
       skip: [
         { method: 'GET', path: '/documentation' },
-        { method: 'GET', path: '/documentation/json' }
+        { method: 'GET', path: '/documentation/json' },
       ],
       exporter: {
         type: 'otlp',
@@ -362,14 +349,14 @@ class Watt {
           url:
             this.#instanceConfig?.iccServices?.riskEngine?.url + '/v1/traces',
           headers: {
-            'x-platformatic-application-id': this.#instanceConfig?.applicationId
+            'x-platformatic-application-id': this.#instanceConfig?.applicationId,
           },
           keepAlive: true,
           httpAgentOptions: {
-            rejectUnauthorized: false
-          }
-        }
-      }
+            rejectUnauthorized: false,
+          },
+        },
+      },
     }
   }
 
@@ -388,16 +375,16 @@ class Watt {
       ...config.httpCache,
       cacheTagsHeader,
       store: require.resolve('undici-cache-redis'),
-      clientOpts: httpCache
+      clientOpts: httpCache,
     }
   }
 
   #configureHealth (config) {
     config.health = {
-      // ...config.health,
-      enabled: false
-      // interval: 1000,
-      // maxUnhealthyChecks: 30
+      ...config.health,
+      enabled: true,
+      interval: 1000,
+      maxUnhealthyChecks: 30,
     }
   }
 
@@ -407,7 +394,7 @@ class Watt {
     if (config.scheduler) {
       config.scheduler = config.scheduler.map((scheduler) => ({
         ...scheduler,
-        enabled: false
+        enabled: false,
       }))
     }
   }
@@ -426,7 +413,7 @@ class Watt {
         [
           '@platformatic/service',
           '@platformatic/composer',
-          '@platformatic/db'
+          '@platformatic/db',
         ].includes(app.type)
       ) {
         await this.#configurePlatformaticServices(runtime, app)
@@ -470,8 +457,8 @@ class Watt {
             adapter: 'valkey',
             url: `valkey://${username}:${password}@${host}:${port}`,
             prefix: keyPrefix,
-            maxTTL: 604800 // 86400 * 7
-          }
+            maxTTL: 604800, // 86400 * 7
+          },
         })
       }
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platformatic/watt-extra",
-  "version": "1.2.1-alpha.4",
+  "version": "1.3.0",
   "description": "The Platformatic runtime manager",
   "type": "module",
   "scripts": {
@@ -23,7 +23,7 @@
     "@platformatic/next": "^3.8.0",
     "@platformatic/node": "^3.8.0",
     "@platformatic/service": "^3.8.0",
-    "borp": "^0.20.0",
+    "borp": "^0.21.0",
     "eslint": "9",
     "fastify": "^5.4.0",
     "fastify-plugin": "^5.0.1",

package/plugins/auth.js

@@ -48,7 +48,7 @@ async function authPlugin (app) {
   }
 
   const getAuthorizationHeader = async (headers = {}) => {
-    if (isTokenExpired(app.token, offset)) {
+    if (app.token && isTokenExpired(app.token, offset)) {
       app.log.info('JWT token expired, reloading')
       app.token = await loadToken()
 
@@ -74,11 +74,11 @@ async function authPlugin (app) {
 
   app.token = await loadToken()
 
-  await setInterval(async () => {
+  setInterval(async () => {
     // Check if token is expired to propagate it to the runtime
     // via the shared context
     await getAuthorizationHeader()
-  }, offset * 1000 / 2).unref()
+  }, offset ? offset * 1000 / 2 : 30000).unref()
 
   // We cannot change the global dispatcher because it's shared with the runtime main thread.
   const wattDispatcher = new Agent()

package/plugins/env.js

@@ -19,6 +19,8 @@ const schema = {
     PLT_CACHE_CONFIG: { type: 'string' },
     PLT_DISABLE_FLAMEGRAPHS: { type: 'boolean', default: false },
     PLT_FLAMEGRAPHS_INTERVAL_SEC: { type: 'number', default: 60 },
+    PLT_FLAMEGRAPHS_ELU_THRESHOLD: { type: 'number', default: 0.4 },
+    PLT_FLAMEGRAPHS_GRACE_PERIOD: { type: 'number', default: 3000 },
     PLT_JWT_EXPIRATION_OFFSET_SEC: { type: 'number', default: 60 },
     PLT_UPDATES_RECONNECT_INTERVAL_SEC: { type: 'number', default: 1 }
   }

package/plugins/flamegraphs.js

@@ -1,12 +1,17 @@
 'use strict'
 
+import { setTimeout as sleep } from 'node:timers/promises'
 import { request } from 'undici'
 
 async function flamegraphs (app, _opts) {
   const isFlamegraphsDisabled = app.env.PLT_DISABLE_FLAMEGRAPHS
   const flamegraphsIntervalSec = app.env.PLT_FLAMEGRAPHS_INTERVAL_SEC
+  const flamegraphsELUThreshold = app.env.PLT_FLAMEGRAPHS_ELU_THRESHOLD
+  const flamegraphsGracePeriod = app.env.PLT_FLAMEGRAPHS_GRACE_PERIOD
 
   const durationMillis = parseInt(flamegraphsIntervalSec) * 1000
+  const eluThreshold = parseInt(flamegraphsELUThreshold)
+  const gracePeriod = parseInt(flamegraphsGracePeriod)
 
   app.setupFlamegraphs = async () => {
     if (isFlamegraphsDisabled) {
@@ -16,6 +21,8 @@ async function flamegraphs (app, _opts) {
 
     app.log.info('Start profiling services')
 
+    await sleep(gracePeriod)
+
     const runtime = app.watt.runtime
     const { applications } = await runtime.getApplications()
 
@@ -24,7 +31,7 @@ async function flamegraphs (app, _opts) {
       const promise = runtime.sendCommandToApplication(
         application.id,
         'startProfiling',
-        { durationMillis }
+        { durationMillis, eluThreshold }
       )
       promises.push(promise)
     }
@@ -43,7 +50,7 @@ async function flamegraphs (app, _opts) {
       return
     }
 
-    let { serviceIds, alertId } = options
+    let { serviceIds, alertId, profileType = 'cpu' } = options
 
     const scalerUrl = app.instanceConfig?.iccServices?.scaler?.url
     if (!scalerUrl) {
@@ -71,7 +78,12 @@ async function flamegraphs (app, _opts) {
 
         const url = `${scalerUrl}/pods/${podId}/services/${serviceId}/flamegraph`
 
-        app.log.info({ serviceId, podId }, 'Sending flamegraph')
+        app.log.info({ serviceId, podId, profileType }, 'Sending flamegraph')
+
+        const query = { profileType }
+        if (alertId) {
+          query.alertId = alertId
+        }
 
         const { statusCode, body } = await request(url, {
           method: 'POST',
@@ -79,7 +91,7 @@ async function flamegraphs (app, _opts) {
             'Content-Type': 'application/octet-stream',
             ...authHeaders
           },
-          query: alertId ? { alertId } : {},
+          query,
           body: profile
         })
 
@@ -89,7 +101,13 @@ async function flamegraphs (app, _opts) {
           throw new Error(`Failed to send flamegraph: ${error}`)
         }
       } catch (err) {
-        app.log.warn({ err, serviceId, podId }, 'Failed to send flamegraph from service')
+        if (err.code === 'PLT_PPROF_NO_PROFILE_AVAILABLE') {
+          app.log.info({ serviceId, podId }, 'No profile available for the service')
+        } else if (err.code === 'PLT_PPROF_NOT_ENOUGH_ELU') {
+          app.log.info({ serviceId, podId }, 'ELU low, CPU profiling not active')
+        } else {
+          app.log.warn({ err, serviceId, podId }, 'Failed to send flamegraph from service')
+        }
       }
     })
 

package/plugins/update.js

@@ -23,7 +23,14 @@ async function updatePlugin (app) {
       // Handle trigger-flamegraph command from ICC
       if (command === 'trigger-flamegraph') {
         app.log.info({ command }, 'Received trigger-flamegraph command from ICC')
-        await app.sendFlamegraphs()
+        await app.sendFlamegraphs({ profileType: 'cpu' })
+        return
+      }
+
+      // Handle trigger-heapprofile command from ICC
+      if (command === 'trigger-heapprofile') {
+        app.log.info({ command }, 'Received trigger-heapprofile command from ICC')
+        await app.sendFlamegraphs({ profileType: 'heap' })
         return
       }
 

package/test/auth.test.js

@@ -126,3 +126,47 @@ test('auth plugin reloads expired token', async (t) => {
   const reloadLogMessage = logMessages.find(msg => msg === 'JWT token expired, reloading')
   equal(!!reloadLogMessage, true, 'Should log message about token reload')
 })
+
+test('auth plugin does not reload when token is undefined', async (t) => {
+  const originalEnv = { ...process.env }
+
+  t.after(() => {
+    process.env = originalEnv
+  })
+
+  delete process.env.PLT_TEST_TOKEN
+
+  const logMessages = []
+  const app = {
+    log: {
+      info: (msg) => {
+        if (typeof msg === 'string') {
+          logMessages.push(msg)
+        }
+      },
+      warn: () => {},
+      error: () => {}
+    }
+  }
+
+  const server = fastify()
+  server.get('/', async (request) => {
+    return { headers: request.headers }
+  })
+  await server.listen({ port: 0 })
+  const url = `http://localhost:${server.server.address().port}`
+
+  t.after(async () => server.close())
+
+  await authPlugin(app)
+
+  equal(app.token, undefined, 'Token should be undefined when not available')
+
+  const response = await request(url, { dispatcher: app.dispatcher })
+  const responseBody = await response.body.json()
+
+  equal(response.statusCode, 200)
+  equal(responseBody.headers.authorization, 'Bearer undefined')
+  const reloadLogMessage = logMessages.find(msg => msg === 'JWT token expired, reloading')
+  equal(reloadLogMessage, undefined, 'Should not attempt to reload undefined token')
+})

package/test/trigger-flamegraphs.test.js

@@ -39,9 +39,9 @@ function createMockApp (port, includeScalerUrl = true) {
   const mockWatt = {
     runtime: {
       getApplications: () => ({
-        applications: [{ id: 'service-1' }, { id: 'service-2' }],
-      }),
-    },
+        applications: [{ id: 'service-1' }, { id: 'service-2' }]
+      })
+    }
   }
 
   const app = {
@@ -49,10 +49,10 @@ function createMockApp (port, includeScalerUrl = true) {
       info: () => {},
       error: () => {},
       warn: () => {},
-      debug: () => {},
+      debug: () => {}
     },
     instanceConfig: {
-      applicationId: 'test-application-id',
+      applicationId: 'test-application-id'
     },
     instanceId: 'test-pod-123',
     getAuthorizationHeader: async () => {
@@ -63,16 +63,18 @@ function createMockApp (port, includeScalerUrl = true) {
       PLT_APP_DIR: '/path/to/app',
       PLT_ICC_URL: `http://localhost:${port}`,
       PLT_DISABLE_FLAMEGRAPHS: false,
-      PLT_FLAMEGRAPHS_INTERVAL_SEC: 1
+      PLT_FLAMEGRAPHS_INTERVAL_SEC: 1,
+      PLT_FLAMEGRAPHS_ELU_THRESHOLD: 0,
+      PLT_FLAMEGRAPHS_GRACE_PERIOD: 0
     },
-    watt: mockWatt,
+    watt: mockWatt
   }
 
   if (includeScalerUrl) {
     app.instanceConfig.iccServices = {
       scaler: {
-        url: `http://localhost:${port}/scaler`,
-      },
+        url: `http://localhost:${port}/scaler`
+      }
     }
   }
 
@@ -111,7 +113,7 @@ test('should handle trigger-flamegraph command and upload flamegraphs from servi
       if (getFlamegraphReqs.length === 2) {
         uploadResolve()
       }
-      return { success: true }
+      return new Uint8Array([1, 2, 3, 4, 5])
     }
     return { success: false }
   }
@@ -125,7 +127,7 @@ test('should handle trigger-flamegraph command and upload flamegraphs from servi
   await waitForClientSubscription
 
   const triggerFlamegraphMessage = {
-    command: 'trigger-flamegraph',
+    command: 'trigger-flamegraph'
   }
 
   getWs().send(JSON.stringify(triggerFlamegraphMessage))
@@ -169,7 +171,7 @@ test('should handle trigger-flamegraph when no runtime is available', async (t)
   await waitForClientSubscription
 
   const triggerFlamegraphMessage = {
-    command: 'trigger-flamegraph',
+    command: 'trigger-flamegraph'
   }
 
   getWs().send(JSON.stringify(triggerFlamegraphMessage))
@@ -215,7 +217,7 @@ test('should handle trigger-flamegraph when flamegraph upload fails', async (t)
   await waitForClientSubscription
 
   const triggerFlamegraphMessage = {
-    command: 'trigger-flamegraph',
+    command: 'trigger-flamegraph'
   }
 
   getWs().send(JSON.stringify(triggerFlamegraphMessage))
@@ -224,3 +226,207 @@ test('should handle trigger-flamegraph when flamegraph upload fails', async (t)
 
   await app.closeUpdates()
 })
+
+test('should handle trigger-heapprofile command and upload heap profiles from services', async (t) => {
+  setUpEnvironment()
+
+  const receivedMessages = []
+  const getHeapProfileReqs = []
+  let uploadResolve
+  const allUploadsComplete = new Promise((resolve) => {
+    uploadResolve = resolve
+  })
+
+  const wss = new WebSocketServer({ port: port + 3 })
+  t.after(async () => wss.close())
+
+  const { waitForClientSubscription, getWs } = setupMockIccServer(
+    wss,
+    receivedMessages,
+    true
+  )
+
+  const app = createMockApp(port + 3)
+
+  app.watt.runtime.sendCommandToApplication = async (
+    serviceId,
+    command
+  ) => {
+    if (command === 'getLastProfile') {
+      getHeapProfileReqs.push({ serviceId })
+      if (getHeapProfileReqs.length === 2) {
+        uploadResolve()
+      }
+      return new Uint8Array([1, 2, 3, 4, 5])
+    }
+    return { success: false }
+  }
+
+  await updatePlugin(app)
+  await flamegraphsPlugin(app)
+
+  await app.connectToUpdates()
+  await app.setupFlamegraphs()
+
+  await waitForClientSubscription
+
+  const triggerHeapProfileMessage = {
+    command: 'trigger-heapprofile'
+  }
+
+  getWs().send(JSON.stringify(triggerHeapProfileMessage))
+
+  await allUploadsComplete
+
+  equal(getHeapProfileReqs.length, 2)
+
+  const service1Req = getHeapProfileReqs.find(
+    (f) => f.serviceId === 'service-1'
+  )
+  const service2Req = getHeapProfileReqs.find(
+    (f) => f.serviceId === 'service-2'
+  )
+
+  equal(service1Req.serviceId, 'service-1')
+  equal(service2Req.serviceId, 'service-2')
+
+  await app.closeUpdates()
+})
+
+test('should handle PLT_PPROF_NO_PROFILE_AVAILABLE error with info log', async (t) => {
+  setUpEnvironment()
+
+  const receivedMessages = []
+  const infoLogs = []
+  let errorCount = 0
+  let uploadResolve
+  const allUploadsComplete = new Promise((resolve) => {
+    uploadResolve = resolve
+  })
+
+  const wss = new WebSocketServer({ port: port + 4 })
+  t.after(async () => wss.close())
+
+  const { waitForClientSubscription, getWs } = setupMockIccServer(
+    wss,
+    receivedMessages,
+    true
+  )
+
+  const app = createMockApp(port + 4)
+  const originalInfo = app.log.info
+  app.log.info = (...args) => {
+    originalInfo(...args)
+    if (args[1] && args[1].includes('No profile available for the service')) {
+      infoLogs.push(args)
+      errorCount++
+      if (errorCount === 2) {
+        uploadResolve()
+      }
+    }
+  }
+
+  app.watt.runtime.sendCommandToApplication = async (
+    serviceId,
+    command
+  ) => {
+    if (command === 'getLastProfile') {
+      const error = new Error('No profile available - wait for profiling to complete or trigger manual capture')
+      error.code = 'PLT_PPROF_NO_PROFILE_AVAILABLE'
+      throw error
+    }
+    return { success: false }
+  }
+
+  await updatePlugin(app)
+  await flamegraphsPlugin(app)
+
+  await app.connectToUpdates()
+  await app.setupFlamegraphs()
+
+  await waitForClientSubscription
+
+  const triggerFlamegraphMessage = {
+    command: 'trigger-flamegraph'
+  }
+
+  getWs().send(JSON.stringify(triggerFlamegraphMessage))
+
+  await allUploadsComplete
+
+  equal(infoLogs.length, 2)
+  equal(infoLogs[0][0].serviceId, 'service-1')
+  equal(infoLogs[0][0].podId, 'test-pod-123')
+  equal(infoLogs[0][1], 'No profile available for the service')
+
+  await app.closeUpdates()
+})
+
+test('should handle PLT_PPROF_NOT_ENOUGH_ELU error with info log', async (t) => {
+  setUpEnvironment()
+
+  const receivedMessages = []
+  const infoLogs = []
+  let errorCount = 0
+  let uploadResolve
+  const allUploadsComplete = new Promise((resolve) => {
+    uploadResolve = resolve
+  })
+
+  const wss = new WebSocketServer({ port: port + 5 })
+  t.after(async () => wss.close())
+
+  const { waitForClientSubscription, getWs } = setupMockIccServer(
+    wss,
+    receivedMessages,
+    true
+  )
+
+  const app = createMockApp(port + 5)
+  const originalInfo = app.log.info
+  app.log.info = (...args) => {
+    originalInfo(...args)
+    if (args[1] && args[1].includes('ELU low, CPU profiling not active')) {
+      infoLogs.push(args)
+      errorCount++
+      if (errorCount === 2) {
+        uploadResolve()
+      }
+    }
+  }
+
+  app.watt.runtime.sendCommandToApplication = async (
+    serviceId,
+    command
+  ) => {
+    if (command === 'getLastProfile') {
+      const error = new Error('No profile available - event loop utilization has been below threshold for too long')
+      error.code = 'PLT_PPROF_NOT_ENOUGH_ELU'
+      throw error
+    }
+    return { success: false }
+  }
+
+  await updatePlugin(app)
+  await flamegraphsPlugin(app)
+
+  await app.connectToUpdates()
+  await app.setupFlamegraphs()
+
+  await waitForClientSubscription
+
+  const triggerFlamegraphMessage = {
+    command: 'trigger-flamegraph'
+  }
+
+  getWs().send(JSON.stringify(triggerFlamegraphMessage))
+
+  await allUploadsComplete
+
+  equal(infoLogs.length, 2)
+  equal(infoLogs[0][0].serviceId, 'service-1')
+  equal(infoLogs[0][0].podId, 'test-pod-123')
+  equal(infoLogs[0][1], 'ELU low, CPU profiling not active')
+
+  await app.closeUpdates()
+})