@platformatic/watt-extra 1.4.0 → 1.5.0-alpha.1

package/app.js

@@ -8,11 +8,12 @@ import scheduler from './plugins/scheduler.js'
 import auth from './plugins/auth.js'
 import update from './plugins/update.js'
 import alert from './plugins/alerts.js'
+import healthSignals from './plugins/health-signals.js'
 import flamegraphs from './plugins/flamegraphs.js'
 
 async function buildApp (logger) {
   const app = {
-    log: logger,
+    log: logger
   }
 
   avvio(app)
@@ -22,6 +23,7 @@ async function buildApp (logger) {
     .use(auth)
     .use(init)
     .use(alert)
+    .use(healthSignals)
     .use(metadata)
     .use(compliancy)
     .use(scheduler)
@@ -101,7 +103,7 @@ async function buildApp (logger) {
           {
             err: err.message,
             attemptNumber: retries,
-            nextRetryMs: currentRetryInterval,
+            nextRetryMs: currentRetryInterval
           },
           `Failed to send info to ICC, retrying in ${currentRetryInterval}ms`
         )

package/index.js

@@ -22,6 +22,7 @@ async function start () {
 
   app.log.info('Setup health check')
   await app.setupAlerts()
+  await app.setupHealthSignals()
   await app.setupFlamegraphs()
 
   app.log.info('Sending info to ICC')

package/lib/watt.js

@@ -98,6 +98,11 @@ class Watt {
     await this.runtime?.updateSharedContext?.({ context })
   }
 
+  getRuntimeVersion () {
+    const { version } = this.#require('@platformatic/runtime/package.json')
+    return version
+  }
+
   async #createRuntime () {
     this.#logger.info('Creating runtime')
     const { create, transform } = this.#require('@platformatic/runtime')

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platformatic/watt-extra",
-  "version": "1.4.0",
+  "version": "1.5.0-alpha.1",
   "description": "The Platformatic runtime manager",
   "type": "module",
   "scripts": {
@@ -19,25 +19,27 @@
   },
   "devDependencies": {
     "@fastify/websocket": "^11.1.0",
-    "@platformatic/composer": "^3.8.0",
-    "@platformatic/next": "^3.8.0",
-    "@platformatic/node": "^3.8.0",
-    "@platformatic/service": "^3.8.0",
+    "@platformatic/composer": "^3.14.0",
+    "@platformatic/next": "^3.14.0",
+    "@platformatic/node": "^3.14.0",
+    "@platformatic/service": "^3.14.0",
+    "atomic-sleep": "^1.0.0",
     "borp": "^0.21.0",
     "eslint": "9",
     "fastify": "^5.4.0",
     "fastify-plugin": "^5.0.1",
     "neostandard": "^0.12.0",
     "next": "^15.3.4",
-    "platformatic": "^3.8.0",
+    "platformatic": "^3.14.0",
     "pprof-format": "^2.1.0",
     "why-is-node-running": "^2.3.0"
   },
   "dependencies": {
     "@datadog/pprof": "^5.9.0",
     "@fastify/error": "^4.2.0",
-    "@platformatic/runtime": "^3.12.0",
-    "@platformatic/wattpm-pprof-capture": "^3.12.0",
+    "@platformatic/foundation": "^3.14.0",
+    "@platformatic/runtime": "^3.14.0",
+    "@platformatic/wattpm-pprof-capture": "^3.14.0",
     "avvio": "^9.1.0",
     "chalk": "^4.1.2",
     "commist": "^3.2.0",
@@ -47,6 +49,7 @@
     "minimist": "^1.2.8",
     "pino": "^10.0.0",
     "pino-pretty": "^13.0.0",
+    "semver": "^7.7.3",
     "undici": "^7.11.0",
     "undici-cache-redis": "^1.0.0",
     "undici-slicer-interceptor": "^0.4.1",

package/plugins/alerts.js

@@ -10,6 +10,9 @@ async function alerts (app, _opts) {
   const lastServicesAlertTime = {}
 
   async function setupAlerts () {
+    const scalerAlgorithmVersion = app.instanceConfig?.scaler?.version ?? 'v1'
+    if (scalerAlgorithmVersion !== 'v1') return
+
     // Skip alerts setup if ICC is not configured
     if (!app.env.PLT_ICC_URL) {
       app.log.info('PLT_ICC_URL not set, skipping alerts setup')
@@ -85,13 +88,13 @@ async function alerts (app, _opts) {
           method: 'POST',
           headers: {
             'Content-Type': 'application/json',
-            ...authHeaders,
+            ...authHeaders
           },
           body: JSON.stringify({
             applicationId: app.instanceConfig?.applicationId,
             alert: healthInfo,
-            healthHistory: healthCache,
-          }),
+            healthHistory: healthCache
+          })
         })
 
         if (statusCode !== 200) {

package/plugins/env.js

@@ -22,7 +22,9 @@ const schema = {
     PLT_FLAMEGRAPHS_ELU_THRESHOLD: { type: 'number', default: 0.4 },
     PLT_FLAMEGRAPHS_GRACE_PERIOD: { type: 'number', default: 3000 },
     PLT_JWT_EXPIRATION_OFFSET_SEC: { type: 'number', default: 60 },
-    PLT_UPDATES_RECONNECT_INTERVAL_SEC: { type: 'number', default: 1 }
+    PLT_UPDATES_RECONNECT_INTERVAL_SEC: { type: 'number', default: 1 },
+    PLT_ELU_HEALTH_SIGNAL_THRESHOLD: { type: 'number', default: 0.9 },
+    PLT_HEAP_HEALTH_SIGNAL_THRESHOLD: { type: ['number', 'string'], default: '4GB' }
   }
 }
 

package/plugins/health-signals.js

@@ -0,0 +1,191 @@
+import { request } from 'undici'
+import semver from 'semver'
+import { parseMemorySize } from '@platformatic/foundation'
+
+class HealthSignalsCache {
+  #signals = []
+  #size = 100
+
+  constructor () {
+    this.#signals = []
+  }
+
+  add (signals) {
+    for (const signal of signals) {
+      this.#signals.push(signal)
+    }
+    if (this.#signals.length > this.#size) {
+      this.#signals.splice(0, this.#signals.length - this.#size)
+    }
+  }
+
+  getAll () {
+    const values = this.#signals
+    this.#signals = []
+    return values
+  }
+}
+
+async function healthSignals (app, _opts) {
+  const signalsCaches = {}
+  const servicesSendingStatuses = {}
+
+  // TODO: needed to the UI compatibility
+  // remove after depricating the Scaler v1 UI
+  const servicesMetrics = {}
+
+  async function setupHealthSignals () {
+    const scalerAlgorithmVersion = app.instanceConfig?.scaler?.version ?? 'v1'
+    if (scalerAlgorithmVersion !== 'v2') return
+
+    const runtimeVersion = app.watt.getRuntimeVersion()
+    if (semver.lt(runtimeVersion, '1.4.0')) {
+      app.log.warn(
+        `Watt version "${runtimeVersion}" does not support health signals for the Signal Scaler Algorithm.` +
+          'Please update your watt-extra to version 1.4.0 or higher.'
+      )
+      return
+    }
+
+    const eluThreshold = app.env.PLT_ELU_HEALTH_SIGNAL_THRESHOLD
+
+    let heapThreshold = app.env.PLT_HEAP_HEALTH_SIGNAL_THRESHOLD
+    if (typeof heapThreshold === 'string') {
+      heapThreshold = parseMemorySize(heapThreshold)
+    }
+
+    // Skip alerts setup if ICC is not configured
+    if (!app.env.PLT_ICC_URL) {
+      app.log.info('PLT_ICC_URL not set, skipping alerts setup')
+      return
+    }
+
+    const scalerUrl = app.instanceConfig?.iccServices?.scaler?.url
+    const runtime = app.watt.runtime
+
+    if (!scalerUrl) {
+      app.log.warn(
+        'No scaler URL found in ICC services, health alerts disabled'
+      )
+      return
+    }
+
+    runtime.on('application:worker:health:metrics', async (healthInfo) => {
+      if (!healthInfo) {
+        app.log.error('No health metrics info received')
+      }
+
+      const {
+        application: serviceId,
+        currentHealth,
+        healthSignals
+      } = healthInfo
+
+      const { elu, heapUsed, heapTotal } = currentHealth
+
+      if (elu > eluThreshold) {
+        healthSignals.push({
+          type: 'elu',
+          value: currentHealth.elu,
+          description:
+            `The ${serviceId} has an ELU of ${(elu * 100).toFixed(2)} %, ` +
+              `above the maximum allowed usage of ${(eluThreshold * 100).toFixed(2)} %`,
+          timestamp: Date.now()
+        })
+      }
+
+      if (heapThreshold && heapUsed > heapThreshold) {
+        const usedHeapMb = Math.round(heapUsed / 1024 / 1024)
+        const heapThresholdMb = Math.round(heapThreshold / 1024 / 1024)
+
+        healthSignals.push({
+          type: 'heapUsed',
+          value: currentHealth.heapUsed,
+          description:
+            `The ${serviceId} is using ${usedHeapMb} MB of heap, ` +
+              `above the maximum allowed usage of ${heapThresholdMb} MB`,
+          timestamp: Date.now()
+        })
+      }
+
+      // TODO: needed to the UI compatibility
+      // remove after depricating the Scaler v1 UI
+      servicesMetrics[serviceId] ??= { elu: 0, heapUsed: 0, heapTotal: 0 }
+      const metrics = servicesMetrics[serviceId]
+      if (elu > metrics.elu) {
+        metrics.elu = elu
+      }
+      if (heapUsed > metrics.heapUsed) {
+        metrics.heapUsed = heapUsed
+        metrics.heapTotal = heapTotal
+      }
+
+      if (healthSignals.length > 0) {
+        await sendHealthSignalsWithTimeout(serviceId, healthSignals)
+      }
+    })
+  }
+  app.setupHealthSignals = setupHealthSignals
+
+  async function sendHealthSignalsWithTimeout (serviceId, signals) {
+    signalsCaches[serviceId] ??= new HealthSignalsCache()
+    servicesSendingStatuses[serviceId] ??= false
+
+    const signalsCache = signalsCaches[serviceId]
+    signalsCache.add(signals)
+
+    if (!servicesSendingStatuses[serviceId]) {
+      servicesSendingStatuses[serviceId] = true
+      setTimeout(async () => {
+        servicesSendingStatuses[serviceId] = false
+
+        const metrics = servicesMetrics[serviceId]
+        servicesMetrics[serviceId] = null
+
+        try {
+          const signals = signalsCache.getAll()
+          await sendHealthSignals(serviceId, signals, metrics)
+        } catch (err) {
+          app.log.error({ err }, 'Failed to send health signals to scaler')
+        }
+      }, 5000).unref()
+    }
+  }
+
+  async function sendHealthSignals (serviceId, signals, metrics) {
+    const scalerUrl = app.instanceConfig?.iccServices?.scaler?.url
+    const applicationId = app.instanceConfig?.applicationId
+    const authHeaders = await app.getAuthorizationHeader()
+
+    const { statusCode, body } = await request(`${scalerUrl}/signals`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        ...authHeaders
+      },
+      body: JSON.stringify({
+        applicationId,
+        serviceId,
+        signals,
+        elu: metrics.elu,
+        heapUsed: metrics.heapUsed,
+        heapTotal: metrics.heapTotal
+      })
+    })
+
+    if (statusCode !== 200) {
+      const error = await body.text()
+      app.log.error({ error }, 'Failed to send health signals to scaler')
+    }
+
+    const alert = await body.json()
+
+    try {
+      await app.sendFlamegraphs({ serviceIds: [serviceId], alertId: alert.id })
+    } catch (err) {
+      app.log.error({ err }, 'Failed to send a flamegraph')
+    }
+  }
+}
+
+export default healthSignals

package/plugins/init.js

@@ -40,6 +40,8 @@ async function initPlugin (app) {
     const instanceConfig = await initApplicationInstance(instanceId, applicationName)
     app.log.info({ applicationId: instanceConfig.applicationId }, 'Got application info')
 
+    instanceConfig.scaler ??= { version: 'v1' }
+
     // Use the application name from the ICC response if not provided
     applicationName = applicationName || instanceConfig.applicationName
     app.log.info({ applicationName }, 'Application name resolved')

package/plugins/metadata.js

@@ -42,15 +42,17 @@ async function metadata (app, _opts) {
           )
         )
 
-        const verticalScalerConfig = runtimeConfig.verticalScaler
-        if (verticalScalerConfig?.enabled) {
-          for (const applicationId in verticalScalerConfig.applications) {
-            const service = services.find((s) => s.id === applicationId)
-            if (service) {
-              const appScalerConfig = verticalScalerConfig.applications[applicationId]
-              service.maxWorkers = appScalerConfig.maxWorkers
-              service.minWorkers = appScalerConfig.minWorkers
-            }
+        const workersCount = getWorkersCount(runtimeConfig)
+        for (const service of services) {
+          const serviceWorkers = workersCount[service.id]
+          if (serviceWorkers?.workers) {
+            service.workers = serviceWorkers.workers
+          }
+          if (serviceWorkers?.minWorkers) {
+            service.minWorkers = serviceWorkers.minWorkers
+          }
+          if (serviceWorkers?.maxWorkers) {
+            service.maxWorkers = serviceWorkers.maxWorkers
           }
         }
 
@@ -91,6 +93,42 @@ async function metadata (app, _opts) {
     }
   }
   app.sendMetadata = sendMetadata
+
+  function getWorkersCount (runtimeConfig) {
+    const verticalScalerConfig = runtimeConfig.verticalScaler
+    const serviceWorkers = {}
+
+    for (const application of runtimeConfig.applications) {
+      const { workers } = application
+      if (!workers) continue
+
+      if (typeof workers === 'number') {
+        serviceWorkers[application.id] = {
+          workers,
+          minWorkers: workers,
+          maxWorkers: workers
+        }
+      }
+      if (typeof workers === 'object') {
+        serviceWorkers[application.id] = {
+          workers: workers.static,
+          minWorkers: workers.minimum ?? workers.static,
+          maxWorkers: workers.maximum ?? workers.static
+        }
+      }
+    }
+
+    if (verticalScalerConfig?.enabled) {
+      for (const applicationId in verticalScalerConfig.applications) {
+        const scalingConfig = verticalScalerConfig.applications[applicationId]
+        serviceWorkers[applicationId] ??= {}
+        serviceWorkers[applicationId].maxWorkers ??= scalingConfig.maxWorkers
+        serviceWorkers[applicationId].minWorkers ??= scalingConfig.minWorkers
+      }
+    }
+
+    return serviceWorkers
+  }
 }
 
 export default metadata

package/test/fixtures/service-1/routes/root.cjs

@@ -1,48 +1,66 @@
-"use strict";
+'use strict'
 
-const { join } = require("node:path");
-const { readFile } = require("node:fs/promises");
-const { request } = require("undici");
+const { join } = require('node:path')
+const { readFile } = require('node:fs/promises')
+const { request } = require('undici')
+const atomicSleep = require('atomic-sleep')
 
 module.exports = async function (fastify) {
-  fastify.get("/example", async () => {
-    return { hello: "world" };
-  });
+  fastify.get('/example', async () => {
+    return { hello: 'world' }
+  })
 
-  fastify.get("/config", async () => {
-    return fastify.platformatic.config;
-  });
+  fastify.get('/config', async () => {
+    return fastify.platformatic.config
+  })
 
-  fastify.get("/preprocess", async () => {
+  fastify.get('/preprocess', async () => {
     return {
-      base: "~PLT_BASE_PATH",
-      leadingSlash: "/~PLT_BASE_PATH",
-      withPrefix: "~PLT_BASE_PATH/foo",
-      externalUrl: "~PLT_EXTERNAL_APP_URL",
-    };
-  });
-
-  fastify.get("/custom-ext-file", async () => {
-    const customExtFilePath = join(__dirname, "..", "file.custom");
-    const customExtFile = await readFile(customExtFilePath, "utf8");
-    return { data: customExtFile };
-  });
-
-  fastify.get("/env", async () => {
-    return { env: process.env };
-  });
-
-  fastify.post("/request", async (req) => {
-    const { method, url } = req.body;
+      base: '~PLT_BASE_PATH',
+      leadingSlash: '/~PLT_BASE_PATH',
+      withPrefix: '~PLT_BASE_PATH/foo',
+      externalUrl: '~PLT_EXTERNAL_APP_URL'
+    }
+  })
+
+  fastify.get('/custom-ext-file', async () => {
+    const customExtFilePath = join(__dirname, '..', 'file.custom')
+    const customExtFile = await readFile(customExtFilePath, 'utf8')
+    return { data: customExtFile }
+  })
+
+  fastify.get('/env', async () => {
+    return { env: process.env }
+  })
+
+  fastify.post('/request', async (req) => {
+    const { method, url } = req.body
 
     const { statusCode, headers, body } = await request(url, {
-      method: method ?? "GET",
+      method: method ?? 'GET',
       headers: {
-        "content-type": "application/json",
-      },
-    });
-    const data = await body.text();
-
-    return { statusCode, headers, data };
-  });
-};
+        'content-type': 'application/json'
+      }
+    })
+    const data = await body.text()
+
+    return { statusCode, headers, data }
+  })
+
+  fastify.post('/cpu-intensive', async (req) => {
+    // Simulate a CPU intensive operation
+    const timeout = req.query.timeout || 10000
+    atomicSleep(timeout)
+
+    return { status: 'ok' }
+  })
+
+  fastify.post('/custom-health-signal', async (req) => {
+    const { type, value, description } = req.body
+    await globalThis.platformatic.sendHealthSignal({
+      type,
+      value,
+      description
+    })
+  })
+}

package/test/health-signals.test.js

@@ -0,0 +1,130 @@
+import assert from 'node:assert'
+import { test } from 'node:test'
+import { randomUUID } from 'node:crypto'
+import { join, dirname } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { setTimeout as sleep } from 'node:timers/promises'
+import { Profile } from 'pprof-format'
+import { request } from 'undici'
+import { setUpEnvironment, startICC } from './helper.js'
+import { start } from '../index.js'
+
+const __filename = fileURLToPath(import.meta.url)
+const __dirname = dirname(__filename)
+
+test('should send health signals when service becomes unhealthy', async (t) => {
+  const applicationName = 'test-app'
+  const applicationId = randomUUID()
+  const applicationPath = join(__dirname, 'fixtures', 'service-1')
+
+  const receivedSignalReqs = []
+  const receivedFlamegraphReqs = []
+
+  const getAuthorizationHeader = async (headers) => {
+    return { ...headers, authorization: 'Bearer test-token' }
+  }
+
+  const icc = await startICC(t, {
+    applicationId,
+    applicationName,
+    scaler: { version: 'v2' },
+    processSignals: (req) => {
+      assert.equal(req.headers.authorization, 'Bearer test-token')
+      receivedSignalReqs.push(req.body)
+      return { id: 'test-alert-id' }
+    },
+    processFlamegraphs: (req) => {
+      const alertId = req.query.alertId
+      assert.strictEqual(alertId, 'test-alert-id')
+      assert.strictEqual(req.headers.authorization, 'Bearer test-token')
+      receivedFlamegraphReqs.push(req.body)
+    }
+  })
+
+  setUpEnvironment({
+    PLT_APP_NAME: applicationName,
+    PLT_APP_DIR: applicationPath,
+    PLT_ICC_URL: 'http://127.0.0.1:3000',
+    PLT_DISABLE_FLAMEGRAPHS: false,
+    PLT_FLAMEGRAPHS_INTERVAL_SEC: 2,
+    PLT_FLAMEGRAPHS_ELU_THRESHOLD: 0
+  })
+
+  const app = await start()
+  app.getAuthorizationHeader = getAuthorizationHeader
+
+  t.after(async () => {
+    await app.close()
+    await icc.close()
+  })
+
+  // Wait for the first flamegraph to be generated
+  await sleep(5000)
+
+  {
+    const { statusCode } = await request('http://127.0.0.1:3042/custom-health-signal', {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json'
+      },
+      body: JSON.stringify({
+        type: 'custom',
+        value: 42,
+        description: 'This is a custom health signal'
+      })
+    })
+    assert.strictEqual(statusCode, 200)
+  }
+
+  {
+    const { statusCode } = await request('http://127.0.0.1:3042/cpu-intensive', {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json'
+      },
+      body: JSON.stringify({ timeout: 3000 })
+    })
+    assert.strictEqual(statusCode, 200)
+  }
+
+  assert.strictEqual(receivedSignalReqs.length, 1)
+
+  const receivedSignalReq = receivedSignalReqs[0]
+  assert.ok(receivedSignalReq, 'Alert should have been received')
+  assert.strictEqual(receivedSignalReq.applicationId, applicationId)
+  assert.strictEqual(receivedSignalReq.serviceId, 'main')
+  assert.ok(receivedSignalReq.elu > 0.9)
+  assert.ok(receivedSignalReq.heapUsed > 0)
+  assert.ok(receivedSignalReq.heapTotal > 0)
+
+  const receivedSignals = receivedSignalReq.signals
+  assert.ok(receivedSignals.length > 5)
+
+  const eluSignals = receivedSignals.filter(
+    (signal) => signal.type === 'elu'
+  )
+  const customSignals = receivedSignals.filter(
+    (signal) => signal.type === 'custom'
+  )
+  assert.strictEqual(customSignals.length, 1)
+
+  for (const receivedSignal of eluSignals) {
+    assert.strictEqual(receivedSignal.type, 'elu')
+    assert.ok(receivedSignal.value > 0.9)
+    assert.ok(receivedSignal.timestamp > 0)
+  }
+  for (const receivedSignal of customSignals) {
+    assert.strictEqual(receivedSignal.type, 'custom')
+    assert.strictEqual(receivedSignal.value, 42)
+    assert.ok(receivedSignal.timestamp > 0)
+  }
+
+  // Wait for the second flamegraph to be generated
+  await sleep(2000)
+
+  // assert.strictEqual(receivedFlamegraphReqs.length, 1)
+
+  const receivedFlamegraph = receivedFlamegraphReqs[0]
+  const profile = Profile.decode(receivedFlamegraph)
+  assert.ok(profile, 'Profile should be decoded')
+})

package/test/helper.js

@@ -35,6 +35,7 @@ async function startICC (t, opts = {}) {
     applicationId,
     applicationName,
     applicationMetricsLabel,
+    scaler,
     iccServices,
     iccConfig = {},
     enableOpenTelemetry = false,
@@ -111,6 +112,7 @@ async function startICC (t, opts = {}) {
         applicationName,
         applicationMetricsLabel,
         iccServices,
+        scaler,
         config: iccConfig,
         enableOpenTelemetry,
         enableSlicerInterceptor,
@@ -191,6 +193,9 @@ async function startICC (t, opts = {}) {
     icc.post('/alerts', async (req) => {
       return opts.processAlerts?.(req)
     })
+    icc.post('/signals', async (req) => {
+      return opts.processSignals?.(req)
+    })
     icc.post('/pods/:podId/services/:serviceId/flamegraph', async (req) => {
       return opts.processFlamegraphs?.(req)
     })

package/.claude/settings.local.json

@@ -1,11 +0,0 @@
-{
-  "permissions": {
-    "allow": [
-      "Read(//work/workspaces/workspace-platformatic/platformatic/**)",
-      "Bash(npx borp:*)",
-      "Bash(timeout 30 npx borp -c 1 --timeout=20000 ./test/trigger-flamegraphs.test.js)"
-    ],
-    "deny": [],
-    "ask": []
-  }
-}