@platformatic/watt-extra 0.1.0

package/plugins/alerts.js

@@ -0,0 +1,115 @@
+import { request } from 'undici'
+
+async function alerts (app, _opts) {
+  const healthCache = [] // It's OK to have this in memory, this is per-pod.
+  const podHealthWindow =
+    app.instanceConfig?.config?.scaler?.podHealthWindow || 60 * 1000
+  const alertRetentionWindow =
+    app.instanceConfig?.config?.scaler?.alertRetentionWindow || 30 * 1000
+
+  let lastAlertTime = 0
+  async function setupAlerts () {
+    // Skip alerts setup if ICC is not configured
+    if (!app.env.PLT_ICC_URL) {
+      app.log.info('PLT_ICC_URL not set, skipping alerts setup')
+      return
+    }
+
+    const scalerUrl = app.instanceConfig?.iccServices?.scaler?.url
+    const runtime = app.wattpro.runtime
+
+    if (!scalerUrl) {
+      app.log.warn(
+        'No scaler URL found in ICC services, health alerts disabled'
+      )
+      return
+    }
+
+    runtime.on('health', async (healthInfo) => {
+      if (!healthInfo) {
+        app.log.error('No health info received')
+        return
+      }
+
+      const timestamp = Date.now()
+      const healthWithTimestamp = { ...healthInfo, timestamp }
+      delete healthWithTimestamp.healthConfig // we don't need to store this
+
+      healthCache.push(healthWithTimestamp)
+
+      const cutoffTime = timestamp - podHealthWindow
+      const validIndex = healthCache.findIndex(
+        (entry) => entry.timestamp >= cutoffTime
+      )
+      if (validIndex > 0) {
+        healthCache.splice(0, validIndex)
+      }
+
+      // healthInfo is an object with the following structure:
+      // id: "service-1"
+      // service: "service-1"
+      // currentHealth: {
+      //   "elu": 0.003816403352054066,
+      //   "heapUsed": 76798040,
+      //   "heapTotal": 99721216
+      // }
+      // unhealthy: false
+      // healthConfig: {
+      //   "enabled": true,
+      //   "interval": 1000,
+      //   "gracePeriod": 1000,
+      //   "maxUnhealthyChecks": 10,
+      //   "maxELU": 0.99,
+      //   "maxHeapUsed": 0.99,
+      //   "maxHeapTotal": 4294967296
+      // }
+
+      if (healthInfo.unhealthy) {
+        const currentTime = Date.now()
+        if (currentTime - lastAlertTime < alertRetentionWindow) {
+          app.log.debug('Skipping alert, within retention window')
+          return
+        }
+
+        lastAlertTime = currentTime
+        delete healthInfo.healthConfig
+
+        const authHeaders = await app.getAuthorizationHeader()
+
+        const { statusCode, body } = await request(`${scalerUrl}/alerts`, {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+            ...authHeaders,
+          },
+          body: JSON.stringify({
+            applicationId: app.instanceConfig?.applicationId,
+            alert: healthInfo,
+            healthHistory: healthCache,
+          }),
+        })
+
+        if (statusCode !== 200) {
+          const error = await body.text()
+          app.log.error({ error }, 'Failed to send alert to scaler')
+          return
+        }
+
+        const alert = await body.json()
+        const serviceId = healthInfo.id
+
+        try {
+          await app.sendFlamegraphs({
+            serviceIds: [serviceId],
+            alertId: alert.id
+          })
+        } catch (err) {
+          app.log.error({ err }, 'Failed to send a flamegraph')
+        }
+      }
+    })
+  }
+  app.setupAlerts = setupAlerts
+}
+
+export default alerts

package/plugins/auth.js

@@ -0,0 +1,89 @@
+import { readFile, stat } from 'node:fs/promises'
+import { Agent } from 'undici'
+
+const K8S_TOKEN_PATH = '/var/run/secrets/kubernetes.io/serviceaccount/token'
+
+function decodeJwtPayload (token) {
+  try {
+    if (!token) return null
+    const base64Payload = token.split('.')[1]
+    if (!base64Payload) return null
+    const payload = Buffer.from(base64Payload, 'base64').toString('utf8')
+    return JSON.parse(payload)
+  } catch (err) {
+    return null
+  }
+}
+
+function isTokenExpired (token, offset = 0) {
+  const payload = decodeJwtPayload(token)
+  if (!payload || !payload.exp) return true
+
+  // Check if token is expired
+  const currentTime = Math.floor(Date.now() / 1000)
+  return payload.exp <= (currentTime + offset)
+}
+
+async function authPlugin (app) {
+  // Add a 1 min offset to update the token before it expires
+  // via runtime shared context
+  const offset = parseInt(process.env.PLT_JWT_EXPIRATION_OFFSET_SEC ?? 0)
+
+  async function loadToken () {
+    let token
+    try {
+      await stat(K8S_TOKEN_PATH)
+      app.log.info('Loading JWT token from K8s service account')
+      token = await readFile(K8S_TOKEN_PATH, 'utf8')
+    } catch (err) {
+      app.log.warn('Failed to load JWT token from K8s service account')
+    }
+
+    if (!token) {
+      app.log.warn('K8s token not found, falling back to environment variable')
+      token = process.env.PLT_TEST_TOKEN
+    }
+
+    return token
+  }
+
+  const getAuthorizationHeader = async (headers = {}) => {
+    if (isTokenExpired(app.token, offset)) {
+      app.log.info('JWT token expired, reloading')
+      app.token = await loadToken()
+
+      app.wattpro?.updateSharedContext({
+        iccAuthHeaders: { authorization: `Bearer ${app.token}` }
+      }).catch((err) => {
+        app.log.error({ err }, 'Failed to update jwt token in shared context')
+      })
+    }
+
+    return {
+      ...headers,
+      authorization: `Bearer ${app.token}`
+    }
+  }
+
+  const authorizationTokenInterceptor = dispatch => {
+    return async function InterceptedDispatch (opts, handler) {
+      opts.headers = await getAuthorizationHeader(opts.headers)
+      return dispatch(opts, handler)
+    }
+  }
+
+  app.token = await loadToken()
+
+  await setInterval(async () => {
+    // Check if token is expired to propagate it to the runtime
+    // via the shared context
+    await getAuthorizationHeader()
+  }, offset * 1000 / 2).unref()
+
+  // We cannot change the global dispatcher because it's shared with the runtime main thread.
+  const wattDispatcher = new Agent()
+  app.dispatcher = wattDispatcher.compose(authorizationTokenInterceptor)
+  app.getAuthorizationHeader = getAuthorizationHeader
+}
+
+export default authPlugin

package/plugins/compliancy.js

@@ -0,0 +1,70 @@
+import { getCompliancyMetadata } from '../compliance/index.js'
+import {
+  CompliancyMetadataError,
+  CompliancyStatusError
+} from '../lib/errors.js'
+
+async function compliancy (app, _opts) {
+  async function checkCompliancy () {
+    if (app.env.PLT_DISABLE_COMPLIANCE_CHECK === true) return
+
+    // Skip compliance check if ICC is not configured
+    if (!app.env.PLT_ICC_URL) {
+      app.log.info('PLT_ICC_URL not set, skipping compliance check')
+      return
+    }
+
+    const runtime = app.wattpro.runtime
+    const applicationId = app.instanceConfig?.applicationId
+    const appDir = app.env.PLT_APP_DIR
+
+    const { default: build, setDefaultHeaders } = await import('../clients/compliance/compliance.mjs')
+    const complianceUrl = app.instanceConfig?.iccServices?.compliance?.url
+
+    if (!complianceUrl) {
+      app.log.warn('No compliance URL found in ICC services')
+      return
+    }
+    const compliancyClient = build(complianceUrl)
+
+    // There is a better way? We need to set the default headers for the client
+    // every time, because the token might be expired
+    // And we cannot set the global dispatcher because it's shared with the runtime main thread.
+    setDefaultHeaders(await app.getAuthorizationHeader())
+    const compliancyMetadata = await getCompliancyMetadata({
+      projectDir: appDir,
+      runtime
+    })
+
+    {
+      const res = await compliancyClient.postMetadata({
+        applicationId,
+        data: compliancyMetadata
+      })
+
+      if (res.statusCode !== 200 && res.statusCode !== 201) {
+        app.log.error(res, 'Failed to send compliancy metadata')
+        throw new CompliancyMetadataError()
+      }
+
+      app.log.info('Compliancy metadata sent')
+    }
+
+    {
+      const res = await compliancyClient.postCompliance({ applicationId })
+      if (res.statusCode !== 200) {
+        app.log.error(res, 'Failed to get compliance status')
+        throw new CompliancyStatusError()
+      }
+
+      const { compliant, report } = JSON.parse(res.body)
+
+      if (!compliant) {
+        app.log.error(report, 'Compliancy check failed')
+      }
+    }
+  }
+  app.checkCompliancy = checkCompliancy
+}
+
+export default compliancy

package/plugins/env.js

@@ -0,0 +1,58 @@
+import envSchema from 'env-schema'
+
+const schema = {
+  type: 'object',
+  required: [],
+  properties: {
+    PLT_APP_NAME: { type: 'string' },
+    PLT_ICC_URL: { type: 'string' },
+    PLT_APP_DIR: { type: 'string' },
+    PLT_TEST_TOKEN: { type: 'string' }, // JWT token for authentication when not in K8s
+    PLT_APP_HOSTNAME: { type: 'string', default: '' },
+    PLT_APP_PORT: { type: 'number' },
+    PLT_METRICS_PORT: { type: 'number', default: 9090 },
+    PLT_LOG_LEVEL: { type: 'string', default: 'info' },
+    PLT_ICC_RETRY_TIME: { type: 'number', default: 5000 },
+    PLT_DISABLE_COMPLIANCE_CHECK: { type: 'boolean', default: false },
+    PLT_APP_INTERNAL_SUB_DOMAIN: { type: 'string', default: 'plt.local' },
+    PLT_DEFAULT_CACHE_TAGS_HEADER: { type: 'string', default: 'x-plt-cache-tags' },
+    PLT_CACHE_CONFIG: { type: 'string' },
+    PLT_DISABLE_FLAMEGRAPHS: { type: 'boolean', default: false },
+    PLT_FLAMEGRAPHS_INTERVAL_SEC: { type: 'number', default: 60 },
+    PLT_JWT_EXPIRATION_OFFSET_SEC: { type: 'number', default: 60 },
+    PLT_UPDATES_RECONNECT_INTERVAL_SEC: { type: 'number', default: 1 }
+  }
+}
+
+// Loads the config calling ICC and populates the env variables,
+// using default to allow the app to start without ICC
+const getDefaultEnv = (iccUrl) => {
+  const defaults = {
+    PLT_APP_DIR: process.cwd()
+  }
+
+  if (iccUrl) {
+    defaults.PLT_CONTROL_PLANE_URL = `${iccUrl}/control-plane`
+  }
+
+  return defaults
+}
+
+async function envPlugin (app) {
+  const env = envSchema({
+    schema,
+    dotenv: true
+  })
+
+  const iccURL = env.PLT_ICC_URL
+  const defaultEnv = getDefaultEnv(iccURL)
+
+  app.env = {
+    ...defaultEnv,
+    ...env
+  }
+
+  app.log.info('Environment variables set up')
+}
+
+export default envPlugin

package/plugins/flamegraphs.js

@@ -0,0 +1,100 @@
+'use strict'
+
+import { request } from 'undici'
+
+async function flamegraphs (app, _opts) {
+  const isFlamegraphsDisabled = app.env.PLT_DISABLE_FLAMEGRAPHS
+  const flamegraphsIntervalSec = app.env.PLT_FLAMEGRAPHS_INTERVAL_SEC
+
+  const durationMillis = parseInt(flamegraphsIntervalSec) * 1000
+
+  app.setupFlamegraphs = async () => {
+    if (isFlamegraphsDisabled) {
+      app.log.info('PLT_DISABLE_FLAMEGRAPHS is set, skipping profiling')
+      return
+    }
+
+    app.log.info('Start profiling services')
+
+    const runtime = app.wattpro.runtime
+    const { applications } = await runtime.getApplications()
+
+    const promises = []
+    for (const application of applications) {
+      const promise = runtime.sendCommandToApplication(
+        application.id,
+        'startProfiling',
+        { durationMillis }
+      )
+      promises.push(promise)
+    }
+
+    const results = await Promise.allSettled(promises)
+    for (const result of results) {
+      if (result.status === 'rejected') {
+        app.log.error({ result }, 'Failed to start profiling')
+      }
+    }
+  }
+
+  app.sendFlamegraphs = async (options = {}) => {
+    if (isFlamegraphsDisabled) {
+      app.log.info('PLT_DISABLE_FLAMEGRAPHS is set, flamegraphs are disabled')
+      return
+    }
+
+    let { serviceIds, alertId } = options
+
+    const scalerUrl = app.instanceConfig?.iccServices?.scaler?.url
+    if (!scalerUrl) {
+      app.log.error('No scaler URL found in ICC services, cannot send flamegraph')
+      throw new Error('No scaler URL found in ICC services, cannot send flamegraph')
+    }
+
+    const podId = app.instanceId
+    const runtime = app.wattpro.runtime
+
+    if (!serviceIds) {
+      const { applications } = await runtime.getApplications()
+      serviceIds = applications.map(app => app.id)
+    }
+
+    const authHeaders = await app.getAuthorizationHeader()
+
+    const uploadPromises = serviceIds.map(async (serviceId) => {
+      try {
+        const profile = await runtime.sendCommandToApplication(serviceId, 'getLastProfile')
+        if (!profile || !(profile instanceof Uint8Array)) {
+          app.log.error({ serviceId }, 'Failed to get profile from service')
+          return
+        }
+
+        const url = `${scalerUrl}/pods/${podId}/services/${serviceId}/flamegraph`
+
+        app.log.info({ serviceId, podId }, 'Sending flamegraph')
+
+        const { statusCode, body } = await request(url, {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/octet-stream',
+            ...authHeaders
+          },
+          query: alertId ? { alertId } : {},
+          body: profile
+        })
+
+        if (statusCode !== 200) {
+          const error = await body.text()
+          app.log.error({ error }, 'Failed to send flamegraph')
+          throw new Error(`Failed to send flamegraph: ${error}`)
+        }
+      } catch (err) {
+        app.log.warn({ err, serviceId, podId }, 'Failed to send flamegraph from service')
+      }
+    })
+
+    await Promise.all(uploadPromises)
+  }
+}
+
+export default flamegraphs

package/plugins/init.js

@@ -0,0 +1,70 @@
+import WattPro from '../lib/wattpro.js'
+import os from 'node:os'
+
+async function initPlugin (app) {
+  async function initApplicationInstance (podId, applicationName = null) {
+    const { default: build, setDefaultHeaders } = await import('../clients/control-plane/control-plane.mjs')
+    const controlPlaneClient = build(app.env.PLT_CONTROL_PLANE_URL)
+    // There is a better way? We need to set the default headers for the client
+    // every time, because the token might be expired
+    // And we cannot set the global dispatcher because it's shared with the runtime main thread.
+    setDefaultHeaders(await app.getAuthorizationHeader())
+    const request = { podId }
+    if (applicationName) {
+      request.applicationName = applicationName
+    }
+    return controlPlaneClient.initApplicationInstance(request)
+  }
+
+  async function initApplication () {
+    app.log.info('Starting WattPro runtime manager')
+
+    let applicationName = app.env.PLT_APP_NAME
+    const applicationDir = app.env.PLT_APP_DIR
+    const instanceId = os.hostname()
+
+    app.log.info({ applicationName, applicationDir }, 'Loading wattpro application')
+
+    // Skip ICC initialization if PLT_ICC_URL is not set
+    if (!app.env.PLT_ICC_URL) {
+      app.log.info('PLT_ICC_URL not set, skipping ICC initialization')
+      app.applicationName = applicationName
+      app.instanceConfig = null
+      app.instanceId = instanceId
+      return
+    }
+
+    const instanceConfig = await initApplicationInstance(instanceId, applicationName)
+    app.log.info({ applicationId: instanceConfig.applicationId }, 'Got application info')
+
+    // Use the application name from the ICC response if not provided
+    applicationName = applicationName || instanceConfig.applicationName
+    app.log.info({ applicationName }, 'Application name resolved')
+
+    app.applicationName = applicationName
+    app.instanceConfig = instanceConfig
+    app.instanceId = instanceId
+  }
+  try {
+    await initApplication()
+  } catch (err) {
+    // We don't re-throw here because we can continue without application info
+    // and nothing here should block the app from start
+    app.log.error(err, 'Failed to get application information')
+
+    // Set fallback values when ICC connection fails
+    if (!app.applicationName) {
+      app.applicationName = app.env.PLT_APP_NAME
+      app.instanceConfig = null
+      app.instanceId = os.hostname()
+    }
+  }
+  const wattpro = new WattPro(app)
+  app.wattpro = wattpro
+  app.initApplication = initApplication
+
+  const headers = await app.getAuthorizationHeader()
+  await app.wattpro.updateSharedContext({ iccAuthHeaders: headers })
+}
+
+export default initPlugin

package/plugins/metadata.js

@@ -0,0 +1,84 @@
+import {
+  MetadataRuntimeError,
+  MetadataError,
+  MetadataStateError,
+  MetadataAppIdError,
+  MetadataRuntimeNotStartedError
+} from '../lib/errors.js'
+
+async function metadata (app, _opts) {
+  async function sendMetadata () {
+    // Skip metadata processing if ICC is not configured
+    if (!app.env.PLT_ICC_URL) {
+      app.log.info('PLT_ICC_URL not set, skipping metadata processing')
+      return
+    }
+
+    const applicationId = app.instanceConfig?.applicationId
+    const runtime = app.wattpro.runtime
+    if (!applicationId) {
+      app.log.warn('Cannot process metadata: no applicationId available')
+      throw new MetadataAppIdError()
+    }
+
+    if (!runtime) {
+      app.log.warn('Cannot process metadata: runtime not started')
+      throw new MetadataRuntimeNotStartedError()
+    }
+
+    try {
+      const { default: build, setDefaultHeaders } = await import('../clients/control-plane/control-plane.mjs')
+      const controlPlaneClient = build(app.env.PLT_CONTROL_PLANE_URL)
+
+      try {
+        const [runtimeConfig, runtimeMetadata] = await Promise.all([
+          runtime.getRuntimeConfig(),
+          runtime.getRuntimeMetadata()
+        ])
+
+        const applications = await Promise.all(
+          runtimeConfig.applications.map((application) =>
+            runtime.getApplicationDetails(application.id)
+          )
+        )
+
+        try {
+          // There is a better way? We need to set the default headers for the client
+          // every time, because the token might be expired
+          // And we cannot set the global dispatcher because it's shared with the runtime main thread.
+          setDefaultHeaders(await app.getAuthorizationHeader())
+          await controlPlaneClient.saveApplicationInstanceState({
+            id: app.instanceId,
+            applications,
+            metadata: runtimeMetadata
+          }, {
+            headers: await app.getAuthorizationHeader()
+          })
+        } catch (error) {
+          app.log.error('Failed to save application state to Control Plane', error)
+          throw new MetadataStateError()
+        }
+
+        app.log.info('Runtime metadata processed')
+      } catch (error) {
+        if (error.code === 'PLT_METADATA_STATE_ERROR') {
+          throw error
+        }
+        app.log.error(error, 'Failed in getting and processing runtime metadata')
+        throw new MetadataRuntimeError()
+      }
+    } catch (error) {
+      if (error.code === 'PLT_METADATA_APP_ID_ERROR' ||
+          error.code === 'PLT_METADATA_RUNTIME_NOT_STARTED_ERROR' ||
+          error.code === 'PLT_METADATA_RUNTIME_ERROR' ||
+          error.code === 'PLT_METADATA_STATE_ERROR') {
+        throw error
+      }
+      app.log.error(error, 'Failure in metadata processing')
+      throw new MetadataError()
+    }
+  }
+  app.sendMetadata = sendMetadata
+}
+
+export default metadata

package/plugins/scheduler.js

@@ -0,0 +1,48 @@
+async function scheduler (app, _opts) {
+  async function sendSchedulerInfo () {
+    // Skip scheduler configuration if ICC is not configured
+    if (!app.env.PLT_ICC_URL) {
+      app.log.info('PLT_ICC_URL not set, skipping scheduler configuration')
+      return
+    }
+
+    try {
+      const applicationId = app.instanceConfig?.applicationId
+      const runtime = app.wattpro.runtime
+      const config = await runtime.getRuntimeConfig()
+      const { default: build, setDefaultHeaders } = await import('../clients/cron/cron.mjs')
+
+      const cronUrl = app.instanceConfig?.iccServices?.cron?.url
+      if (!cronUrl) {
+        app.log.warn('No cron URL found in ICC services')
+        return
+      }
+      const cronClient = build(cronUrl)
+      setDefaultHeaders(await app.getAuthorizationHeader())
+
+      const jobs = config.scheduler || []
+
+      const saveJobs = []
+      for (const job of jobs) {
+        const iccJob = { ...job, applicationId }
+        iccJob.schedule = iccJob.cron // unfortunately, the ICC API uses `schedule` instead of `cron`
+        delete iccJob.cron
+        delete iccJob.enabled
+        saveJobs.push(cronClient.putWattJobs(iccJob))
+      }
+      const result = await Promise.allSettled(saveJobs)
+      const errors = result.filter((job) => job.status === 'rejected')
+      if (errors.length > 0) {
+        app.log.error(errors, 'Failed to save jobs in ICC')
+        throw new AggregateError('Failed to save jobs in ICC', { cause: errors.map(job => job.reason) })
+      }
+
+      app.log.info('Scheduler configured')
+    } catch (error) {
+      app.log.error(error, 'Failed in configuring watt jobs in ICC')
+    }
+  }
+  app.sendSchedulerInfo = sendSchedulerInfo
+}
+
+export default scheduler