@platformatic/runtime 3.9.0 → 3.11.0

package/config.d.ts

@@ -31,10 +31,39 @@ export type PlatformaticRuntimeConfig = {
           maxHeapTotal?: number | string;
           maxYoungGeneration?: number | string;
         };
-        preload?: string | string[];
         dependencies?: string[];
         arguments?: string[];
+        env?: {
+          [k: string]: string;
+        };
+        envfile?: string;
+        sourceMaps?: boolean;
+        packageManager?: "npm" | "pnpm" | "yarn";
+        preload?: string | string[];
         nodeOptions?: string;
+        permissions?: {
+          fs?: {
+            read?: string[];
+            write?: string[];
+          };
+        };
+        telemetry?: {
+          /**
+           * An array of instrumentations loaded if telemetry is enabled
+           */
+          instrumentations?: (
+            | string
+            | {
+                package: string;
+                exportName?: string;
+                options?: {
+                  [k: string]: unknown;
+                };
+                [k: string]: unknown;
+              }
+          )[];
+          [k: string]: unknown;
+        };
       };
     };
   };
@@ -255,6 +284,7 @@ export type PlatformaticRuntimeConfig = {
               };
             };
         plugins?: string[];
+        timeout?: number | string;
       };
   telemetry?: {
     enabled?: boolean | string;
@@ -335,14 +365,16 @@ export type PlatformaticRuntimeConfig = {
   verticalScaler?: {
     enabled?: boolean;
     maxTotalWorkers?: number;
+    maxTotalMemory?: number;
     minWorkers?: number;
     maxWorkers?: number;
     scaleUpELU?: number;
     scaleDownELU?: number;
-    minELUDiff?: number;
     timeWindowSec?: number;
+    scaleDownTimeWindowSec?: number;
     cooldownSec?: number;
     scaleIntervalSec?: number;
+    gracePeriod?: number;
     applications?: {
       [k: string]: {
         minWorkers?: number;

package/lib/config.js

@@ -45,7 +45,7 @@ function raiseInvalidWorkersError (location, received, hint) {
   throw new InvalidArgumentError(`${location} workers must be a positive integer; received "${received}"${extra}`)
 }
 
-export function autoDetectPprofCapture (config) {
+export function pprofCapturePreloadPath () {
   const require = createRequire(import.meta.url)
 
   let pprofCapturePath
@@ -55,6 +55,12 @@ export function autoDetectPprofCapture (config) {
     // No-op
   }
 
+  return pprofCapturePath
+}
+
+export function autoDetectPprofCapture (config) {
+  const pprofCapturePath = pprofCapturePreloadPath()
+
   // Add to preload if not already present
   if (!config.preload) {
     config.preload = []
@@ -192,14 +198,7 @@ export async function transform (config, _, context) {
         config = join(entryPath, configFilename)
       }
 
-      const application = {
-        id,
-        config,
-        path: entryPath,
-        useHttp: !!mapping.useHttp,
-        health: mapping.health,
-        dependencies: mapping.dependencies
-      }
+      const application = { id, config, path: entryPath, ...mapping }
       const existingApplicationId = applications.findIndex(application => application.id === id)
 
       if (existingApplicationId !== -1) {
@@ -336,6 +335,13 @@ export async function transform (config, _, context) {
     // like adding other applications.
   }
 
+  if (config.metrics === true) {
+    config.metrics = {
+      enabled: true,
+      timeout: 1000
+    }
+  }
+
   config.applications = applications
   config.web = undefined
   config.services = undefined

package/lib/metrics.js

@@ -0,0 +1,73 @@
+import { readFile } from 'node:fs/promises'
+import si from 'systeminformation'
+
+async function readNumberFromCgroupFile (path) {
+  try {
+    const raw = (await readFile(path, 'utf8')).trim()
+    if (raw === 'max') return null
+    return Number(raw)
+  } catch {
+    return null
+  }
+}
+
+async function getCgroupV2MemoryInfo () {
+  let [total, used] = await Promise.all([
+    readNumberFromCgroupFile('/sys/fs/cgroup/memory.max'),
+    readNumberFromCgroupFile('/sys/fs/cgroup/memory.current')
+  ])
+  if (total == null && used == null) return null
+
+  if (total === null) {
+    const mem = await si.mem()
+    total = mem.total
+  }
+
+  return { scope: 'cgroup-v2', used, total }
+}
+
+async function getCgroupV1MemoryInfo () {
+  let [total, used] = await Promise.all([
+    readNumberFromCgroupFile('/sys/fs/cgroup/memory/memory.limit_in_bytes'),
+    readNumberFromCgroupFile('/sys/fs/cgroup/memory/memory.usage_in_bytes')
+  ])
+  if (total == null && used == null) return null
+
+  // Some v1 setups report 9.22e18 (≈unlimited)
+  if (total === null || total > 1e18) {
+    const mem = await si.mem()
+    total = mem.total
+  }
+
+  return { scope: 'cgroup-v1', used, total }
+}
+
+async function readHostMemoryInfo () {
+  const mem = await si.mem()
+  return { scope: 'host', used: mem.active, total: mem.total }
+}
+
+export async function getMemoryInfo (options = {}) {
+  const scope = options.scope
+
+  if (scope === 'cgroup-v2') {
+    return getCgroupV2MemoryInfo()
+  }
+  if (scope === 'cgroup-v1') {
+    return getCgroupV1MemoryInfo()
+  }
+  if (scope === 'host') {
+    return readHostMemoryInfo()
+  }
+
+  let memInfo = await getCgroupV2MemoryInfo()
+
+  if (!memInfo) {
+    memInfo = await getCgroupV1MemoryInfo()
+  }
+  if (!memInfo) {
+    memInfo = await readHostMemoryInfo()
+  }
+
+  return memInfo
+}

package/lib/runtime.js

@@ -9,7 +9,6 @@ import {
   kTimeout,
   parseMemorySize
 } from '@platformatic/foundation'
-import os from 'node:os'
 import { ITC } from '@platformatic/itc'
 import fastify from 'fastify'
 import { EventEmitter, once } from 'node:events'
@@ -17,13 +16,15 @@ import { existsSync } from 'node:fs'
 import { readFile } from 'node:fs/promises'
 import { STATUS_CODES } from 'node:http'
 import { createRequire } from 'node:module'
-import { join } from 'node:path'
+import os from 'node:os'
+import { dirname, isAbsolute, join } from 'node:path'
 import { setImmediate as immediate, setTimeout as sleep } from 'node:timers/promises'
 import { pathToFileURL } from 'node:url'
 import { Worker } from 'node:worker_threads'
 import SonicBoom from 'sonic-boom'
 import { Agent, request, interceptors as undiciInterceptors } from 'undici'
 import { createThreadInterceptor } from 'undici-thread-interceptor'
+import { pprofCapturePreloadPath } from './config.js'
 import {
   ApplicationAlreadyStartedError,
   ApplicationNotFoundError,
@@ -40,12 +41,13 @@ import {
 import { abstractLogger, createLogger } from './logger.js'
 import { startManagementApi } from './management-api.js'
 import { startPrometheusServer } from './prom-server.js'
+import ScalingAlgorithm from './scaling-algorithm.js'
 import { startScheduler } from './scheduler.js'
 import { createSharedStore } from './shared-http-cache.js'
 import { version } from './version.js'
 import { sendViaITC, waitEventFromITC } from './worker/itc.js'
 import { RoundRobinMap } from './worker/round-robin-map.js'
-import ScalingAlgorithm from './scaling-algorithm.js'
+import { getMemoryInfo } from './metrics.js'
 import {
   kApplicationId,
   kConfig,
@@ -57,7 +59,8 @@ import {
   kStderrMarker,
   kWorkerId,
   kWorkersBroadcast,
-  kWorkerStatus
+  kWorkerStatus,
+  kWorkerStartTime
 } from './worker/symbols.js'
 
 const kWorkerFile = join(import.meta.dirname, 'worker/main.js')
@@ -277,7 +280,7 @@ export class Runtime extends EventEmitter {
     }
 
     if (this.#config.verticalScaler?.enabled) {
-      this.#setupVerticalScaler()
+      await this.#setupVerticalScaler()
     }
 
     this.#showUrl()
@@ -891,8 +894,12 @@ export class Runtime extends EventEmitter {
           continue
         }
 
-        const applicationMetrics = await sendViaITC(worker, 'getMetrics', format)
-        if (applicationMetrics) {
+        const applicationMetrics = await executeWithTimeout(
+          sendViaITC(worker, 'getMetrics', format),
+          this.#config.metrics?.timeout ?? 10000
+        )
+
+        if (applicationMetrics && applicationMetrics !== kTimeout) {
           if (metrics === null) {
             metrics = format === 'json' ? [] : ''
           }
@@ -1315,6 +1322,17 @@ export class Runtime extends EventEmitter {
       execArgv.push('--enable-source-maps')
     }
 
+    if (applicationConfig.permissions?.fs) {
+      execArgv.push(...this.#setupPermissions(applicationConfig))
+    }
+
+    let preload = config.preload
+    if (execArgv.includes('--permission')) {
+      // Remove wattpm-pprof-capture from preload since it is not supported
+      const pprofCapturePath = pprofCapturePreloadPath()
+      preload = preload.filter(p => p !== pprofCapturePath)
+    }
+
     const workerEnv = structuredClone(this.#env)
 
     if (applicationConfig.nodeOptions?.trim().length > 0) {
@@ -1337,7 +1355,10 @@ export class Runtime extends EventEmitter {
 
     const worker = new Worker(kWorkerFile, {
       workerData: {
-        config,
+        config: {
+          ...config,
+          preload
+        },
         applicationConfig: {
           ...applicationConfig,
           isProduction: this.#isProduction,
@@ -1642,6 +1663,8 @@ export class Runtime extends EventEmitter {
       }
 
       worker[kWorkerStatus] = 'started'
+      worker[kWorkerStartTime] = Date.now()
+
       this.emitAndNotify('application:worker:started', eventPayload)
       this.#broadcastWorkers()
 
@@ -2441,36 +2464,55 @@ export class Runtime extends EventEmitter {
     }
   }
 
-  #setupVerticalScaler () {
-    const isWorkersFixed = this.#config.workers !== undefined
-    if (isWorkersFixed) return
+  async #setupVerticalScaler () {
+    const fixedWorkersCount = this.#config.workers
+    if (fixedWorkersCount !== undefined) {
+      this.logger.warn(
+        `Vertical scaler disabled because the "workers" configuration is set to ${fixedWorkersCount}`
+      )
+      return
+    }
 
     const scalerConfig = this.#config.verticalScaler
+    const memInfo = await getMemoryInfo()
+    const memScope = memInfo.scope
 
     scalerConfig.maxTotalWorkers ??= os.availableParallelism()
+    scalerConfig.maxTotalMemory ??= memInfo.total * 0.9
     scalerConfig.maxWorkers ??= scalerConfig.maxTotalWorkers
     scalerConfig.minWorkers ??= 1
     scalerConfig.cooldownSec ??= 60
     scalerConfig.scaleUpELU ??= 0.8
     scalerConfig.scaleDownELU ??= 0.2
-    scalerConfig.minELUDiff ??= 0.2
     scalerConfig.scaleIntervalSec ??= 60
-    scalerConfig.timeWindowSec ??= 60
+    scalerConfig.timeWindowSec ??= 10
+    scalerConfig.scaleDownTimeWindowSec ??= 60
+    scalerConfig.gracePeriod ??= 30 * 1000
     scalerConfig.applications ??= {}
 
     const maxTotalWorkers = scalerConfig.maxTotalWorkers
+    const maxTotalMemory = scalerConfig.maxTotalMemory
     const maxWorkers = scalerConfig.maxWorkers
     const minWorkers = scalerConfig.minWorkers
     const cooldown = scalerConfig.cooldownSec
     const scaleUpELU = scalerConfig.scaleUpELU
     const scaleDownELU = scalerConfig.scaleDownELU
-    const minELUDiff = scalerConfig.minELUDiff
     const scaleIntervalSec = scalerConfig.scaleIntervalSec
     const timeWindowSec = scalerConfig.timeWindowSec
+    const scaleDownTimeWindowSec = scalerConfig.scaleDownTimeWindowSec
     const applicationsConfigs = scalerConfig.applications
+    const gracePeriod = scalerConfig.gracePeriod
+    const healthCheckInterval = 1000
+
+    const initialResourcesUpdates = []
 
     for (const application of this.#config.applications) {
       if (application.entrypoint && !features.node.reusePort) {
+        this.logger.warn(
+          `The "${application.id}" application cannot be scaled because it is an entrypoint` +
+          ' and the "reusePort" feature is not available in your OS.'
+        )
+
         applicationsConfigs[application.id] = {
           minWorkers: 1,
           maxWorkers: 1
@@ -2478,6 +2520,10 @@ export class Runtime extends EventEmitter {
         continue
       }
       if (application.workers !== undefined) {
+        this.logger.warn(
+          `The "${application.id}" application cannot be scaled because` +
+          ` it has a fixed number of workers (${application.workers}).`
+        )
         applicationsConfigs[application.id] = {
           minWorkers: application.workers,
           maxWorkers: application.workers
@@ -2488,12 +2534,22 @@ export class Runtime extends EventEmitter {
       applicationsConfigs[application.id] ??= {}
       applicationsConfigs[application.id].minWorkers ??= minWorkers
       applicationsConfigs[application.id].maxWorkers ??= maxWorkers
+
+      const appMinWorkers = applicationsConfigs[application.id].minWorkers
+      if (appMinWorkers > 1) {
+        initialResourcesUpdates.push({
+          application: application.id,
+          workers: minWorkers
+        })
+      }
+    }
+
+    if (initialResourcesUpdates.length > 0) {
+      await this.updateApplicationsResources(initialResourcesUpdates)
     }
 
     for (const applicationId in applicationsConfigs) {
-      const application = this.#config.applications.find(
-        app => app.id === applicationId
-      )
+      const application = this.#config.applications.find(app => app.id === applicationId)
       if (!application) {
         delete applicationsConfigs[applicationId]
 
@@ -2507,23 +2563,50 @@ export class Runtime extends EventEmitter {
       maxTotalWorkers,
       scaleUpELU,
       scaleDownELU,
-      minELUDiff,
-      timeWindowSec,
+      scaleUpTimeWindowSec: timeWindowSec,
+      scaleDownTimeWindowSec,
       applications: applicationsConfigs
     })
 
-    this.on('application:worker:health', async (healthInfo) => {
-      if (!healthInfo) {
-        this.logger.error('No health info received')
-        return
-      }
+    const healthCheckTimeout = setTimeout(async () => {
+      let shouldCheckForScaling = false
+
+      const now = Date.now()
 
-      scalingAlgorithm.addWorkerHealthInfo(healthInfo)
+      for (const worker of this.#workers.values()) {
+        if (
+          worker[kWorkerStatus] !== 'started' ||
+          worker[kWorkerStartTime] + gracePeriod > now
+        ) {
+          continue
+        }
+
+        try {
+          const health = await this.#getHealth(worker)
+          if (!health) continue
+
+          scalingAlgorithm.addWorkerHealthInfo({
+            workerId: worker[kId],
+            applicationId: worker[kApplicationId],
+            elu: health.elu,
+            heapUsed: health.heapUsed,
+            heapTotal: health.heapTotal
+          })
+
+          if (health.elu > scaleUpELU) {
+            shouldCheckForScaling = true
+          }
+        } catch (err) {
+          this.logger.error({ err }, 'Failed to get health for worker')
+        }
+      }
 
-      if (healthInfo.currentHealth.elu > scaleUpELU) {
+      if (shouldCheckForScaling) {
         await checkForScaling()
       }
-    })
+
+      healthCheckTimeout.refresh()
+    }, healthCheckInterval).unref()
 
     let isScaling = false
     let lastScaling = 0
@@ -2535,6 +2618,7 @@ export class Runtime extends EventEmitter {
 
       try {
         const workersInfo = await this.getWorkers()
+        const mem = await getMemoryInfo({ scope: memScope })
 
         const appsWorkersInfo = {}
         for (const worker of Object.values(workersInfo)) {
@@ -2545,19 +2629,22 @@ export class Runtime extends EventEmitter {
           appsWorkersInfo[applicationId]++
         }
 
-        const recommendations = scalingAlgorithm.getRecommendations(appsWorkersInfo)
+        const availableMemory = maxTotalMemory - mem.used
+        const recommendations = scalingAlgorithm.getRecommendations(appsWorkersInfo, {
+          availableMemory
+        })
         if (recommendations.length > 0) {
           await applyRecommendations(recommendations)
+          lastScaling = Date.now()
         }
       } catch (err) {
         this.logger.error({ err }, 'Failed to scale applications')
       } finally {
         isScaling = false
-        lastScaling = Date.now()
       }
     }
 
-    const applyRecommendations = async (recommendations) => {
+    const applyRecommendations = async recommendations => {
       const resourcesUpdates = []
       for (const recommendation of recommendations) {
         const { applicationId, workersCount, direction } = recommendation
@@ -2574,4 +2661,48 @@ export class Runtime extends EventEmitter {
     // Interval for periodic scaling checks
     setInterval(checkForScaling, scaleIntervalSec * 1000).unref()
   }
+
+  #setupPermissions (applicationConfig) {
+    const argv = []
+    const allows = new Set()
+    const { read, write } = applicationConfig.permissions.fs
+
+    if (read?.length) {
+      for (const p of read) {
+        allows.add(`--allow-fs-read=${isAbsolute(p) ? p : join(applicationConfig.path, p)}`)
+      }
+    }
+
+    if (write?.length) {
+      for (const p of write) {
+        allows.add(`--allow-fs-write=${isAbsolute(p) ? p : join(applicationConfig.path, p)}`)
+      }
+    }
+
+    if (allows.size === 0) {
+      return argv
+    }
+
+    // We need to allow read access to the node_modules folder both at the runtime level and at the application level
+    allows.add(`--allow-fs-read=${join(this.#root, 'node_modules', '*')}`)
+    allows.add(`--allow-fs-read=${join(applicationConfig.path, 'node_modules', '*')}`)
+
+    // Since we can't really predict how dependencies are installed (symlinks, pnpm store, and so forth), we also
+    // add any node_modules folder found in the ancestors of the current file
+    let lastPath = import.meta.dirname
+    let currentPath = import.meta.dirname
+
+    do {
+      lastPath = currentPath
+      const nodeModules = join(currentPath, 'node_modules')
+      if (existsSync(nodeModules)) {
+        allows.add(`--allow-fs-read=${join(nodeModules, '*')}`)
+      }
+
+      currentPath = dirname(currentPath)
+    } while (lastPath !== currentPath)
+
+    argv.push('--permission', ...allows)
+    return argv
+  }
 }

package/lib/scaling-algorithm.js

@@ -2,151 +2,201 @@ class ScalingAlgorithm {
   #scaleUpELU
   #scaleDownELU
   #maxTotalWorkers
-  #timeWindowSec
-  #appsELUs
-  #minELUDiff
+  #scaleUpTimeWindowSec
+  #scaleDownTimeWindowSec
+  #appsMetrics
   #appsConfigs
 
   constructor (options = {}) {
     this.#scaleUpELU = options.scaleUpELU ?? 0.8
     this.#scaleDownELU = options.scaleDownELU ?? 0.2
-    this.#maxTotalWorkers = options.maxTotalWorkers
-    this.#minELUDiff = options.minELUDiff ?? 0.2
-    this.#timeWindowSec = options.timeWindowSec ?? 60
+    this.#maxTotalWorkers = options.maxTotalWorkers ?? Infinity
+    this.#scaleUpTimeWindowSec = options.scaleUpTimeWindowSec ?? 10
+    this.#scaleDownTimeWindowSec = options.scaleDownTimeWindowSec ?? 60
     this.#appsConfigs = options.applications ?? {}
 
-    this.#appsELUs = {}
+    this.#appsMetrics = {}
   }
 
   addWorkerHealthInfo (healthInfo) {
-    const workerId = healthInfo.id
-    const applicationId = healthInfo.application
-    const elu = healthInfo.currentHealth.elu
+    const { workerId, applicationId, elu, heapUsed } = healthInfo
     const timestamp = Date.now()
 
-    if (!this.#appsELUs[applicationId]) {
-      this.#appsELUs[applicationId] = {}
+    if (!this.#appsMetrics[applicationId]) {
+      this.#appsMetrics[applicationId] = {}
     }
-    if (!this.#appsELUs[applicationId][workerId]) {
-      this.#appsELUs[applicationId][workerId] = []
+    if (!this.#appsMetrics[applicationId][workerId]) {
+      this.#appsMetrics[applicationId][workerId] = []
     }
-    this.#appsELUs[applicationId][workerId].push({ elu, timestamp })
+    this.#appsMetrics[applicationId][workerId].push({
+      elu,
+      timestamp,
+      heapUsed
+    })
     this.#removeOutdatedAppELUs(applicationId)
   }
 
-  getRecommendations (appsWorkersInfo) {
+  getRecommendations (appsWorkersInfo, options = {}) {
     let totalWorkersCount = 0
-    let appsInfo = []
+    let totalAvailableMemory = options.availableMemory ?? Infinity
+
+    const appsInfo = []
 
     for (const applicationId in appsWorkersInfo) {
       const workersCount = appsWorkersInfo[applicationId]
-      const elu = this.#calculateAppAvgELU(applicationId)
-      appsInfo.push({ applicationId, workersCount, elu })
+
+      const { heapUsed } = this.#calculateAppAvgMetrics(applicationId)
+
+      appsInfo.push({
+        applicationId,
+        workersCount,
+        avgHeapUsed: heapUsed,
+      })
+
       totalWorkersCount += workersCount
     }
 
-    appsInfo = appsInfo.sort(
-      (app1, app2) => {
-        if (app1.elu > app2.elu) return 1
-        if (app1.elu < app2.elu) return -1
-        if (app1.workersCount < app2.workersCount) return 1
-        if (app1.workersCount > app2.workersCount) return -1
-        return 0
-      }
-    )
-
     const recommendations = []
 
-    for (const { applicationId, elu, workersCount } of appsInfo) {
+    for (const { applicationId, workersCount, avgHeapUsed } of appsInfo) {
       const appMinWorkers = this.#appsConfigs[applicationId]?.minWorkers ?? 1
+      const appMaxWorkers = this.#appsConfigs[applicationId]?.maxWorkers ?? this.#maxTotalWorkers
 
-      if (elu < this.#scaleDownELU && workersCount > appMinWorkers) {
+      if (workersCount < appMinWorkers) {
         recommendations.push({
           applicationId,
-          workersCount: workersCount - 1,
-          direction: 'down'
+          workersCount: appMinWorkers,
+          direction: 'up'
         })
-        totalWorkersCount--
+
+        const newWorkersCount = appMinWorkers - workersCount
+        totalWorkersCount += newWorkersCount
+        totalAvailableMemory += newWorkersCount * avgHeapUsed
+        continue
       }
-    }
 
-    for (const scaleUpCandidate of appsInfo.toReversed()) {
-      if (scaleUpCandidate.elu < this.#scaleUpELU) break
+      if (workersCount > appMaxWorkers) {
+        recommendations.push({
+          applicationId,
+          workersCount: appMaxWorkers,
+          direction: 'down'
+        })
 
-      const { applicationId, workersCount } = scaleUpCandidate
+        const removedWorkersCount = workersCount - appMaxWorkers
+        totalWorkersCount -= removedWorkersCount
+        totalAvailableMemory -= removedWorkersCount * avgHeapUsed
+        continue
+      }
 
-      const appMaxWorkers = this.#appsConfigs[applicationId]?.maxWorkers ?? this.#maxTotalWorkers
-      if (workersCount >= appMaxWorkers) continue
-
-      if (totalWorkersCount >= this.#maxTotalWorkers) {
-        let scaleDownCandidate = null
-        for (const app of appsInfo) {
-          const appMinWorkers = this.#appsConfigs[app.applicationId]?.minWorkers ?? 1
-          if (app.workersCount > appMinWorkers) {
-            scaleDownCandidate = app
-            break
-          }
+      if (workersCount > appMinWorkers) {
+        const recommendation = this.#getApplicationScaleRecommendation(applicationId)
+        if (recommendation.recommendation === 'scaleDown') {
+          recommendations.push({
+            applicationId,
+            workersCount: workersCount - 1,
+            direction: 'down'
+          })
+
+          const removedWorkersCount = 1
+          totalWorkersCount -= removedWorkersCount
+          totalAvailableMemory -= removedWorkersCount * avgHeapUsed
         }
+      }
+    }
 
-        if (scaleDownCandidate) {
-          const eluDiff = scaleUpCandidate.elu - scaleDownCandidate.elu
-          const workersDiff = scaleDownCandidate.workersCount - scaleUpCandidate.workersCount
-
-          if (eluDiff >= this.#minELUDiff || workersDiff >= 2) {
-            recommendations.push({
-              applicationId: scaleDownCandidate.applicationId,
-              workersCount: scaleDownCandidate.workersCount - 1,
-              direction: 'down'
-            })
-            recommendations.push({
-              applicationId,
-              workersCount: workersCount + 1,
-              direction: 'up'
-            })
+    if (totalWorkersCount < this.#maxTotalWorkers) {
+      let scaleUpCandidate = null
+
+      for (const { applicationId, workersCount, avgHeapUsed } of appsInfo) {
+        const appMaxWorkers = this.#appsConfigs[applicationId]?.maxWorkers ?? this.#maxTotalWorkers
+        if (workersCount >= appMaxWorkers) continue
+        if (avgHeapUsed >= totalAvailableMemory) continue
+
+        const isScaled = recommendations.some(
+          r => r.applicationId === applicationId
+        )
+        if (isScaled) continue
+
+        const recommendation = this.#getApplicationScaleRecommendation(applicationId)
+        if (recommendation.recommendation !== 'scaleUp') continue
+
+        if (
+          !scaleUpCandidate ||
+          (recommendation.scaleUpELU > scaleUpCandidate.scaleUpELU) ||
+          (recommendation.scaleUpELU === scaleUpCandidate.scaleUpELU &&
+            workersCount < scaleUpCandidate.workersCount
+          )
+        ) {
+          scaleUpCandidate = {
+            applicationId,
+            workersCount,
+            heapUsed: recommendation.avgHeapUsage,
+            elu: recommendation.scaleUpELU
           }
         }
-      } else {
+      }
+
+      if (scaleUpCandidate) {
         recommendations.push({
-          applicationId,
-          workersCount: workersCount + 1,
+          applicationId: scaleUpCandidate.applicationId,
+          workersCount: scaleUpCandidate.workersCount + 1,
           direction: 'up'
         })
         totalWorkersCount++
+        totalAvailableMemory -= scaleUpCandidate.heapUsed
       }
-      break
     }
 
     return recommendations
   }
 
-  #calculateAppAvgELU (applicationId) {
+  #calculateAppAvgMetrics (applicationId, options = {}) {
     this.#removeOutdatedAppELUs(applicationId)
 
-    const appELUs = this.#appsELUs[applicationId]
-    if (!appELUs) return 0
+    const appMetrics = this.#appsMetrics[applicationId]
+    if (!appMetrics) return { elu: 0, heapUsed: 0 }
+
+    const defaultTimeWindow = this.#getMetricsTimeWindow()
+    const timeWindow = options.timeWindow ?? defaultTimeWindow
 
     let eluSum = 0
-    let eluCount = 0
+    let heapUsedSum = 0
+    let count = 0
 
-    for (const workerId in appELUs) {
-      const workerELUs = appELUs[workerId]
-      const workerELUSum = workerELUs.reduce(
-        (sum, workerELU) => sum + workerELU.elu, 0
-      )
-      eluSum += workerELUSum / workerELUs.length
-      eluCount++
-    }
+    const now = Date.now()
+
+    for (const workerId in appMetrics) {
+      const workerMetrics = appMetrics[workerId]
+
+      let workerELUSum = 0
+      let workerHeapUsedSum = 0
+      let metricCount = 0
+
+      for (const metric of workerMetrics) {
+        if (metric.timestamp < now - timeWindow) continue
+        workerELUSum += metric.elu
+        workerHeapUsedSum += metric.heapUsed
+        metricCount++
+      }
+
+      if (metricCount === 0) continue
 
-    if (eluCount === 0) return 0
+      eluSum += workerELUSum / metricCount
+      heapUsedSum += workerHeapUsedSum / metricCount
+      count++
+    }
 
-    return Math.round(eluSum / eluCount * 100) / 100
+    const elu = Math.round(eluSum / count * 100) / 100
+    const heapUsed = Math.round(heapUsedSum / count * 100) / 100
+    return { elu, heapUsed }
   }
 
   #removeOutdatedAppELUs (applicationId) {
-    const appELUs = this.#appsELUs[applicationId]
+    const appELUs = this.#appsMetrics[applicationId]
     if (!appELUs) return
 
     const now = Date.now()
+    const timeWindow = this.#getMetricsTimeWindow()
 
     for (const workerId in appELUs) {
       const workerELUs = appELUs[workerId]
@@ -154,7 +204,7 @@ class ScalingAlgorithm {
       let firstValidIndex = -1
       for (let i = 0; i < workerELUs.length; i++) {
         const timestamp = workerELUs[i].timestamp
-        if (timestamp >= now - this.#timeWindowSec * 1000) {
+        if (timestamp >= now - timeWindow) {
           firstValidIndex = i
           break
         }
@@ -174,6 +224,30 @@ class ScalingAlgorithm {
       }
     }
   }
+
+  #getMetricsTimeWindow () {
+    return Math.max(this.#scaleUpTimeWindowSec, this.#scaleDownTimeWindowSec) * 1000
+  }
+
+  #getApplicationScaleRecommendation (applicationId) {
+    const { elu: scaleUpELU } = this.#calculateAppAvgMetrics(applicationId, {
+      timeWindow: this.#scaleUpTimeWindowSec * 1000
+    })
+    const { elu: scaleDownELU } = this.#calculateAppAvgMetrics(applicationId, {
+      timeWindow: this.#scaleDownTimeWindowSec * 1000
+    })
+    const { heapUsed: avgHeapUsage } = this.#calculateAppAvgMetrics(applicationId)
+
+    let recommendation = null
+    if (scaleUpELU > this.#scaleUpELU) {
+      recommendation = 'scaleUp'
+    }
+    if (scaleDownELU < this.#scaleDownELU) {
+      recommendation = 'scaleDown'
+    }
+
+    return { recommendation, scaleUpELU, scaleDownELU, avgHeapUsage }
+  }
 }
 
 export default ScalingAlgorithm

package/lib/worker/symbols.js

@@ -6,6 +6,7 @@ export const kWorkerId = Symbol.for('plt.runtime.worker.id')
 export const kITC = Symbol.for('plt.runtime.itc')
 export const kHealthCheckTimer = Symbol.for('plt.runtime.worker.healthCheckTimer')
 export const kWorkerStatus = Symbol('plt.runtime.worker.status')
+export const kWorkerStartTime = Symbol.for('plt.runtime.worker.startTime')
 export const kInterceptors = Symbol.for('plt.runtime.worker.interceptors')
 export const kLastELU = Symbol.for('plt.runtime.worker.lastELU')
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platformatic/runtime",
-  "version": "3.9.0",
+  "version": "3.11.0",
   "description": "",
   "main": "index.js",
   "type": "module",
@@ -18,8 +18,8 @@
     "@fastify/compress": "^8.0.0",
     "@fastify/express": "^4.0.0",
     "@fastify/formbody": "^8.0.0",
-    "autocannon": "^8.0.0",
     "atomic-sleep": "^1.0.0",
+    "autocannon": "^8.0.0",
     "c8": "^10.0.0",
     "cleaner-spec-reporter": "^0.5.0",
     "eslint": "9",
@@ -35,14 +35,14 @@
     "typescript": "^5.5.4",
     "undici-oidc-interceptor": "^0.5.0",
     "why-is-node-running": "^2.2.2",
-    "@platformatic/composer": "3.9.0",
-    "@platformatic/db": "3.9.0",
-    "@platformatic/gateway": "3.9.0",
-    "@platformatic/sql-graphql": "3.9.0",
-    "@platformatic/node": "3.9.0",
-    "@platformatic/sql-mapper": "3.9.0",
-    "@platformatic/service": "3.9.0",
-    "@platformatic/wattpm-pprof-capture": "3.9.0"
+    "@platformatic/composer": "3.11.0",
+    "@platformatic/db": "3.11.0",
+    "@platformatic/gateway": "3.11.0",
+    "@platformatic/node": "3.11.0",
+    "@platformatic/service": "3.11.0",
+    "@platformatic/sql-mapper": "3.11.0",
+    "@platformatic/wattpm-pprof-capture": "3.11.0",
+    "@platformatic/sql-graphql": "3.11.0"
   },
   "dependencies": {
     "@fastify/accepts": "^5.0.0",
@@ -69,15 +69,16 @@
     "prom-client": "^15.1.2",
     "semgrator": "^0.3.0",
     "sonic-boom": "^4.2.0",
+    "systeminformation": "^5.27.11",
     "undici": "^7.0.0",
     "undici-thread-interceptor": "^0.14.0",
     "ws": "^8.16.0",
-    "@platformatic/foundation": "3.9.0",
-    "@platformatic/generators": "3.9.0",
-    "@platformatic/itc": "3.9.0",
-    "@platformatic/basic": "3.9.0",
-    "@platformatic/metrics": "3.9.0",
-    "@platformatic/telemetry": "3.9.0"
+    "@platformatic/basic": "3.11.0",
+    "@platformatic/foundation": "3.11.0",
+    "@platformatic/itc": "3.11.0",
+    "@platformatic/generators": "3.11.0",
+    "@platformatic/metrics": "3.11.0",
+    "@platformatic/telemetry": "3.11.0"
   },
   "engines": {
     "node": ">=22.19.0"

package/schema.json

@@ -1,5 +1,5 @@
 {
-  "$id": "https://schemas.platformatic.dev/@platformatic/runtime/3.9.0.json",
+  "$id": "https://schemas.platformatic.dev/@platformatic/runtime/3.11.0.json",
   "$schema": "http://json-schema.org/draft-07/schema#",
   "title": "Platformatic Runtime Config",
   "type": "object",
@@ -171,6 +171,40 @@
                 },
                 "additionalProperties": false
               },
+              "dependencies": {
+                "type": "array",
+                "items": {
+                  "type": "string"
+                },
+                "default": []
+              },
+              "arguments": {
+                "type": "array",
+                "items": {
+                  "type": "string"
+                }
+              },
+              "env": {
+                "type": "object",
+                "additionalProperties": {
+                  "type": "string"
+                }
+              },
+              "envfile": {
+                "type": "string"
+              },
+              "sourceMaps": {
+                "type": "boolean",
+                "default": false
+              },
+              "packageManager": {
+                "type": "string",
+                "enum": [
+                  "npm",
+                  "pnpm",
+                  "yarn"
+                ]
+              },
               "preload": {
                 "anyOf": [
                   {
@@ -186,20 +220,66 @@
                   }
                 ]
               },
-              "dependencies": {
-                "type": "array",
-                "items": {
-                  "type": "string"
-                }
-              },
-              "arguments": {
-                "type": "array",
-                "items": {
-                  "type": "string"
-                }
-              },
               "nodeOptions": {
                 "type": "string"
+              },
+              "permissions": {
+                "type": "object",
+                "properties": {
+                  "fs": {
+                    "type": "object",
+                    "properties": {
+                      "read": {
+                        "type": "array",
+                        "items": {
+                          "type": "string"
+                        }
+                      },
+                      "write": {
+                        "type": "array",
+                        "items": {
+                          "type": "string"
+                        }
+                      }
+                    },
+                    "additionalProperties": false
+                  }
+                },
+                "additionalProperties": false
+              },
+              "telemetry": {
+                "type": "object",
+                "properties": {
+                  "instrumentations": {
+                    "type": "array",
+                    "description": "An array of instrumentations loaded if telemetry is enabled",
+                    "items": {
+                      "oneOf": [
+                        {
+                          "type": "string"
+                        },
+                        {
+                          "type": "object",
+                          "properties": {
+                            "package": {
+                              "type": "string"
+                            },
+                            "exportName": {
+                              "type": "string"
+                            },
+                            "options": {
+                              "type": "object",
+                              "additionalProperties": true
+                            }
+                          },
+                          "required": [
+                            "package"
+                          ]
+                        }
+                      ]
+                    }
+                  }
+                }
               }
             }
           }
@@ -405,6 +485,30 @@
           "nodeOptions": {
             "type": "string"
           },
+          "permissions": {
+            "type": "object",
+            "properties": {
+              "fs": {
+                "type": "object",
+                "properties": {
+                  "read": {
+                    "type": "array",
+                    "items": {
+                      "type": "string"
+                    }
+                  },
+                  "write": {
+                    "type": "array",
+                    "items": {
+                      "type": "string"
+                    }
+                  }
+                },
+                "additionalProperties": false
+              }
+            },
+            "additionalProperties": false
+          },
           "telemetry": {
             "type": "object",
             "properties": {
@@ -641,6 +745,30 @@
           "nodeOptions": {
             "type": "string"
           },
+          "permissions": {
+            "type": "object",
+            "properties": {
+              "fs": {
+                "type": "object",
+                "properties": {
+                  "read": {
+                    "type": "array",
+                    "items": {
+                      "type": "string"
+                    }
+                  },
+                  "write": {
+                    "type": "array",
+                    "items": {
+                      "type": "string"
+                    }
+                  }
+                },
+                "additionalProperties": false
+              }
+            },
+            "additionalProperties": false
+          },
           "telemetry": {
             "type": "object",
             "properties": {
@@ -877,6 +1005,30 @@
           "nodeOptions": {
             "type": "string"
           },
+          "permissions": {
+            "type": "object",
+            "properties": {
+              "fs": {
+                "type": "object",
+                "properties": {
+                  "read": {
+                    "type": "array",
+                    "items": {
+                      "type": "string"
+                    }
+                  },
+                  "write": {
+                    "type": "array",
+                    "items": {
+                      "type": "string"
+                    }
+                  }
+                },
+                "additionalProperties": false
+              }
+            },
+            "additionalProperties": false
+          },
           "telemetry": {
             "type": "object",
             "properties": {
@@ -1691,6 +1843,17 @@
                   }
                 ]
               }
+            },
+            "timeout": {
+              "anyOf": [
+                {
+                  "type": "integer"
+                },
+                {
+                  "type": "string"
+                }
+              ],
+              "default": 10000
             }
           },
           "additionalProperties": false
@@ -1838,6 +2001,10 @@
           "type": "number",
           "minimum": 1
         },
+        "maxTotalMemory": {
+          "type": "number",
+          "minimum": 0
+        },
         "minWorkers": {
           "type": "number",
           "minimum": 1
@@ -1856,12 +2023,11 @@
           "minimum": 0,
           "maximum": 1
         },
-        "minELUDiff": {
+        "timeWindowSec": {
           "type": "number",
-          "minimum": 0,
-          "maximum": 1
+          "minimum": 0
         },
-        "timeWindowSec": {
+        "scaleDownTimeWindowSec": {
           "type": "number",
           "minimum": 0
         },
@@ -1873,6 +2039,10 @@
           "type": "number",
           "minimum": 0
         },
+        "gracePeriod": {
+          "type": "number",
+          "minimum": 0
+        },
         "applications": {
           "type": "object",
           "additionalProperties": {