@platformatic/runtime 3.52.2 → 3.53.0-alpha.0

package/lib/runtime.js

@@ -61,6 +61,7 @@ import {
   kHealthCheckTimer,
   kId,
   kInterceptorReadyPromise,
+  kIsSubprocessHost,
   kITC,
   kLastHealthCheckELU,
   kStderrMarker,
@@ -73,6 +74,9 @@ import {
 
 const kWorkerFile = join(import.meta.dirname, 'worker/main.js')
 const kInspectorOptions = Symbol('plt.runtime.worker.inspectorOptions')
+const kHeapCheckCounter = Symbol('plt.runtime.worker.heapCheckCounter')
+const kLastHeapStats = Symbol('plt.runtime.worker.lastHeapStats')
+const kHealthITCTimeoutMs = 5000
 
 const MAX_LISTENERS_COUNT = 100
 
@@ -1515,12 +1519,57 @@ export class Runtime extends EventEmitter {
     return status
   }
 
-  async getWorkerHealth (worker, options = {}) {
-    // Use ITC to get health from the worker. This correctly forwards to child
-    // processes when the service uses startWithCommand (subprocess mode),
-    // instead of reading the coordinator thread's metrics.
-    const { currentELU, heapUsed, heapTotal } = await sendViaITC(worker, 'getHealth')
+  getWorkerHealth (worker, options = {}) {
+    // For subprocess workers we must round-trip through ITC to reach the child;
+    // for pure worker-thread workers we can read ELU/heap directly from the
+    // worker handle, which is served by Node's C++ layer and does not depend
+    // on the worker's event loop being responsive. Going through ITC for
+    // thread workers means a CPU-bound or stuck worker can freeze the whole
+    // health-collection loop, which in turn blocks the management API.
+    if (worker[kIsSubprocessHost]) {
+      return this.#getSubprocessWorkerHealth(worker, options)
+    }
 
+    const currentELU = worker.performance.eventLoopUtilization()
+    const previousELU = options.previousELU
+
+    let elu = currentELU
+    if (previousELU) {
+      elu = worker.performance.eventLoopUtilization(elu, previousELU)
+    }
+
+    if (!features.node.worker.getHeapStatistics) {
+      return { elu: elu.utilization, currentELU }
+    }
+
+    // Only refresh heap statistics every 60 health checks (once per minute).
+    // This keeps the common path fully synchronous — no promise allocation.
+    const counter = (worker[kHeapCheckCounter] ?? 0) + 1
+    worker[kHeapCheckCounter] = counter >= 60 ? 0 : counter
+
+    if (counter >= 60 || !worker[kLastHeapStats]) {
+      return worker.getHeapStatistics().then(({ used_heap_size: heapUsed, total_heap_size: heapTotal }) => {
+        worker[kLastHeapStats] = { heapUsed, heapTotal }
+        return { elu: elu.utilization, heapUsed, heapTotal, currentELU }
+      })
+    }
+
+    const { heapUsed, heapTotal } = worker[kLastHeapStats]
+    return { elu: elu.utilization, heapUsed, heapTotal, currentELU }
+  }
+
+  async #getSubprocessWorkerHealth (worker, options) {
+    // Bound the ITC call so a hung child cannot freeze the health loop.
+    // On timeout we fall back to last-known ELU with a null heap reading so
+    // that the loop keeps rescheduling and signals remain observable.
+    const result = await executeWithTimeout(sendViaITC(worker, 'getHealth'), kHealthITCTimeoutMs, kTimeout)
+
+    if (result === kTimeout) {
+      const previousELU = options.previousELU ?? worker.performance.eventLoopUtilization()
+      return { elu: 1, heapUsed: null, heapTotal: null, currentELU: previousELU }
+    }
+
+    const { currentELU, heapUsed, heapTotal } = result
     const previousELU = options.previousELU
     let elu = currentELU
     if (previousELU) {
@@ -1853,6 +1902,14 @@ export class Runtime extends EventEmitter {
       this.logger.trace({ event, payload, id: workerId, application: applicationId, worker: index }, 'Runtime event')
     })
 
+    // The worker notifies us when its capability has spawned a child process
+    // (e.g. Next.js in dev mode). From that point on health metrics must come
+    // from the child via ITC; for thread-only workers we keep reading the
+    // handle directly in getWorkerHealth().
+    worker[kITC].on('subprocess:started', () => {
+      worker[kIsSubprocessHost] = true
+    })
+
     worker[kITC].on('request:restart', async () => {
       // Do not restart applications that are not fully started yet or when the runtime is still starting.
       // The gateway sends request:restart when it receives application:added events,
@@ -1966,41 +2023,47 @@ export class Runtime extends EventEmitter {
         return
       }
 
-      // Iterate through all workers and collect health metrics
+      // Collect health from all workers in parallel so that a slow ITC
+      // round-trip (e.g. subprocess timeout) does not block every other worker.
+      const pending = []
       for (const worker of this.#workers.values()) {
         if (worker[kWorkerStatus] !== 'started') {
           continue
         }
 
-        const id = worker[kApplicationId]
-        const index = worker[kWorkerId]
-        const errorLabel = this.#workerExtendedLabel(id, index, worker[kConfig].workers)
+        pending.push((async () => {
+          const id = worker[kApplicationId]
+          const index = worker[kWorkerId]
+          const errorLabel = this.#workerExtendedLabel(id, index, worker[kConfig].workers)
 
-        let health = null
-        try {
-          health = await this.getWorkerHealth(worker, {
-            previousELU: worker[kLastHealthCheckELU]
-          })
-        } catch (err) {
-          this.logger.error({ err }, `Failed to get health for ${errorLabel}.`)
-        } finally {
-          worker[kLastHealthCheckELU] = health?.currentELU ?? null
-        }
-
-        const healthSignals = worker[kWorkerHealthSignals]?.getAll() ?? []
+          let health = null
+          try {
+            health = await this.getWorkerHealth(worker, {
+              previousELU: worker[kLastHealthCheckELU]
+            })
+          } catch (err) {
+            this.logger.error({ err }, `Failed to get health for ${errorLabel}.`)
+          } finally {
+            worker[kLastHealthCheckELU] = health?.currentELU ?? null
+          }
 
-        // We use emit instead of emitAndNotify to avoid sending a postMessages
-        // to each workers even if they are not interested in health metrics.
-        // No one of the known capabilities use this event yet.
-        this.emit('application:worker:health:metrics', {
-          id: worker[kId],
-          application: id,
-          worker: index,
-          currentHealth: health,
-          healthSignals
-        })
+          const healthSignals = worker[kWorkerHealthSignals]?.getAll() ?? []
+
+          // We use emit instead of emitAndNotify to avoid sending a postMessages
+          // to each workers even if they are not interested in health metrics.
+          // No one of the known capabilities use this event yet.
+          this.emit('application:worker:health:metrics', {
+            id: worker[kId],
+            application: id,
+            worker: index,
+            currentHealth: health,
+            healthSignals
+          })
+        })())
       }
 
+      await Promise.allSettled(pending)
+
       // Reschedule the next check. We are not using .refresh() because it's more
       // expensive (weird).
       this.#healthMetricsTimer = setTimeout(collectHealthMetrics, 1000).unref()
@@ -2052,53 +2115,45 @@ export class Runtime extends EventEmitter {
     worker[kHealthCheckTimer] = setTimeout(async () => {
       if (worker[kWorkerStatus] !== 'started') return
 
-      if (lastHealthMetrics) {
-        const health = lastHealthMetrics.currentHealth
-        const memoryUsage = health.heapUsed / maxHeapTotal
-        const unhealthy = health.elu > maxELU || memoryUsage > maxHeapUsed
+      // No health data received yet — reschedule and wait.
+      if (!lastHealthMetrics) {
+        worker[kHealthCheckTimer].refresh()
+        return
+      }
+
+      const health = lastHealthMetrics.currentHealth
+
+      // When health collection failed (threw) or timed out, currentHealth is
+      // null.  Treat this as an unhealthy check so that a stuck worker that
+      // cannot even report its own health is eventually replaced.
+      if (!health) {
+        unhealthyChecks++
+
+        this.logger.error(
+          `Health collection failed for the ${errorLabel}. ` +
+            `Unhealthy check ${unhealthyChecks}/${maxUnhealthyChecks}.`
+        )
 
         this.emit('application:worker:health', {
           id: worker[kId],
           application: id,
           worker: index,
-          currentHealth: health,
-          unhealthy,
+          currentHealth: null,
+          unhealthy: true,
           healthConfig
         })
 
-        if (health.elu > maxELU) {
-          this.logger.error(
-            `The ${errorLabel} has an ELU of ${(health.elu * 100).toFixed(2)} %, ` +
-              `above the maximum allowed usage of ${(maxELU * 100).toFixed(2)} %.`
-          )
-        }
-
-        if (memoryUsage > maxHeapUsed) {
-          this.logger.error(
-            `The ${errorLabel} is using ${(memoryUsage * 100).toFixed(2)} % of the memory, ` +
-              `above the maximum allowed usage of ${(maxHeapUsed * 100).toFixed(2)} %.`
-          )
-        }
-
-        if (unhealthy) {
-          unhealthyChecks++
-        } else {
-          unhealthyChecks = 0
-        }
-
         if (unhealthyChecks === maxUnhealthyChecks) {
           try {
             this.emitAndNotify('application:worker:unhealthy', { application: id, worker: index })
 
             this.logger.error(
-              { elu: health.elu, maxELU, memoryUsage: health.heapUsed, maxMemoryUsage: maxHeapUsed },
-              `The ${errorLabel} is unhealthy. Replacing it ...`
+              `The ${errorLabel} is unhealthy (health collection failed). Replacing it ...`
             )
 
             await this.#replaceWorker(config, applicationConfig, workersCount, id, index, worker)
           } catch (e) {
             this.logger.error(
-              { elu: health.elu, maxELU, memoryUsage: health.heapUsed, maxMemoryUsage: maxHeapUsed },
               `Cannot replace the ${errorLabel}. Forcefully terminating it ...`
             )
 
@@ -2107,6 +2162,61 @@ export class Runtime extends EventEmitter {
         } else {
           worker[kHealthCheckTimer].refresh()
         }
+        return
+      }
+
+      const memoryUsage = health.heapUsed != null ? health.heapUsed / maxHeapTotal : 0
+      const unhealthy = health.elu > maxELU || memoryUsage > maxHeapUsed
+
+      this.emit('application:worker:health', {
+        id: worker[kId],
+        application: id,
+        worker: index,
+        currentHealth: health,
+        unhealthy,
+        healthConfig
+      })
+
+      if (health.elu > maxELU) {
+        this.logger.error(
+          `The ${errorLabel} has an ELU of ${(health.elu * 100).toFixed(2)} %, ` +
+            `above the maximum allowed usage of ${(maxELU * 100).toFixed(2)} %.`
+        )
+      }
+
+      if (memoryUsage > maxHeapUsed) {
+        this.logger.error(
+          `The ${errorLabel} is using ${(memoryUsage * 100).toFixed(2)} % of the memory, ` +
+            `above the maximum allowed usage of ${(maxHeapUsed * 100).toFixed(2)} %.`
+        )
+      }
+
+      if (unhealthy) {
+        unhealthyChecks++
+      } else {
+        unhealthyChecks = 0
+      }
+
+      if (unhealthyChecks === maxUnhealthyChecks) {
+        try {
+          this.emitAndNotify('application:worker:unhealthy', { application: id, worker: index })
+
+          this.logger.error(
+            { elu: health.elu, maxELU, memoryUsage: health.heapUsed, maxMemoryUsage: maxHeapUsed },
+            `The ${errorLabel} is unhealthy. Replacing it ...`
+          )
+
+          await this.#replaceWorker(config, applicationConfig, workersCount, id, index, worker)
+        } catch (e) {
+          this.logger.error(
+            { elu: health.elu, maxELU, memoryUsage: health.heapUsed, maxMemoryUsage: maxHeapUsed },
+            `Cannot replace the ${errorLabel}. Forcefully terminating it ...`
+          )
+
+          worker.terminate()
+        }
+      } else {
+        worker[kHealthCheckTimer].refresh()
       }
     }, interval).unref()
   }

package/lib/worker/symbols.js

@@ -12,6 +12,7 @@ export const kInterceptors = Symbol.for('plt.runtime.worker.interceptors')
 export const kLastHealthCheckELU = Symbol.for('plt.runtime.worker.lastHealthCheckELU')
 export const kLastWorkerScalerELU = Symbol.for('plt.runtime.worker.lastWorkerScalerELU')
 export const kInterceptorReadyPromise = Symbol.for('plt.runtime.worker.interceptorReadyPromise')
+export const kIsSubprocessHost = Symbol.for('plt.runtime.worker.isSubprocessHost')
 
 // This string marker should be safe to use since it belongs to Unicode private area
 export const kStderrMarker = '\ue002'

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platformatic/runtime",
-  "version": "3.52.2",
+  "version": "3.53.0-alpha.0",
   "description": "",
   "main": "index.js",
   "type": "module",
@@ -36,14 +36,14 @@
     "typescript": "^5.5.4",
     "undici-oidc-interceptor": "^0.5.0",
     "why-is-node-running": "^2.2.2",
-    "@platformatic/composer": "3.52.2",
-    "@platformatic/gateway": "3.52.2",
-    "@platformatic/db": "3.52.2",
-    "@platformatic/node": "3.52.2",
-    "@platformatic/service": "3.52.2",
-    "@platformatic/sql-graphql": "3.52.2",
-    "@platformatic/sql-mapper": "3.52.2",
-    "@platformatic/wattpm-pprof-capture": "3.52.2"
+    "@platformatic/composer": "3.53.0-alpha.0",
+    "@platformatic/gateway": "3.53.0-alpha.0",
+    "@platformatic/db": "3.53.0-alpha.0",
+    "@platformatic/service": "3.53.0-alpha.0",
+    "@platformatic/sql-graphql": "3.53.0-alpha.0",
+    "@platformatic/node": "3.53.0-alpha.0",
+    "@platformatic/sql-mapper": "3.53.0-alpha.0",
+    "@platformatic/wattpm-pprof-capture": "3.53.0-alpha.0"
   },
   "dependencies": {
     "@fastify/accepts": "^5.0.0",
@@ -73,12 +73,12 @@
     "undici": "^7.0.0",
     "undici-thread-interceptor": "^1.3.1",
     "ws": "^8.16.0",
-    "@platformatic/basic": "3.52.2",
-    "@platformatic/generators": "3.52.2",
-    "@platformatic/foundation": "3.52.2",
-    "@platformatic/metrics": "3.52.2",
-    "@platformatic/itc": "3.52.2",
-    "@platformatic/telemetry": "3.52.2"
+    "@platformatic/foundation": "3.53.0-alpha.0",
+    "@platformatic/basic": "3.53.0-alpha.0",
+    "@platformatic/generators": "3.53.0-alpha.0",
+    "@platformatic/metrics": "3.53.0-alpha.0",
+    "@platformatic/itc": "3.53.0-alpha.0",
+    "@platformatic/telemetry": "3.53.0-alpha.0"
   },
   "engines": {
     "node": ">=22.19.0"

package/schema.json

@@ -1,5 +1,5 @@
 {
-  "$id": "https://schemas.platformatic.dev/@platformatic/runtime/3.52.2.json",
+  "$id": "https://schemas.platformatic.dev/@platformatic/runtime/3.53.0-alpha.0.json",
   "$schema": "http://json-schema.org/draft-07/schema#",
   "title": "Platformatic Runtime Config",
   "type": "object",