@platformatic/runtime 3.52.3 → 3.53.0-alpha.0

package/lib/runtime.js

@@ -2023,41 +2023,47 @@ export class Runtime extends EventEmitter {
         return
       }
 
-      // Iterate through all workers and collect health metrics
+      // Collect health from all workers in parallel so that a slow ITC
+      // round-trip (e.g. subprocess timeout) does not block every other worker.
+      const pending = []
       for (const worker of this.#workers.values()) {
         if (worker[kWorkerStatus] !== 'started') {
           continue
         }
 
-        const id = worker[kApplicationId]
-        const index = worker[kWorkerId]
-        const errorLabel = this.#workerExtendedLabel(id, index, worker[kConfig].workers)
+        pending.push((async () => {
+          const id = worker[kApplicationId]
+          const index = worker[kWorkerId]
+          const errorLabel = this.#workerExtendedLabel(id, index, worker[kConfig].workers)
 
-        let health = null
-        try {
-          health = await this.getWorkerHealth(worker, {
-            previousELU: worker[kLastHealthCheckELU]
-          })
-        } catch (err) {
-          this.logger.error({ err }, `Failed to get health for ${errorLabel}.`)
-        } finally {
-          worker[kLastHealthCheckELU] = health?.currentELU ?? null
-        }
-
-        const healthSignals = worker[kWorkerHealthSignals]?.getAll() ?? []
+          let health = null
+          try {
+            health = await this.getWorkerHealth(worker, {
+              previousELU: worker[kLastHealthCheckELU]
+            })
+          } catch (err) {
+            this.logger.error({ err }, `Failed to get health for ${errorLabel}.`)
+          } finally {
+            worker[kLastHealthCheckELU] = health?.currentELU ?? null
+          }
 
-        // We use emit instead of emitAndNotify to avoid sending a postMessages
-        // to each workers even if they are not interested in health metrics.
-        // No one of the known capabilities use this event yet.
-        this.emit('application:worker:health:metrics', {
-          id: worker[kId],
-          application: id,
-          worker: index,
-          currentHealth: health,
-          healthSignals
-        })
+          const healthSignals = worker[kWorkerHealthSignals]?.getAll() ?? []
+
+          // We use emit instead of emitAndNotify to avoid sending a postMessages
+          // to each workers even if they are not interested in health metrics.
+          // No one of the known capabilities use this event yet.
+          this.emit('application:worker:health:metrics', {
+            id: worker[kId],
+            application: id,
+            worker: index,
+            currentHealth: health,
+            healthSignals
+          })
+        })())
       }
 
+      await Promise.allSettled(pending)
+
       // Reschedule the next check. We are not using .refresh() because it's more
       // expensive (weird).
       this.#healthMetricsTimer = setTimeout(collectHealthMetrics, 1000).unref()
@@ -2109,53 +2115,45 @@ export class Runtime extends EventEmitter {
     worker[kHealthCheckTimer] = setTimeout(async () => {
       if (worker[kWorkerStatus] !== 'started') return
 
-      if (lastHealthMetrics) {
-        const health = lastHealthMetrics.currentHealth
-        const memoryUsage = health.heapUsed / maxHeapTotal
-        const unhealthy = health.elu > maxELU || memoryUsage > maxHeapUsed
+      // No health data received yet — reschedule and wait.
+      if (!lastHealthMetrics) {
+        worker[kHealthCheckTimer].refresh()
+        return
+      }
+
+      const health = lastHealthMetrics.currentHealth
+
+      // When health collection failed (threw) or timed out, currentHealth is
+      // null.  Treat this as an unhealthy check so that a stuck worker that
+      // cannot even report its own health is eventually replaced.
+      if (!health) {
+        unhealthyChecks++
+
+        this.logger.error(
+          `Health collection failed for the ${errorLabel}. ` +
+            `Unhealthy check ${unhealthyChecks}/${maxUnhealthyChecks}.`
+        )
 
         this.emit('application:worker:health', {
           id: worker[kId],
           application: id,
           worker: index,
-          currentHealth: health,
-          unhealthy,
+          currentHealth: null,
+          unhealthy: true,
           healthConfig
         })
 
-        if (health.elu > maxELU) {
-          this.logger.error(
-            `The ${errorLabel} has an ELU of ${(health.elu * 100).toFixed(2)} %, ` +
-              `above the maximum allowed usage of ${(maxELU * 100).toFixed(2)} %.`
-          )
-        }
-
-        if (memoryUsage > maxHeapUsed) {
-          this.logger.error(
-            `The ${errorLabel} is using ${(memoryUsage * 100).toFixed(2)} % of the memory, ` +
-              `above the maximum allowed usage of ${(maxHeapUsed * 100).toFixed(2)} %.`
-          )
-        }
-
-        if (unhealthy) {
-          unhealthyChecks++
-        } else {
-          unhealthyChecks = 0
-        }
-
         if (unhealthyChecks === maxUnhealthyChecks) {
           try {
             this.emitAndNotify('application:worker:unhealthy', { application: id, worker: index })
 
             this.logger.error(
-              { elu: health.elu, maxELU, memoryUsage: health.heapUsed, maxMemoryUsage: maxHeapUsed },
-              `The ${errorLabel} is unhealthy. Replacing it ...`
+              `The ${errorLabel} is unhealthy (health collection failed). Replacing it ...`
             )
 
             await this.#replaceWorker(config, applicationConfig, workersCount, id, index, worker)
           } catch (e) {
             this.logger.error(
-              { elu: health.elu, maxELU, memoryUsage: health.heapUsed, maxMemoryUsage: maxHeapUsed },
               `Cannot replace the ${errorLabel}. Forcefully terminating it ...`
             )
 
@@ -2164,6 +2162,61 @@ export class Runtime extends EventEmitter {
         } else {
           worker[kHealthCheckTimer].refresh()
         }
+        return
+      }
+
+      const memoryUsage = health.heapUsed != null ? health.heapUsed / maxHeapTotal : 0
+      const unhealthy = health.elu > maxELU || memoryUsage > maxHeapUsed
+
+      this.emit('application:worker:health', {
+        id: worker[kId],
+        application: id,
+        worker: index,
+        currentHealth: health,
+        unhealthy,
+        healthConfig
+      })
+
+      if (health.elu > maxELU) {
+        this.logger.error(
+          `The ${errorLabel} has an ELU of ${(health.elu * 100).toFixed(2)} %, ` +
+            `above the maximum allowed usage of ${(maxELU * 100).toFixed(2)} %.`
+        )
+      }
+
+      if (memoryUsage > maxHeapUsed) {
+        this.logger.error(
+          `The ${errorLabel} is using ${(memoryUsage * 100).toFixed(2)} % of the memory, ` +
+            `above the maximum allowed usage of ${(maxHeapUsed * 100).toFixed(2)} %.`
+        )
+      }
+
+      if (unhealthy) {
+        unhealthyChecks++
+      } else {
+        unhealthyChecks = 0
+      }
+
+      if (unhealthyChecks === maxUnhealthyChecks) {
+        try {
+          this.emitAndNotify('application:worker:unhealthy', { application: id, worker: index })
+
+          this.logger.error(
+            { elu: health.elu, maxELU, memoryUsage: health.heapUsed, maxMemoryUsage: maxHeapUsed },
+            `The ${errorLabel} is unhealthy. Replacing it ...`
+          )
+
+          await this.#replaceWorker(config, applicationConfig, workersCount, id, index, worker)
+        } catch (e) {
+          this.logger.error(
+            { elu: health.elu, maxELU, memoryUsage: health.heapUsed, maxMemoryUsage: maxHeapUsed },
+            `Cannot replace the ${errorLabel}. Forcefully terminating it ...`
+          )
+
+          worker.terminate()
+        }
+      } else {
+        worker[kHealthCheckTimer].refresh()
       }
     }, interval).unref()
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platformatic/runtime",
-  "version": "3.52.3",
+  "version": "3.53.0-alpha.0",
   "description": "",
   "main": "index.js",
   "type": "module",
@@ -36,14 +36,14 @@
     "typescript": "^5.5.4",
     "undici-oidc-interceptor": "^0.5.0",
     "why-is-node-running": "^2.2.2",
-    "@platformatic/composer": "3.52.3",
-    "@platformatic/db": "3.52.3",
-    "@platformatic/gateway": "3.52.3",
-    "@platformatic/node": "3.52.3",
-    "@platformatic/service": "3.52.3",
-    "@platformatic/sql-mapper": "3.52.3",
-    "@platformatic/sql-graphql": "3.52.3",
-    "@platformatic/wattpm-pprof-capture": "3.52.3"
+    "@platformatic/composer": "3.53.0-alpha.0",
+    "@platformatic/gateway": "3.53.0-alpha.0",
+    "@platformatic/db": "3.53.0-alpha.0",
+    "@platformatic/service": "3.53.0-alpha.0",
+    "@platformatic/sql-graphql": "3.53.0-alpha.0",
+    "@platformatic/node": "3.53.0-alpha.0",
+    "@platformatic/sql-mapper": "3.53.0-alpha.0",
+    "@platformatic/wattpm-pprof-capture": "3.53.0-alpha.0"
   },
   "dependencies": {
     "@fastify/accepts": "^5.0.0",
@@ -73,12 +73,12 @@
     "undici": "^7.0.0",
     "undici-thread-interceptor": "^1.3.1",
     "ws": "^8.16.0",
-    "@platformatic/basic": "3.52.3",
-    "@platformatic/foundation": "3.52.3",
-    "@platformatic/generators": "3.52.3",
-    "@platformatic/metrics": "3.52.3",
-    "@platformatic/telemetry": "3.52.3",
-    "@platformatic/itc": "3.52.3"
+    "@platformatic/foundation": "3.53.0-alpha.0",
+    "@platformatic/basic": "3.53.0-alpha.0",
+    "@platformatic/generators": "3.53.0-alpha.0",
+    "@platformatic/metrics": "3.53.0-alpha.0",
+    "@platformatic/itc": "3.53.0-alpha.0",
+    "@platformatic/telemetry": "3.53.0-alpha.0"
   },
   "engines": {
     "node": ">=22.19.0"

package/schema.json

@@ -1,5 +1,5 @@
 {
-  "$id": "https://schemas.platformatic.dev/@platformatic/runtime/3.52.3.json",
+  "$id": "https://schemas.platformatic.dev/@platformatic/runtime/3.53.0-alpha.0.json",
   "$schema": "http://json-schema.org/draft-07/schema#",
   "title": "Platformatic Runtime Config",
   "type": "object",