@platformatic/runtime 2.9.1 → 2.10.0

package/config.d.ts

@@ -5,7 +5,7 @@
  * and run json-schema-to-typescript to regenerate this file.
  */
 
-export type HttpsSchemasPlatformaticDevPlatformaticRuntime291Json = {
+export type HttpsSchemasPlatformaticDevPlatformaticRuntime2100Json = {
   [k: string]: unknown;
 } & {
   $schema?: string;
@@ -20,6 +20,16 @@ export type HttpsSchemasPlatformaticDevPlatformaticRuntime291Json = {
         id: string;
         config?: string;
         useHttp?: boolean;
+        workers?: number | string;
+        health?: {
+          enabled?: boolean | string;
+          interval?: number | string;
+          gracePeriod?: number | string;
+          maxUnhealthyChecks?: number | string;
+          maxELU?: number | string;
+          maxHeapUsed?: number | string;
+          maxHeapTotal?: number | string;
+        };
       };
     };
   };
@@ -52,8 +62,6 @@ export type HttpsSchemasPlatformaticDevPlatformaticRuntime291Json = {
               [k: string]: unknown;
             };
             level?: string;
-            additionalProperties?: never;
-            [k: string]: unknown;
           }[];
           options?: {
             [k: string]: unknown;
@@ -101,6 +109,15 @@ export type HttpsSchemasPlatformaticDevPlatformaticRuntime291Json = {
     runtime: number | string;
     service: number | string;
   };
+  health?: {
+    enabled?: boolean | string;
+    interval?: number | string;
+    gracePeriod?: number | string;
+    maxUnhealthyChecks?: number | string;
+    maxELU?: number | string;
+    maxHeapUsed?: number | string;
+    maxHeapTotal?: number | string;
+  };
   undici?: {
     agentOptions?: {
       [k: string]: unknown;

package/lib/errors.js

@@ -17,6 +17,7 @@ module.exports = {
   FailedToRetrieveGraphQLSchemaError: createError(`${ERROR_PREFIX}_FAILED_TO_RETRIEVE_GRAPHQL_SCHEMA`, 'Failed to retrieve GraphQL schema for service with id "%s": %s'),
   FailedToRetrieveMetaError: createError(`${ERROR_PREFIX}_FAILED_TO_RETRIEVE_META`, 'Failed to retrieve metadata for service with id "%s": %s'),
   FailedToRetrieveMetricsError: createError(`${ERROR_PREFIX}_FAILED_TO_RETRIEVE_METRICS`, 'Failed to retrieve metrics for service with id "%s": %s'),
+  FailedToRetrieveHealthError: createError(`${ERROR_PREFIX}_FAILED_TO_RETRIEVE_HEALTH`, 'Failed to retrieve health for service with id "%s": %s'),
   ApplicationAlreadyStartedError: createError(`${ERROR_PREFIX}_APPLICATION_ALREADY_STARTED`, 'Application is already started'),
   ApplicationNotStartedError: createError(`${ERROR_PREFIX}_APPLICATION_NOT_STARTED`, 'Application has not been started'),
   ConfigPathMustBeStringError: createError(`${ERROR_PREFIX}_CONFIG_PATH_MUST_BE_STRING`, 'Config path must be a string'),

package/lib/runtime.js

@@ -7,7 +7,7 @@ const { join } = require('node:path')
 const { setTimeout: sleep } = require('node:timers/promises')
 const { Worker } = require('node:worker_threads')
 const { ITC } = require('@platformatic/itc')
-const { ensureLoggableError, executeWithTimeout } = require('@platformatic/utils')
+const { ensureLoggableError, executeWithTimeout, deepmerge } = require('@platformatic/utils')
 const ts = require('tail-file-stream')
 const { createThreadInterceptor } = require('undici-thread-interceptor')
 
@@ -24,6 +24,7 @@ const {
   kServiceId,
   kWorkerId,
   kITC,
+  kHealthCheckTimer,
   kConfig,
   kLoggerDestination,
   kLoggingPort,
@@ -763,6 +764,9 @@ class Runtime extends EventEmitter {
       }
     }
 
+    const errorLabel = this.#workerExtendedLabel(serviceId, index, workersCount)
+    const health = deepmerge(config.health ?? {}, serviceConfig.health ?? {})
+
     const worker = new Worker(kWorkerFile, {
       workerData: {
         config,
@@ -783,6 +787,9 @@ class Runtime extends EventEmitter {
       execArgv: serviceConfig.isPLTService ? [] : ['--require', openTelemetrySetupPath],
       env: this.#env,
       transferList: [loggingPort],
+      resourceLimits: {
+        maxOldGenerationSizeMb: health.maxHeapTotal
+      },
       /*
         Important: always set stdout and stderr to true, so that worker's output is not automatically
         piped to the parent thread. We actually never output the thread output since we replace it
@@ -814,8 +821,6 @@ class Runtime extends EventEmitter {
 
       // Wait for the next tick so that crashed from the thread are logged first
       setImmediate(() => {
-        const errorLabel = workersCount > 1 ? `worker ${index} of the service "${serviceId}"` : `service "${serviceId}"`
-
         if (started && (!config.watch || code !== 0)) {
           this.logger.warn(`The ${errorLabel} unexpectedly exited with code ${code}.`)
         }
@@ -837,7 +842,6 @@ class Runtime extends EventEmitter {
     worker[kId] = workersCount > 1 ? workerId : serviceId
     worker[kServiceId] = serviceId
     worker[kWorkerId] = workersCount > 1 ? index : undefined
-    worker[kConfig] = serviceConfig
     worker[kLoggerDestination] = loggerDestination
     worker[kLoggingPort] = loggingPort
 
@@ -900,7 +904,6 @@ class Runtime extends EventEmitter {
 
     // Store dependencies
     const [{ dependencies }] = await waitEventFromITC(worker, 'init')
-    worker[kWorkerStatus] = 'boot'
 
     if (autoload) {
       serviceConfig.dependencies = dependencies
@@ -910,11 +913,50 @@ class Runtime extends EventEmitter {
         }
       }
     }
+
+    // This must be done here as the dependencies are filled above
+    worker[kConfig] = { ...serviceConfig, health }
+    worker[kWorkerStatus] = 'boot'
+  }
+
+  #setupHealthCheck (worker, errorLabel) {
+    // Clear the timeout when exiting
+    worker.on('exit', () => clearTimeout(worker[kHealthCheckTimer]))
+
+    const { maxELU, maxHeapUsed, maxHeapTotal, maxUnhealthyChecks, interval } = worker[kConfig].health
+    let unhealthyChecks = 0
+
+    worker[kHealthCheckTimer] = setTimeout(async () => {
+      const health = await worker[kITC].send('getHealth')
+      this.emit('health', {
+        id: worker[kId],
+        service: worker[kServiceId],
+        worker: worker[kWorkerId],
+        currentHealth: health,
+        healthConfig: worker[kConfig].health
+      })
+
+      const { elu, heapUsed } = health
+      const memoryUsage = heapUsed / maxHeapTotal
+      if (elu > maxELU || memoryUsage > maxHeapUsed) {
+        unhealthyChecks++
+      } else {
+        unhealthyChecks = 0
+      }
+
+      if (unhealthyChecks >= maxUnhealthyChecks) {
+        this.logger.error(`The ${errorLabel} is unhealthy. Forcefully terminating it ...`)
+        worker.terminate()
+        return
+      }
+
+      worker[kHealthCheckTimer].refresh()
+    }, interval)
   }
 
   async #startWorker (config, serviceConfig, workersCount, id, index, silent, bootstrapAttempt = 0) {
     const workerId = `${id}:${index}`
-    const label = workersCount > 1 ? `worker ${index} of the service "${id}"` : `service "${id}"`
+    const label = this.#workerExtendedLabel(id, index, workersCount)
 
     if (!silent) {
       this.logger?.info(`Starting the ${label}...`)
@@ -941,6 +983,13 @@ class Runtime extends EventEmitter {
       if (!silent) {
         this.logger?.info(`Started the ${label}...`)
       }
+
+      const { enabled, gracePeriod } = worker[kConfig].health
+      if (enabled && config.restartOnError > 0) {
+        worker[kHealthCheckTimer] = setTimeout(() => {
+          this.#setupHealthCheck(worker, label)
+        }, gracePeriod > 0 ? gracePeriod : 1)
+      }
     } catch (error) {
       // TODO: handle port allocation error here
       if (error.code === 'EADDRINUSE') throw error
@@ -983,7 +1032,7 @@ class Runtime extends EventEmitter {
 
     worker[kWorkerStatus] = 'stopping'
 
-    const label = workersCount > 1 ? `worker ${index} of the service "${id}"` : `service "${id}"`
+    const label = this.#workerExtendedLabel(id, index, workersCount)
 
     if (!silent) {
       this.logger?.info(`Stopping the ${label}...`)
@@ -1022,6 +1071,11 @@ class Runtime extends EventEmitter {
     worker[kITC].close()
     worker[kLoggerDestination].close()
     worker[kLoggingPort].close()
+    clearTimeout(worker[kHealthCheckTimer])
+  }
+
+  #workerExtendedLabel (serviceId, workerId, workersCount) {
+    return workersCount > 1 ? `worker ${workerId} of the service "${serviceId}"` : `service "${serviceId}"`
   }
 
   async #restartCrashedWorker (config, serviceConfig, workersCount, id, index, silent, bootstrapAttempt) {
@@ -1037,6 +1091,12 @@ class Runtime extends EventEmitter {
       setTimeout(async () => {
         this.#restartingWorkers.delete(workerId)
 
+        // If some processes were scheduled to restart
+        // but the runtime is stopped, ignore it
+        if (!this.#status.startsWith('start')) {
+          return
+        }
+
         try {
           await this.#setupWorker(config, serviceConfig, workersCount, id, index)
           await this.#startWorker(config, serviceConfig, workersCount, id, index, silent, bootstrapAttempt)

package/lib/schema.js

@@ -3,7 +3,7 @@
 
 const telemetry = require('@platformatic/telemetry').schema
 const {
-  schemaComponents: { server, logger }
+  schemaComponents: { server, logger, health }
 } = require('@platformatic/utils')
 
 const workers = {
@@ -38,7 +38,8 @@ const services = {
       useHttp: {
         type: 'boolean'
       },
-      workers
+      workers,
+      health: { ...health, default: undefined }
     }
   }
 }
@@ -94,7 +95,9 @@ const platformaticRuntimeSchema = {
               },
               useHttp: {
                 type: 'boolean'
-              }
+              },
+              workers,
+              health: { ...health, default: undefined }
             }
           }
         }
@@ -143,6 +146,7 @@ const platformaticRuntimeSchema = {
       required: ['runtime', 'service'],
       additionalProperties: false
     },
+    health,
     undici: {
       type: 'object',
       properties: {

package/lib/worker/app.js

@@ -3,6 +3,7 @@
 const { existsSync } = require('node:fs')
 const { EventEmitter } = require('node:events')
 const { resolve } = require('node:path')
+const { performance: { eventLoopUtilization } } = require('node:perf_hooks')
 const { workerData } = require('node:worker_threads')
 const { ConfigManager } = require('@platformatic/config')
 const { FileWatcher } = require('@platformatic/utils')
@@ -21,6 +22,7 @@ class PlatformaticApp extends EventEmitter {
   #fileWatcher
   #debouncedRestart
   #context
+  #lastELU
 
   constructor (
     appConfig,
@@ -42,6 +44,7 @@ class PlatformaticApp extends EventEmitter {
     this.#listening = false
     this.stackable = null
     this.#fileWatcher = null
+    this.#lastELU = eventLoopUtilization()
 
     this.#context = {
       serviceId: this.serviceId,
@@ -209,6 +212,20 @@ class PlatformaticApp extends EventEmitter {
     return this.stackable.getMetrics({ format })
   }
 
+  async getHealth () {
+    const currentELU = eventLoopUtilization()
+    const elu = eventLoopUtilization(currentELU, this.#lastELU).utilization
+    this.#lastELU = currentELU
+
+    const { heapUsed, heapTotal } = process.memoryUsage()
+
+    return {
+      elu,
+      heapUsed,
+      heapTotal
+    }
+  }
+
   #fetchServiceUrl (key, { parent, context: service }) {
     if (service.localServiceEnvVars.has(key)) {
       return service.localServiceEnvVars.get(key)

package/lib/worker/itc.js

@@ -150,6 +150,14 @@ function setupITC (app, service, dispatcher) {
         }
       },
 
+      async getHealth () {
+        try {
+          return await app.getHealth()
+        } catch (err) {
+          throw new errors.FailedToRetrieveHealthError(service.id, err.message)
+        }
+      },
+
       inject (injectParams) {
         return app.stackable.inject(injectParams)
       }

package/lib/worker/symbols.js

@@ -5,8 +5,19 @@ const kId = Symbol.for('plt.runtime.id') // This is also used to detect if we ar
 const kServiceId = Symbol.for('plt.runtime.service.id')
 const kWorkerId = Symbol.for('plt.runtime.worker.id')
 const kITC = Symbol.for('plt.runtime.itc')
+const kHealthCheckTimer = Symbol.for('plt.runtime.worker.healthCheckTimer')
 const kLoggerDestination = Symbol.for('plt.runtime.loggerDestination')
 const kLoggingPort = Symbol.for('plt.runtime.logginPort')
 const kWorkerStatus = Symbol('plt.runtime.worker.status')
 
-module.exports = { kConfig, kId, kServiceId, kWorkerId, kITC, kLoggerDestination, kLoggingPort, kWorkerStatus }
+module.exports = {
+  kConfig,
+  kId,
+  kServiceId,
+  kWorkerId,
+  kITC,
+  kHealthCheckTimer,
+  kLoggerDestination,
+  kLoggingPort,
+  kWorkerStatus
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platformatic/runtime",
-  "version": "2.9.1",
+  "version": "2.10.0",
   "description": "",
   "main": "index.js",
   "bin": {
@@ -35,12 +35,12 @@
     "typescript": "^5.5.4",
     "undici-oidc-interceptor": "^0.5.0",
     "why-is-node-running": "^2.2.2",
-    "@platformatic/db": "2.9.1",
-    "@platformatic/composer": "2.9.1",
-    "@platformatic/service": "2.9.1",
-    "@platformatic/node": "2.9.1",
-    "@platformatic/sql-graphql": "2.9.1",
-    "@platformatic/sql-mapper": "2.9.1"
+    "@platformatic/composer": "2.10.0",
+    "@platformatic/db": "2.10.0",
+    "@platformatic/node": "2.10.0",
+    "@platformatic/service": "2.10.0",
+    "@platformatic/sql-mapper": "2.10.0",
+    "@platformatic/sql-graphql": "2.10.0"
   },
   "dependencies": {
     "@fastify/error": "^4.0.0",
@@ -62,22 +62,22 @@
     "help-me": "^5.0.0",
     "minimist": "^1.2.8",
     "pino": "^8.19.0",
-    "pino-pretty": "^11.0.0",
+    "pino-pretty": "^12.0.0",
     "pino-roll": "^2.0.0",
     "prom-client": "^15.1.2",
     "semgrator": "^0.3.0",
     "tail-file-stream": "^0.2.0",
     "thread-cpu-usage": "^0.2.0",
     "undici": "^6.9.0",
-    "undici-thread-interceptor": "^0.7.0",
+    "undici-thread-interceptor": "^0.8.0",
     "ws": "^8.16.0",
-    "@platformatic/basic": "2.9.1",
-    "@platformatic/config": "2.9.1",
-    "@platformatic/itc": "2.9.1",
-    "@platformatic/ts-compiler": "2.9.1",
-    "@platformatic/generators": "2.9.1",
-    "@platformatic/utils": "2.9.1",
-    "@platformatic/telemetry": "2.9.1"
+    "@platformatic/basic": "2.10.0",
+    "@platformatic/generators": "2.10.0",
+    "@platformatic/config": "2.10.0",
+    "@platformatic/itc": "2.10.0",
+    "@platformatic/telemetry": "2.10.0",
+    "@platformatic/utils": "2.10.0",
+    "@platformatic/ts-compiler": "2.10.0"
   },
   "scripts": {
     "test": "npm run lint && borp --concurrency=1 --timeout=300000 && tsd",

package/schema.json

@@ -1,5 +1,5 @@
 {
-  "$id": "https://schemas.platformatic.dev/@platformatic/runtime/2.9.1.json",
+  "$id": "https://schemas.platformatic.dev/@platformatic/runtime/2.10.0.json",
   "$schema": "http://json-schema.org/draft-07/schema#",
   "type": "object",
   "properties": {
@@ -51,6 +51,108 @@
               },
               "useHttp": {
                 "type": "boolean"
+              },
+              "workers": {
+                "anyOf": [
+                  {
+                    "type": "number",
+                    "minimum": 1
+                  },
+                  {
+                    "type": "string"
+                  }
+                ]
+              },
+              "health": {
+                "type": "object",
+                "properties": {
+                  "enabled": {
+                    "default": true,
+                    "anyOf": [
+                      {
+                        "type": "boolean"
+                      },
+                      {
+                        "type": "string"
+                      }
+                    ]
+                  },
+                  "interval": {
+                    "default": 30000,
+                    "anyOf": [
+                      {
+                        "type": "number",
+                        "minimum": 0
+                      },
+                      {
+                        "type": "string"
+                      }
+                    ]
+                  },
+                  "gracePeriod": {
+                    "default": 30000,
+                    "anyOf": [
+                      {
+                        "type": "number",
+                        "minimum": 0
+                      },
+                      {
+                        "type": "string"
+                      }
+                    ]
+                  },
+                  "maxUnhealthyChecks": {
+                    "default": 3,
+                    "anyOf": [
+                      {
+                        "type": "number",
+                        "minimum": 1
+                      },
+                      {
+                        "type": "string"
+                      }
+                    ]
+                  },
+                  "maxELU": {
+                    "default": 0.95,
+                    "anyOf": [
+                      {
+                        "type": "number",
+                        "minimum": 0,
+                        "maximum": 1
+                      },
+                      {
+                        "type": "string"
+                      }
+                    ]
+                  },
+                  "maxHeapUsed": {
+                    "default": 0.95,
+                    "anyOf": [
+                      {
+                        "type": "number",
+                        "minimum": 0,
+                        "maximum": 1
+                      },
+                      {
+                        "type": "string"
+                      }
+                    ]
+                  },
+                  "maxHeapTotal": {
+                    "default": 4294967296,
+                    "anyOf": [
+                      {
+                        "type": "number",
+                        "minimum": 0
+                      },
+                      {
+                        "type": "string"
+                      }
+                    ]
+                  }
+                },
+                "additionalProperties": false
               }
             }
           }
@@ -102,6 +204,97 @@
                 "type": "string"
               }
             ]
+          },
+          "health": {
+            "type": "object",
+            "properties": {
+              "enabled": {
+                "default": true,
+                "anyOf": [
+                  {
+                    "type": "boolean"
+                  },
+                  {
+                    "type": "string"
+                  }
+                ]
+              },
+              "interval": {
+                "default": 30000,
+                "anyOf": [
+                  {
+                    "type": "number",
+                    "minimum": 0
+                  },
+                  {
+                    "type": "string"
+                  }
+                ]
+              },
+              "gracePeriod": {
+                "default": 30000,
+                "anyOf": [
+                  {
+                    "type": "number",
+                    "minimum": 0
+                  },
+                  {
+                    "type": "string"
+                  }
+                ]
+              },
+              "maxUnhealthyChecks": {
+                "default": 3,
+                "anyOf": [
+                  {
+                    "type": "number",
+                    "minimum": 1
+                  },
+                  {
+                    "type": "string"
+                  }
+                ]
+              },
+              "maxELU": {
+                "default": 0.95,
+                "anyOf": [
+                  {
+                    "type": "number",
+                    "minimum": 0,
+                    "maximum": 1
+                  },
+                  {
+                    "type": "string"
+                  }
+                ]
+              },
+              "maxHeapUsed": {
+                "default": 0.95,
+                "anyOf": [
+                  {
+                    "type": "number",
+                    "minimum": 0,
+                    "maximum": 1
+                  },
+                  {
+                    "type": "string"
+                  }
+                ]
+              },
+              "maxHeapTotal": {
+                "default": 4294967296,
+                "anyOf": [
+                  {
+                    "type": "number",
+                    "minimum": 0
+                  },
+                  {
+                    "type": "string"
+                  }
+                ]
+              }
+            },
+            "additionalProperties": false
           }
         }
       }
@@ -163,6 +356,97 @@
                 "type": "string"
               }
             ]
+          },
+          "health": {
+            "type": "object",
+            "properties": {
+              "enabled": {
+                "default": true,
+                "anyOf": [
+                  {
+                    "type": "boolean"
+                  },
+                  {
+                    "type": "string"
+                  }
+                ]
+              },
+              "interval": {
+                "default": 30000,
+                "anyOf": [
+                  {
+                    "type": "number",
+                    "minimum": 0
+                  },
+                  {
+                    "type": "string"
+                  }
+                ]
+              },
+              "gracePeriod": {
+                "default": 30000,
+                "anyOf": [
+                  {
+                    "type": "number",
+                    "minimum": 0
+                  },
+                  {
+                    "type": "string"
+                  }
+                ]
+              },
+              "maxUnhealthyChecks": {
+                "default": 3,
+                "anyOf": [
+                  {
+                    "type": "number",
+                    "minimum": 1
+                  },
+                  {
+                    "type": "string"
+                  }
+                ]
+              },
+              "maxELU": {
+                "default": 0.95,
+                "anyOf": [
+                  {
+                    "type": "number",
+                    "minimum": 0,
+                    "maximum": 1
+                  },
+                  {
+                    "type": "string"
+                  }
+                ]
+              },
+              "maxHeapUsed": {
+                "default": 0.95,
+                "anyOf": [
+                  {
+                    "type": "number",
+                    "minimum": 0,
+                    "maximum": 1
+                  },
+                  {
+                    "type": "string"
+                  }
+                ]
+              },
+              "maxHeapTotal": {
+                "default": 4294967296,
+                "anyOf": [
+                  {
+                    "type": "number",
+                    "minimum": 0
+                  },
+                  {
+                    "type": "string"
+                  }
+                ]
+              }
+            },
+            "additionalProperties": false
           }
         }
       }
@@ -214,17 +498,25 @@
                     "type": "object",
                     "properties": {
                       "target": {
-                        "type": "string",
-                        "resolveModule": true
+                        "anyOf": [
+                          {
+                            "type": "string",
+                            "resolveModule": true
+                          },
+                          {
+                            "type": "string",
+                            "resolvePath": true
+                          }
+                        ]
                       },
                       "options": {
                         "type": "object"
                       },
                       "level": {
                         "type": "string"
-                      },
-                      "additionalProperties": false
-                    }
+                      }
+                    },
+                    "additionalProperties": false
                   }
                 },
                 "options": {
@@ -409,6 +701,98 @@
       ],
       "additionalProperties": false
     },
+    "health": {
+      "type": "object",
+      "default": {},
+      "properties": {
+        "enabled": {
+          "default": true,
+          "anyOf": [
+            {
+              "type": "boolean"
+            },
+            {
+              "type": "string"
+            }
+          ]
+        },
+        "interval": {
+          "default": 30000,
+          "anyOf": [
+            {
+              "type": "number",
+              "minimum": 0
+            },
+            {
+              "type": "string"
+            }
+          ]
+        },
+        "gracePeriod": {
+          "default": 30000,
+          "anyOf": [
+            {
+              "type": "number",
+              "minimum": 0
+            },
+            {
+              "type": "string"
+            }
+          ]
+        },
+        "maxUnhealthyChecks": {
+          "default": 3,
+          "anyOf": [
+            {
+              "type": "number",
+              "minimum": 1
+            },
+            {
+              "type": "string"
+            }
+          ]
+        },
+        "maxELU": {
+          "default": 0.95,
+          "anyOf": [
+            {
+              "type": "number",
+              "minimum": 0,
+              "maximum": 1
+            },
+            {
+              "type": "string"
+            }
+          ]
+        },
+        "maxHeapUsed": {
+          "default": 0.95,
+          "anyOf": [
+            {
+              "type": "number",
+              "minimum": 0,
+              "maximum": 1
+            },
+            {
+              "type": "string"
+            }
+          ]
+        },
+        "maxHeapTotal": {
+          "default": 4294967296,
+          "anyOf": [
+            {
+              "type": "number",
+              "minimum": 0
+            },
+            {
+              "type": "string"
+            }
+          ]
+        }
+      },
+      "additionalProperties": false
+    },
     "undici": {
       "type": "object",
       "properties": {