@platformatic/runtime 2.42.0 → 2.43.0

package/config.d.ts

@@ -5,7 +5,7 @@
  * and run json-schema-to-typescript to regenerate this file.
  */
 
-export type HttpsSchemasPlatformaticDevPlatformaticRuntime2420Json = {
+export type HttpsSchemasPlatformaticDevPlatformaticRuntime2430Json = {
   [k: string]: unknown;
 } & {
   $schema?: string;

package/lib/config.js

@@ -15,6 +15,13 @@ const { parseArgs } = require('node:util')
 async function _transformConfig (configManager, args) {
   const config = configManager.current
 
+  const { values } = parseArgs({
+    args,
+    strict: false,
+    options: { production: { type: 'boolean', short: 'p', default: false } }
+  })
+  const production = values.production
+
   let services
   if (config.web?.length) {
     if (config.services?.length) {
@@ -30,13 +37,7 @@ async function _transformConfig (configManager, args) {
   if (watchType === 'string') {
     config.watch = config.watch === 'true'
   } else if (watchType === 'undefined') {
-    const { values } = parseArgs({
-      args,
-      strict: false,
-      options: { production: { type: 'boolean', short: 'p', default: false } }
-    })
-
-    config.watch = !values.production
+    config.watch = !production
   }
 
   if (config.autoload) {
@@ -109,7 +110,7 @@ async function _transformConfig (configManager, args) {
         // https://github.com/rust-lang/rust/issues/91979
         // https://github.com/rollup/rollup/issues/5761
         // TODO(mcollina): we should expose this inside every stackable configuration.
-        serviceConfig.app.modulesToLoad?.forEach((m) => {
+        serviceConfig.app.modulesToLoad?.forEach(m => {
           const toLoad = _require.resolve(m)
           loadModule(_require, toLoad).catch(() => {})
         })
@@ -142,7 +143,7 @@ async function _transformConfig (configManager, args) {
         // https://github.com/rust-lang/rust/issues/91979
         // https://github.com/rollup/rollup/issues/5761
         // TODO(mcollina): we should expose this inside every stackable configuration.
-        imported.stackable.default.modulesToLoad?.forEach((m) => {
+        imported.stackable.default.modulesToLoad?.forEach(m => {
           const toLoad = _require.resolve(m)
           loadModule(_require, toLoad).catch(() => {})
         })
@@ -210,8 +211,16 @@ async function _transformConfig (configManager, args) {
   configManager.current.services = services
   configManager.current.web = undefined
 
-  if (configManager.current.restartOnError === true) {
-    configManager.current.restartOnError = 5000
+  if (production) {
+    // Any value below 10 is considered as "immediate restart" and won't be processed via setTimeout or similar
+    // Important: do not use 2 otherwise ajv will convert to boolean `true`
+    configManager.current.restartOnError = 2
+  } else {
+    if (configManager.current.restartOnError === true) {
+      configManager.current.restartOnError = 5000
+    } else if (configManager.current.restartOnError < 0) {
+      configManager.current.restartOnError = 0
+    }
   }
 }
 

package/lib/runtime.js

@@ -41,12 +41,14 @@ const platformaticVersion = require('../package.json').version
 const kWorkerFile = join(__dirname, 'worker/main.js')
 
 const kInspectorOptions = Symbol('plt.runtime.worker.inspectorOptions')
+const kForwardEvents = Symbol('plt.runtime.worker.forwardEvents')
 
 const MAX_LISTENERS_COUNT = 100
 const MAX_METRICS_QUEUE_LENGTH = 5 * 60 // 5 minutes in seconds
 const COLLECT_METRICS_TIMEOUT = 1000
 
 const MAX_BOOTSTRAP_ATTEMPTS = 5
+const IMMEDIATE_RESTART_MAX_THRESHOLD = 10
 
 const telemetryPath = require.resolve('@platformatic/telemetry')
 const openTelemetrySetupPath = join(telemetryPath, '..', 'lib', 'node-telemetry.js')
@@ -142,7 +144,7 @@ class Runtime extends EventEmitter {
               serviceConfig.id
             )
 
-            throw new errors.RuntimeAbortedError()
+            await this.closeAndThrow(new errors.RuntimeAbortedError())
           }
         } else {
           this.logger.error(
@@ -150,7 +152,7 @@ class Runtime extends EventEmitter {
             serviceConfig.id
           )
 
-          throw new errors.RuntimeAbortedError()
+          await this.closeAndThrow(new errors.RuntimeAbortedError())
         }
       }
 
@@ -183,8 +185,7 @@ class Runtime extends EventEmitter {
         }
       }
     } catch (e) {
-      await this.close()
-      throw e
+      await this.closeAndThrow(e)
     }
 
     const dispatcherOpts = { ...config.undici }
@@ -249,10 +250,7 @@ class Runtime extends EventEmitter {
         this.#inspectorServer = server
       }
     } catch (error) {
-      // Wait for the next tick so that the error is logged first
-      await sleep(1)
-      await this.close()
-      throw error
+      await this.closeAndThrow(error)
     }
 
     this.#updateStatus('started')
@@ -334,6 +332,16 @@ class Runtime extends EventEmitter {
     this.#updateStatus('closed')
   }
 
+  async closeAndThrow (error) {
+    this.#updateStatus('errored', error)
+
+    // Wait for the next tick so that any pending logging is properly flushed
+    await sleep(1)
+    await this.close()
+
+    throw error
+  }
+
   async startService (id, silent = false) {
     // Since when a service is stopped the worker is deleted, we consider a service start if its first service
     // is no longer in the init phase
@@ -351,9 +359,13 @@ class Runtime extends EventEmitter {
 
     const workersCount = await this.#workers.getCount(serviceConfig.id)
 
+    this.emit('service:starting', id)
+
     for (let i = 0; i < workersCount; i++) {
       await this.#startWorker(config, serviceConfig, workersCount, id, i, silent)
     }
+
+    this.emit('service:started', id)
   }
 
   async stopService (id, silent = false) {
@@ -366,16 +378,22 @@ class Runtime extends EventEmitter {
 
     const workersCount = await this.#workers.getCount(serviceConfig.id)
 
+    this.emit('service:stopping', id)
+
     for (let i = 0; i < workersCount; i++) {
       await this.#stopWorker(workersCount, id, i, silent)
     }
+
+    this.emit('service:stopped', id)
   }
 
   async buildService (id) {
     const service = await this.#getServiceById(id)
 
+    this.emit('service:building', id)
     try {
-      return await sendViaITC(service, 'build')
+      await sendViaITC(service, 'build')
+      this.emit('service:built', id)
     } catch (e) {
       // The service exports no meta, return an empty object
       if (e.code === 'PLT_ITC_HANDLER_NOT_FOUND') {
@@ -600,6 +618,26 @@ class Runtime extends EventEmitter {
     }
   }
 
+  async getWorkers () {
+    const status = {}
+
+    for (const [service, { count }] of Object.entries(this.#workers.configuration)) {
+      for (let i = 0; i < count; i++) {
+        const label = `${service}:${i}`
+        const worker = this.#workers.get(label)
+
+        status[label] = {
+          service,
+          worker: i,
+          status: worker?.[kWorkerStatus] ?? 'exited',
+          thread: worker?.threadId
+        }
+      }
+    }
+
+    return status
+  }
+
   async getServiceDetails (id, allowUnloaded = false) {
     let service
 
@@ -935,9 +973,19 @@ class Runtime extends EventEmitter {
     }
   }
 
-  #updateStatus (status) {
+  emit (event, payload) {
+    for (const worker of this.#workers.values()) {
+      if (worker[kForwardEvents]) {
+        worker[kITC].notify('runtime:event', { event, payload })
+      }
+    }
+
+    return super.emit(event, payload)
+  }
+
+  #updateStatus (status, args) {
     this.#status = status
-    this.emit(status)
+    this.emit(status, args)
   }
 
   #showUrl () {
@@ -954,6 +1002,8 @@ class Runtime extends EventEmitter {
     for (let i = 0; i < workersCount; i++) {
       await this.#setupWorker(config, serviceConfig, workersCount, id, i)
     }
+
+    this.emit('service:init', id)
   }
 
   async #setupWorker (config, serviceConfig, workersCount, serviceId, index) {
@@ -1036,6 +1086,7 @@ class Runtime extends EventEmitter {
     worker.setMaxListeners(1e3)
 
     // Track service exiting
+    const eventPayload = { service: serviceId, worker: index, workersCount }
     worker.once('exit', code => {
       if (worker[kWorkerStatus] === 'exited') {
         return
@@ -1043,6 +1094,7 @@ class Runtime extends EventEmitter {
 
       const started = worker[kWorkerStatus] === 'started'
       worker[kWorkerStatus] = 'exited'
+      this.emit('service:worker:exited', eventPayload)
 
       this.#cleanupWorker(workerId, worker)
 
@@ -1053,17 +1105,24 @@ class Runtime extends EventEmitter {
       // Wait for the next tick so that crashed from the thread are logged first
       setImmediate(() => {
         if (started && (!config.watch || code !== 0)) {
+          this.emit('service:worker:error', { ...eventPayload, code })
           this.logger.warn(`The ${errorLabel} unexpectedly exited with code ${code}.`)
         }
 
         // Restart the service if it was started
         if (started && this.#status === 'started') {
           if (restartOnError > 0) {
-            this.logger.warn(`The ${errorLabel} will be restarted in ${restartOnError}ms...`)
+            if (restartOnError < IMMEDIATE_RESTART_MAX_THRESHOLD) {
+              this.logger.warn(`The ${errorLabel} is being restarted ...`)
+            } else {
+              this.logger.warn(`The ${errorLabel} will be restarted in ${restartOnError}ms ...`)
+            }
+
             this.#restartCrashedWorker(config, serviceConfig, workersCount, serviceId, index, false, 0).catch(err => {
               this.logger.error({ err: ensureLoggableError(err) }, `${errorLabel} could not be restarted.`)
             })
           } else {
+            this.emit('service:worker:unvailable', eventPayload)
             this.logger.warn(`The ${errorLabel} is no longer available.`)
           }
         }
@@ -1073,6 +1132,7 @@ class Runtime extends EventEmitter {
     worker[kId] = workersCount > 1 ? workerId : serviceId
     worker[kServiceId] = serviceId
     worker[kWorkerId] = workersCount > 1 ? index : undefined
+    worker[kForwardEvents] = false
 
     if (inspectorOptions) {
       worker[kInspectorOptions] = {
@@ -1090,10 +1150,14 @@ class Runtime extends EventEmitter {
         getServiceMeta: this.getServiceMeta.bind(this),
         listServices: () => this.#servicesIds,
         getServices: this.getServices.bind(this),
+        getWorkers: this.getWorkers.bind(this),
         getHttpCacheValue: this.#getHttpCacheValue.bind(this),
         setHttpCacheValue: this.#setHttpCacheValue.bind(this),
         deleteHttpCacheValue: this.#deleteHttpCacheValue.bind(this),
-        invalidateHttpCache: this.invalidateHttpCache.bind(this)
+        invalidateHttpCache: this.invalidateHttpCache.bind(this),
+        setEventsForwarding (value) {
+          worker[kForwardEvents] = value
+        }
       }
     })
     worker[kITC].listen()
@@ -1105,6 +1169,8 @@ class Runtime extends EventEmitter {
       // so that services can eventually manually trigger a restart. This mechanism is current
       // used by the composer.
       worker[kITC].on('changed', async () => {
+        this.emit('service:worker:changed', eventPayload)
+
         try {
           const wasStarted = worker[kWorkerStatus].startsWith('start')
           await this.stopService(serviceId)
@@ -1147,6 +1213,7 @@ class Runtime extends EventEmitter {
     // This must be done here as the dependencies are filled above
     worker[kConfig] = { ...serviceConfig, health }
     worker[kWorkerStatus] = 'boot'
+    this.emit('service:worker:boot', eventPayload)
   }
 
   #setupHealthCheck (worker, errorLabel) {
@@ -1158,24 +1225,42 @@ class Runtime extends EventEmitter {
 
     worker[kHealthCheckTimer] = setTimeout(async () => {
       const health = await worker[kITC].send('getHealth')
+      const { elu, heapUsed } = health
+      const memoryUsage = heapUsed / maxHeapTotal
+      const unhealthy = elu > maxELU || memoryUsage > maxHeapUsed
+
       this.emit('health', {
         id: worker[kId],
         service: worker[kServiceId],
         worker: worker[kWorkerId],
         currentHealth: health,
+        unhealthy,
         healthConfig: worker[kConfig].health
       })
 
-      const { elu, heapUsed } = health
-      const memoryUsage = heapUsed / maxHeapTotal
-      if (elu > maxELU || memoryUsage > maxHeapUsed) {
+      if (unhealthy) {
+        if (elu > maxELU) {
+          this.logger.error(
+            `The ${errorLabel} has an ELU of ${(elu * 100).toFixed(2)} %, above the maximum allowed usage of ${(maxELU * 100).toFixed(2)} %.`
+          )
+        }
+
+        if (memoryUsage > maxHeapUsed) {
+          this.logger.error(
+            `The ${errorLabel} is using ${(elu * 100).toFixed(2)} % of the memory, above the maximum allowed usage of ${(maxELU * 100).toFixed(2)} %.`
+          )
+        }
+
         unhealthyChecks++
       } else {
         unhealthyChecks = 0
       }
 
       if (unhealthyChecks >= maxUnhealthyChecks) {
-        this.logger.error(`The ${errorLabel} is unhealthy. Forcefully terminating it ...`)
+        this.logger.error(
+          { elu, maxELU, memoryUsage, maxMemoryUsage: maxHeapUsed },
+          `The ${errorLabel} is unhealthy. Forcefully terminating it ...`
+        )
         worker.terminate()
         return
       }
@@ -1193,6 +1278,7 @@ class Runtime extends EventEmitter {
     }
 
     let worker = await this.#getWorkerById(id, index, false, false)
+    const eventPayload = { service: id, worker: index, workersCount }
 
     // The service was stopped, recreate the thread
     if (!worker) {
@@ -1201,6 +1287,7 @@ class Runtime extends EventEmitter {
     }
 
     worker[kWorkerStatus] = 'starting'
+    this.emit('service:worker:starting', eventPayload)
 
     try {
       let workerUrl
@@ -1208,6 +1295,7 @@ class Runtime extends EventEmitter {
         workerUrl = await executeWithTimeout(sendViaITC(worker, 'start'), config.startTimeout)
 
         if (workerUrl === 'timeout') {
+          this.emit('service:worker:startTimeout', eventPayload)
           this.logger.info(`The ${label} failed to start in ${config.startTimeout}ms. Forcefully killing the thread.`)
           worker.terminate()
           throw new errors.ServiceStartTimeoutError(id, config.startTimeout)
@@ -1223,6 +1311,7 @@ class Runtime extends EventEmitter {
       }
 
       worker[kWorkerStatus] = 'started'
+      this.emit('service:worker:started', eventPayload)
 
       if (!silent) {
         this.logger?.info(`Started the ${label}...`)
@@ -1249,6 +1338,8 @@ class Runtime extends EventEmitter {
         await worker.terminate()
       }
 
+      this.emit('service:worker:start:error', eventPayload)
+
       if (error.code !== 'PLT_RUNTIME_SERVICE_START_TIMEOUT') {
         this.logger.error({ err: ensureLoggableError(error) }, `Failed to start ${label}.`)
       }
@@ -1264,9 +1355,15 @@ class Runtime extends EventEmitter {
         throw error
       }
 
-      this.logger.warn(
-        `Attempt ${bootstrapAttempt} of ${MAX_BOOTSTRAP_ATTEMPTS} to start the ${label} again will be performed in ${restartOnError}ms ...`
-      )
+      if (restartOnError < IMMEDIATE_RESTART_MAX_THRESHOLD) {
+        this.logger.warn(
+          `Performing attempt ${bootstrapAttempt} of ${MAX_BOOTSTRAP_ATTEMPTS} to start the ${label} again ...`
+        )
+      } else {
+        this.logger.warn(
+          `Attempt ${bootstrapAttempt} of ${MAX_BOOTSTRAP_ATTEMPTS} to start the ${label} again will be performed in ${restartOnError}ms ...`
+        )
+      }
 
       await this.#restartCrashedWorker(config, serviceConfig, workersCount, id, index, silent, bootstrapAttempt)
     }
@@ -1279,7 +1376,10 @@ class Runtime extends EventEmitter {
       return
     }
 
+    const eventPayload = { service: id, worker: index, workersCount }
+
     worker[kWorkerStatus] = 'stopping'
+    this.emit('service:worker:stopping', eventPayload)
 
     const label = this.#workerExtendedLabel(id, index, workersCount)
 
@@ -1294,6 +1394,7 @@ class Runtime extends EventEmitter {
     try {
       await executeWithTimeout(sendViaITC(worker, 'stop'), exitTimeout)
     } catch (error) {
+      this.emit('service:worker:stop:timeout', eventPayload)
       this.logger?.info({ error: ensureLoggableError(error) }, `Failed to stop ${label}. Killing a worker thread.`)
     } finally {
       worker[kITC].close()
@@ -1308,12 +1409,14 @@ class Runtime extends EventEmitter {
 
     // If the worker didn't exit in time, kill it
     if (res === 'timeout') {
+      this.emit('service:worker:exit:timeout', eventPayload)
       await worker.terminate()
     }
 
     await this.#avoidOutOfOrderThreadLogs()
 
     worker[kWorkerStatus] = 'stopped'
+    this.emit('service:worker:stopped', eventPayload)
   }
 
   #cleanupWorker (workerId, worker) {
@@ -1338,7 +1441,7 @@ class Runtime extends EventEmitter {
     }
 
     restartPromise = new Promise((resolve, reject) => {
-      setTimeout(async () => {
+      async function restart () {
         this.#restartingWorkers.delete(workerId)
 
         // If some processes were scheduled to restart
@@ -1360,7 +1463,13 @@ class Runtime extends EventEmitter {
 
           reject(err)
         }
-      }, config.restartOnError)
+      }
+
+      if (config.restartOnError < IMMEDIATE_RESTART_MAX_THRESHOLD) {
+        process.nextTick(restart.bind(this))
+      } else {
+        setTimeout(restart.bind(this), config.restartOnError)
+      }
     })
 
     this.#restartingWorkers.set(workerId, restartPromise)

package/lib/schema.js

@@ -206,7 +206,7 @@ const platformaticRuntimeSchema = {
         { type: 'boolean' },
         {
           type: 'number',
-          minimum: 100
+          minimum: 0
         }
       ]
     },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platformatic/runtime",
-  "version": "2.42.0",
+  "version": "2.43.0",
   "description": "",
   "main": "index.js",
   "bin": {
@@ -36,12 +36,12 @@
     "typescript": "^5.5.4",
     "undici-oidc-interceptor": "^0.5.0",
     "why-is-node-running": "^2.2.2",
-    "@platformatic/composer": "2.42.0",
-    "@platformatic/db": "2.42.0",
-    "@platformatic/service": "2.42.0",
-    "@platformatic/node": "2.42.0",
-    "@platformatic/sql-mapper": "2.42.0",
-    "@platformatic/sql-graphql": "2.42.0"
+    "@platformatic/composer": "2.43.0",
+    "@platformatic/node": "2.43.0",
+    "@platformatic/db": "2.43.0",
+    "@platformatic/service": "2.43.0",
+    "@platformatic/sql-mapper": "2.43.0",
+    "@platformatic/sql-graphql": "2.43.0"
   },
   "dependencies": {
     "@fastify/accepts": "^5.0.0",
@@ -75,13 +75,13 @@
     "undici": "^7.0.0",
     "undici-thread-interceptor": "^0.11.0",
     "ws": "^8.16.0",
-    "@platformatic/basic": "2.42.0",
-    "@platformatic/generators": "2.42.0",
-    "@platformatic/config": "2.42.0",
-    "@platformatic/itc": "2.42.0",
-    "@platformatic/telemetry": "2.42.0",
-    "@platformatic/ts-compiler": "2.42.0",
-    "@platformatic/utils": "2.42.0"
+    "@platformatic/basic": "2.43.0",
+    "@platformatic/config": "2.43.0",
+    "@platformatic/generators": "2.43.0",
+    "@platformatic/itc": "2.43.0",
+    "@platformatic/utils": "2.43.0",
+    "@platformatic/ts-compiler": "2.43.0",
+    "@platformatic/telemetry": "2.43.0"
   },
   "scripts": {
     "test": "npm run lint && borp --concurrency=1 --timeout=300000 && tsd",

package/schema.json

@@ -1,5 +1,5 @@
 {
-  "$id": "https://schemas.platformatic.dev/@platformatic/runtime/2.42.0.json",
+  "$id": "https://schemas.platformatic.dev/@platformatic/runtime/2.43.0.json",
   "$schema": "http://json-schema.org/draft-07/schema#",
   "type": "object",
   "properties": {
@@ -113,7 +113,7 @@
                     ]
                   },
                   "maxUnhealthyChecks": {
-                    "default": 3,
+                    "default": 10,
                     "anyOf": [
                       {
                         "type": "number",
@@ -125,7 +125,7 @@
                     ]
                   },
                   "maxELU": {
-                    "default": 0.95,
+                    "default": 0.99,
                     "anyOf": [
                       {
                         "type": "number",
@@ -138,7 +138,7 @@
                     ]
                   },
                   "maxHeapUsed": {
-                    "default": 0.95,
+                    "default": 0.99,
                     "anyOf": [
                       {
                         "type": "number",
@@ -284,7 +284,7 @@
                 ]
               },
               "maxUnhealthyChecks": {
-                "default": 3,
+                "default": 10,
                 "anyOf": [
                   {
                     "type": "number",
@@ -296,7 +296,7 @@
                 ]
               },
               "maxELU": {
-                "default": 0.95,
+                "default": 0.99,
                 "anyOf": [
                   {
                     "type": "number",
@@ -309,7 +309,7 @@
                 ]
               },
               "maxHeapUsed": {
-                "default": 0.95,
+                "default": 0.99,
                 "anyOf": [
                   {
                     "type": "number",
@@ -520,7 +520,7 @@
                 ]
               },
               "maxUnhealthyChecks": {
-                "default": 3,
+                "default": 10,
                 "anyOf": [
                   {
                     "type": "number",
@@ -532,7 +532,7 @@
                 ]
               },
               "maxELU": {
-                "default": 0.95,
+                "default": 0.99,
                 "anyOf": [
                   {
                     "type": "number",
@@ -545,7 +545,7 @@
                 ]
               },
               "maxHeapUsed": {
-                "default": 0.95,
+                "default": 0.99,
                 "anyOf": [
                   {
                     "type": "number",
@@ -870,7 +870,7 @@
         },
         {
           "type": "number",
-          "minimum": 100
+          "minimum": 0
         }
       ]
     },
@@ -949,7 +949,7 @@
           ]
         },
         "maxUnhealthyChecks": {
-          "default": 3,
+          "default": 10,
           "anyOf": [
             {
               "type": "number",
@@ -961,7 +961,7 @@
           ]
         },
         "maxELU": {
-          "default": 0.95,
+          "default": 0.99,
           "anyOf": [
             {
               "type": "number",
@@ -974,7 +974,7 @@
           ]
         },
         "maxHeapUsed": {
-          "default": 0.95,
+          "default": 0.99,
           "anyOf": [
             {
               "type": "number",