@platformatic/runtime 2.69.0 → 2.70.1

package/lib/runtime.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const { ITC } = require('@platformatic/itc')
-const { features, ensureLoggableError, executeWithTimeout, deepmerge } = require('@platformatic/utils')
+const { features, ensureLoggableError, executeWithTimeout, deepmerge, parseMemorySize, kTimeout } = require('@platformatic/utils')
 const { once, EventEmitter } = require('node:events')
 const { createReadStream, watch, existsSync } = require('node:fs')
 const { readdir, readFile, stat, access } = require('node:fs/promises')
@@ -33,7 +33,9 @@ const {
   kHealthCheckTimer,
   kConfig,
   kWorkerStatus,
-  kStderrMarker
+  kStderrMarker,
+  kLastELU,
+  kWorkersBroadcast
 } = require('./worker/symbols')
 
 const fastify = require('fastify')
@@ -74,6 +76,8 @@ class Runtime extends EventEmitter {
   #prometheusServer
   #inspectorServer
   #workers
+  #workersBroadcastChannel
+  #workerITCHandlers
   #restartingWorkers
   #sharedHttpCache
   servicesConfigsPatches
@@ -106,6 +110,18 @@ class Runtime extends EventEmitter {
         stderr: new SonicBoom({ fd: process.stderr.fd })
       }
     }
+
+    this.#workerITCHandlers = {
+      getServiceMeta: this.getServiceMeta.bind(this),
+      listServices: () => this.#servicesIds,
+      getServices: this.getServices.bind(this),
+      getWorkers: this.getWorkers.bind(this),
+      getWorkerMessagingChannel: this.#getWorkerMessagingChannel.bind(this),
+      getHttpCacheValue: this.#getHttpCacheValue.bind(this),
+      setHttpCacheValue: this.#setHttpCacheValue.bind(this),
+      deleteHttpCacheValue: this.#deleteHttpCacheValue.bind(this),
+      invalidateHttpCache: this.invalidateHttpCache.bind(this)
+    }
   }
 
   async init () {
@@ -130,12 +146,15 @@ class Runtime extends EventEmitter {
 
     this.#isProduction = this.#configManager.args?.production ?? false
     this.#servicesIds = config.services.map(service => service.id)
+    this.#createWorkersBroadcastChannel()
 
     const workersConfig = []
     for (const service of config.services) {
       const count = service.workers ?? this.#configManager.current.workers
       if (count > 1 && service.entrypoint && !features.node.reusePort) {
-        this.logger.warn(`"${service.id}" is set as the entrypoint, but reusePort is not available in your OS; setting workers to 1 instead of ${count}`)
+        this.logger.warn(
+          `"${service.id}" is set as the entrypoint, but reusePort is not available in your OS; setting workers to 1 instead of ${count}`
+        )
         workersConfig.push({ id: service.id, workers: 1 })
       } else {
         workersConfig.push({ id: service.id, workers: count })
@@ -225,6 +244,7 @@ class Runtime extends EventEmitter {
       throw new errors.MissingEntrypointError()
     }
     this.#updateStatus('starting')
+    this.#createWorkersBroadcastChannel()
 
     // Important: do not use Promise.all here since it won't properly manage dependencies
     try {
@@ -311,6 +331,7 @@ class Runtime extends EventEmitter {
     }
 
     await this.#meshInterceptor.close()
+    this.#workersBroadcastChannel?.close()
 
     this.#updateStatus('stopped')
   }
@@ -1148,10 +1169,13 @@ class Runtime extends EventEmitter {
       workerEnv['NODE_OPTIONS'] = `${originalNodeOptions} ${serviceConfig.nodeOptions}`.trim()
     }
 
+    const maxHeapTotal = typeof health.maxHeapTotal === 'string' ? parseMemorySize(health.maxHeapTotal) : health.maxHeapTotal
+    const maxYoungGeneration = typeof health.maxYoungGeneration === 'string' ? parseMemorySize(health.maxYoungGeneration) : health.maxYoungGeneration
+
     const maxOldGenerationSizeMb = Math.floor(
-      (health.maxYoungGeneration > 0 ? health.maxHeapTotal - health.maxYoungGeneration : health.maxHeapTotal) / (1024 * 1024)
+      (maxYoungGeneration > 0 ? maxHeapTotal - maxYoungGeneration : maxHeapTotal) / (1024 * 1024)
     )
-    const maxYoungGenerationSizeMb = health.maxYoungGeneration ? Math.floor(health.maxYoungGeneration / (1024 * 1024)) : undefined
+    const maxYoungGenerationSizeMb = maxYoungGeneration ? Math.floor(maxYoungGeneration / (1024 * 1024)) : undefined
 
     const worker = new Worker(kWorkerFile, {
       workerData: {
@@ -1207,6 +1231,8 @@ class Runtime extends EventEmitter {
       setImmediate(() => {
         if (started && (!config.watch || code !== 0)) {
           this.emit('service:worker:error', { ...eventPayload, code })
+          this.#broadcastWorkers()
+
           this.logger.warn(`The ${errorLabel} unexpectedly exited with code ${code}.`)
         }
 
@@ -1250,14 +1276,7 @@ class Runtime extends EventEmitter {
       name: workerId + '-runtime',
       port: worker,
       handlers: {
-        getServiceMeta: this.getServiceMeta.bind(this),
-        listServices: () => this.#servicesIds,
-        getServices: this.getServices.bind(this),
-        getWorkers: this.getWorkers.bind(this),
-        getHttpCacheValue: this.#getHttpCacheValue.bind(this),
-        setHttpCacheValue: this.#setHttpCacheValue.bind(this),
-        deleteHttpCacheValue: this.#deleteHttpCacheValue.bind(this),
-        invalidateHttpCache: this.invalidateHttpCache.bind(this),
+        ...this.#workerITCHandlers,
         setEventsForwarding (value) {
           worker[kForwardEvents] = value
         }
@@ -1282,7 +1301,7 @@ class Runtime extends EventEmitter {
             await this.startService(serviceId)
           }
 
-          this.logger?.info(`Service "${serviceId}" has been successfully reloaded ...`)
+          this.logger?.info(`The service "${serviceId}" has been successfully reloaded ...`)
 
           if (serviceConfig.entrypoint) {
             this.#showUrl()
@@ -1323,18 +1342,46 @@ class Runtime extends EventEmitter {
     return worker
   }
 
-  #setupHealthCheck (config, serviceConfig, workersCount, id, index, workerId, worker, errorLabel) {
+  async #getHealth (worker) {
+    if (features.node.worker.getHeapStatistics) {
+      const { used_heap_size: heapUsed, total_heap_size: heapTotal } = await worker.getHeapStatistics()
+      const currentELU = worker.performance.eventLoopUtilization()
+      const elu = worker[kLastELU]
+        ? worker.performance.eventLoopUtilization(currentELU, worker[kLastELU])
+        : currentELU
+      worker[kLastELU] = currentELU
+      return { elu: elu.utilization, heapUsed, heapTotal }
+    }
+
+    const health = await worker[kITC].send('getHealth')
+    return health
+  }
+
+  #setupHealthCheck (config, serviceConfig, workersCount, id, index, worker, errorLabel, timeout) {
     // Clear the timeout when exiting
     worker.on('exit', () => clearTimeout(worker[kHealthCheckTimer]))
 
     const { maxELU, maxHeapUsed, maxHeapTotal, maxUnhealthyChecks, interval } = worker[kConfig].health
+    const maxHeapTotalNumber = typeof maxHeapTotal === 'string' ? parseMemorySize(maxHeapTotal) : maxHeapTotal
+
     let unhealthyChecks = 0
 
     worker[kHealthCheckTimer] = setTimeout(async () => {
-      const health = await worker[kITC].send('getHealth')
-      const { elu, heapUsed } = health
-      const memoryUsage = heapUsed / maxHeapTotal
-      const unhealthy = elu > maxELU || memoryUsage > maxHeapUsed
+      if (worker[kWorkerStatus] !== 'started') {
+        return
+      }
+
+      let health, unhealthy, memoryUsage
+      try {
+        health = await this.#getHealth(worker)
+        memoryUsage = health.heapUsed / maxHeapTotalNumber
+        unhealthy = health.elu > maxELU || memoryUsage > maxHeapUsed
+      } catch (err) {
+        this.logger.error({ err }, `Failed to get health for ${errorLabel}.`)
+        unhealthy = true
+        memoryUsage = -1
+        health = { elu: -1, heapUsed: -1, heapTotal: -1 }
+      }
 
       const serviceId = worker[kServiceId]
       this.emit('health', {
@@ -1347,9 +1394,9 @@ class Runtime extends EventEmitter {
       })
 
       if (unhealthy) {
-        if (elu > maxELU) {
+        if (health.elu > maxELU) {
           this.logger.error(
-            `The ${errorLabel} has an ELU of ${(elu * 100).toFixed(2)} %, above the maximum allowed usage of ${(maxELU * 100).toFixed(2)} %.`
+            `The ${errorLabel} has an ELU of ${(health.elu * 100).toFixed(2)} %, above the maximum allowed usage of ${(maxELU * 100).toFixed(2)} %.`
           )
         }
 
@@ -1367,14 +1414,14 @@ class Runtime extends EventEmitter {
       if (unhealthyChecks === maxUnhealthyChecks) {
         try {
           this.logger.error(
-            { elu, maxELU, memoryUsage, maxMemoryUsage: maxHeapUsed },
+            { elu: health.elu, maxELU, memoryUsage: health.heapUsed, maxMemoryUsage: maxHeapUsed },
             `The ${errorLabel} is unhealthy. Replacing it ...`
           )
 
-          await this.#replaceWorker(config, serviceConfig, workersCount, id, index, workerId, worker)
+          await this.#replaceWorker(config, serviceConfig, workersCount, id, index, worker)
         } catch (e) {
           this.logger.error(
-            { elu, maxELU, memoryUsage, maxMemoryUsage: maxHeapUsed },
+            { elu: health.elu, maxELU, memoryUsage: health.heapUsed, maxMemoryUsage: maxHeapUsed },
             `Cannot replace the ${errorLabel}. Forcefully terminating it ...`
           )
 
@@ -1383,7 +1430,7 @@ class Runtime extends EventEmitter {
       } else {
         worker[kHealthCheckTimer].refresh()
       }
-    }, interval)
+    }, timeout ?? interval)
   }
 
   async #startWorker (
@@ -1397,7 +1444,6 @@ class Runtime extends EventEmitter {
     worker = undefined,
     disableRestartAttempts = false
   ) {
-    const workerId = `${id}:${index}`
     const label = this.#workerExtendedLabel(id, index, workersCount)
 
     if (!silent) {
@@ -1424,7 +1470,7 @@ class Runtime extends EventEmitter {
       if (config.startTimeout > 0) {
         workerUrl = await executeWithTimeout(sendViaITC(worker, 'start'), config.startTimeout)
 
-        if (workerUrl === 'timeout') {
+        if (workerUrl === kTimeout) {
           this.emit('service:worker:startTimeout', eventPayload)
           this.logger.info(`The ${label} failed to start in ${config.startTimeout}ms. Forcefully killing the thread.`)
           worker.terminate()
@@ -1442,6 +1488,7 @@ class Runtime extends EventEmitter {
 
       worker[kWorkerStatus] = 'started'
       this.emit('service:worker:started', eventPayload)
+      this.#broadcastWorkers()
 
       if (!silent) {
         this.logger?.info(`Started the ${label}...`)
@@ -1449,12 +1496,9 @@ class Runtime extends EventEmitter {
 
       const { enabled, gracePeriod } = worker[kConfig].health
       if (enabled && config.restartOnError > 0) {
-        worker[kHealthCheckTimer] = setTimeout(
-          () => {
-            this.#setupHealthCheck(config, serviceConfig, workersCount, id, index, workerId, worker, label)
-          },
-          gracePeriod > 0 ? gracePeriod : 1
-        )
+        // if gracePeriod is 0, it will be set to 1 to start health checks immediately
+        // however, the health event will start when the worker is started
+        this.#setupHealthCheck(config, serviceConfig, workersCount, id, index, worker, label, gracePeriod > 0 ? gracePeriod : 1)
       }
     } catch (error) {
       // TODO: handle port allocation error here
@@ -1545,7 +1589,7 @@ class Runtime extends EventEmitter {
     const res = await executeWithTimeout(exitPromise, exitTimeout)
 
     // If the worker didn't exit in time, kill it
-    if (res === 'timeout') {
+    if (res === kTimeout) {
       this.emit('service:worker:exit:timeout', eventPayload)
       await worker.terminate()
     }
@@ -1554,6 +1598,7 @@ class Runtime extends EventEmitter {
 
     worker[kWorkerStatus] = 'stopped'
     this.emit('service:worker:stopped', eventPayload)
+    this.#broadcastWorkers()
   }
 
   #cleanupWorker (worker) {
@@ -1603,6 +1648,9 @@ class Runtime extends EventEmitter {
           await this.#setupWorker(config, serviceConfig, workersCount, id, index)
           await this.#startWorker(config, serviceConfig, workersCount, id, index, silent, bootstrapAttempt)
 
+          this.logger.info(
+            `The ${this.#workerExtendedLabel(id, index, workersCount)} has been successfully restarted ...`
+          )
           resolve()
         } catch (err) {
           // The runtime was stopped while the restart was happening, ignore any error.
@@ -1625,7 +1673,8 @@ class Runtime extends EventEmitter {
     await restartPromise
   }
 
-  async #replaceWorker (config, serviceConfig, workersCount, serviceId, index, workerId, worker) {
+  async #replaceWorker (config, serviceConfig, workersCount, serviceId, index, worker) {
+    const workerId = `${serviceId}:${index}`
     let newWorker
 
     try {
@@ -1685,7 +1734,15 @@ class Runtime extends EventEmitter {
         return null
       }
 
-      throw new errors.ServiceNotFoundError(serviceId, Array.from(this.#servicesIds).join(', '))
+      if (this.#servicesIds.includes(serviceId)) {
+        const availableWorkers = Array.from(this.#workers.keys())
+          .filter(key => key.startsWith(serviceId + ':'))
+          .map(key => key.split(':')[1])
+          .join(', ')
+        throw new errors.WorkerNotFoundError(workerId, serviceId, availableWorkers)
+      } else {
+        throw new errors.ServiceNotFoundError(serviceId, Array.from(this.#servicesIds).join(', '))
+      }
     }
 
     if (ensureStarted) {
@@ -1699,6 +1756,59 @@ class Runtime extends EventEmitter {
     return worker
   }
 
+  async #createWorkersBroadcastChannel () {
+    this.#workersBroadcastChannel?.close()
+    this.#workersBroadcastChannel = new BroadcastChannel(kWorkersBroadcast)
+  }
+
+  async #broadcastWorkers () {
+    const workers = new Map()
+
+    // Create the list of workers
+    for (const worker of this.#workers.values()) {
+      if (worker[kWorkerStatus] !== 'started') {
+        continue
+      }
+
+      const service = worker[kServiceId]
+      let serviceWorkers = workers.get(service)
+
+      if (!serviceWorkers) {
+        serviceWorkers = []
+        workers.set(service, serviceWorkers)
+      }
+
+      serviceWorkers.push({
+        id: worker[kId],
+        service: worker[kServiceId],
+        worker: worker[kWorkerId],
+        thread: worker.threadId
+      })
+    }
+
+    this.#workersBroadcastChannel.postMessage(workers)
+  }
+
+  async #getWorkerMessagingChannel ({ service, worker }, context) {
+    const target = await this.#getWorkerById(service, worker, true, true)
+
+    const { port1, port2 } = new MessageChannel()
+
+    // Send the first port to the target
+    const response = await executeWithTimeout(
+      sendViaITC(target, 'saveMessagingChannel', port1, [port1]),
+      this.#configManager.current.messagingTimeout
+    )
+
+    if (response === kTimeout) {
+      throw new errors.MessagingError(service, 'Timeout while establishing a communication channel.')
+    }
+
+    context.transferList = [port2]
+    this.emit('service:worker:messagingChannel', { service, worker })
+    return port2
+  }
+
   async #getRuntimePackageJson () {
     const runtimeDir = this.#configManager.dirname
     const packageJsonPath = join(runtimeDir, 'package.json')
@@ -1804,7 +1914,8 @@ class Runtime extends EventEmitter {
         }
       }
 
-      const pinoLog = typeof message?.level === 'number' && typeof message?.time === 'number' && typeof message?.msg === 'string'
+      const pinoLog =
+        typeof message?.level === 'number' && typeof message?.time === 'number' && typeof message?.msg === 'string'
 
       // Directly write to the Pino destination
       if (pinoLog) {
@@ -1849,10 +1960,16 @@ class Runtime extends EventEmitter {
 
   async getServiceResourcesInfo (id) {
     const workers = this.#workers.getCount(id)
-    return { workers }
+
+    const worker = await this.#getWorkerById(id, 0, false, false)
+    const health = worker[kConfig].health
+
+    return { workers, health }
   }
 
-  async #updateWorkerCount (serviceId, workers) {
+  async #updateServiceConfigWorkers (serviceId, workers) {
+    this.logger.info(`Updating service "${serviceId}" config workers to ${workers}`)
+
     this.#configManager.current.services.find(s => s.id === serviceId).workers = workers
     const service = await this.#getServiceById(serviceId)
     this.#workers.setCount(serviceId, workers)
@@ -1868,14 +1985,109 @@ class Runtime extends EventEmitter {
     const results = await Promise.allSettled(promises)
     for (const result of results) {
       if (result.status === 'rejected') {
+        this.logger.error({ err: result.reason }, `Cannot update service "${serviceId}" workers`)
         throw result.reason
       }
     }
   }
 
+  async #updateServiceConfigHealth (serviceId, health) {
+    this.logger.info(`Updating service "${serviceId}" config health heap to ${JSON.stringify(health)}`)
+    const { maxHeapTotal, maxYoungGeneration } = health
+
+    const service = this.#configManager.current.services.find(s => s.id === serviceId)
+    if (maxHeapTotal) {
+      service.health.maxHeapTotal = maxHeapTotal
+    }
+    if (maxYoungGeneration) {
+      service.health.maxYoungGeneration = maxYoungGeneration
+    }
+  }
+
+  /**
+   * Updates the resources of the services, such as the number of workers and health configurations (e.g., heap memory settings).
+   *
+   * This function handles three update scenarios for each service:
+   *  1. **Updating workers only**: Adjusts the number of workers for the service.
+   *  2. **Updating health configurations only**: Updates health parameters like `maxHeapTotal` or `maxYoungGeneration`.
+   *  3. **Updating both workers and health configurations**: Scales the workers and also applies health settings.
+   *
+   * When updating both workers and health:
+   *  - **Scaling down workers**: Stops extra workers, then restarts the remaining workers with the previous settings.
+   *  - **Scaling up workers**: Starts new workers with the updated heap settings, then restarts the old workers with the updated settings.
+   *
+   * Scaling up new resources (workers and/or heap memory) may fails due to insufficient memory, in this case the operation may fail partially or entirely.
+   * Scaling down is expected to succeed without issues.
+   *
+   * @param {Array<Object>} updates - An array of objects that define the updates for each service.
+   * @param {string} updates[].service - The ID of the service to update.
+   * @param {number} [updates[].workers] - The desired number of workers for the service. If omitted, workers will not be updated.
+   * @param {Object} [updates[].health] - The health configuration to update for the service, which may include:
+   *   @param {string|number} [updates[].health.maxHeapTotal] - The maximum heap memory for the service. Can be a valid memory string (e.g., '1G', '512MB') or a number representing bytes.
+   *   @param {string|number} [updates[].health.maxYoungGeneration] - The maximum young generation memory for the service. Can be a valid memory string (e.g., '128MB') or a number representing bytes.
+   *
+   * @returns {Promise<Array<Object>>} - A promise that resolves to an array of reports for each service, detailing the success or failure of the operations:
+   *   - `service`: The service ID.
+   *   - `workers`: The workers' update report, including the current, new number of workers, started workers, and success status.
+   *   - `health`: The health update report, showing the current and new heap settings, updated workers, and success status.
+   *
+   * @example
+   * await runtime.updateServicesResources([
+   *   { service: 'service-1', workers: 2, health: { maxHeapTotal: '1G', maxYoungGeneration: '128 MB' } },
+   *   { service: 'service-2', health: { maxHeapTotal: '1G' } },
+   *   { service: 'service-3', workers: 2 },
+   * ])
+   *
+   * In this example:
+   * - `service-1` will have 2 workers and updated heap memory configurations.
+   * - `service-2` will have updated heap memory settings (without changing workers).
+   * - `service-3` will have its workers set to 2 but no change in memory settings.
+   *
+   * @throws {InvalidArgumentError} - Throws if any update parameter is invalid, such as:
+   *   - Missing service ID.
+   *   - Invalid worker count (not a positive integer).
+   *   - Invalid memory size format for `maxHeapTotal` or `maxYoungGeneration`.
+   * @throws {ServiceNotFoundError} - Throws if the specified service ID does not exist in the current service configuration.
+   */
   async updateServicesResources (updates) {
-    if (this.#status === 'stopping' || this.#status === 'closed') return
+    if (this.#status === 'stopping' || this.#status === 'closed') {
+      this.logger.warn('Cannot update service resources when the runtime is stopping or closed')
+      return
+    }
+
+    const ups = await this.#validateUpdateServiceResources(updates)
+    const config = this.#configManager.current
+
+    const report = []
+    for (const update of ups) {
+      const { serviceId, config: serviceConfig, workers, health, currentWorkers, currentHealth } = update
+
+      if (workers && health) {
+        const r = await this.#updateServiceWorkersAndHealth(serviceId, config, serviceConfig, workers, health, currentWorkers, currentHealth)
+        report.push({
+          service: serviceId,
+          workers: r.workers,
+          health: r.health
+        })
+      } else if (health) {
+        const r = await this.#updateServiceHealth(serviceId, config, serviceConfig, currentWorkers, currentHealth, health)
+        report.push({
+          service: serviceId,
+          health: r.health
+        })
+      } else if (workers) {
+        const r = await this.#updateServiceWorkers(serviceId, config, serviceConfig, workers, currentWorkers)
+        report.push({
+          service: serviceId,
+          workers: r.workers
+        })
+      }
+    }
+
+    return report
+  }
 
+  async #validateUpdateServiceResources (updates) {
     if (!Array.isArray(updates)) {
       throw new errors.InvalidArgumentError('updates', 'must be an array')
     }
@@ -1884,47 +2096,207 @@ class Runtime extends EventEmitter {
     }
 
     const config = this.#configManager.current
-
+    const validatedUpdates = []
     for (const update of updates) {
-      const { service: serviceId, workers } = update
+      const { service: serviceId } = update
 
-      if (typeof workers !== 'number') {
-        throw new errors.InvalidArgumentError('workers', 'must be a number')
-      }
       if (!serviceId) {
         throw new errors.InvalidArgumentError('service', 'must be a string')
       }
-      if (workers <= 0) {
-        throw new errors.InvalidArgumentError('workers', 'must be greater than 0')
-      }
-      if (workers > MAX_WORKERS) {
-        throw new errors.InvalidArgumentError('workers', `must be less than ${MAX_WORKERS}`)
-      }
       const serviceConfig = config.services.find(s => s.id === serviceId)
       if (!serviceConfig) {
         throw new errors.ServiceNotFoundError(serviceId, Array.from(this.#servicesIds).join(', '))
       }
 
-      const { workers: currentWorkers } = await this.getServiceResourcesInfo(serviceId)
-      if (currentWorkers === workers) {
-        this.logger.warn({ serviceId, workers }, 'No change in the number of workers for service')
-        continue
+      const { workers: currentWorkers, health: currentHealth } = await this.getServiceResourcesInfo(serviceId)
+
+      let workers
+      if (update.workers !== undefined) {
+        if (typeof update.workers !== 'number') {
+          throw new errors.InvalidArgumentError('workers', 'must be a number')
+        }
+        if (update.workers <= 0) {
+          throw new errors.InvalidArgumentError('workers', 'must be greater than 0')
+        }
+        if (update.workers > MAX_WORKERS) {
+          throw new errors.InvalidArgumentError('workers', `must be less than ${MAX_WORKERS}`)
+        }
+
+        if (currentWorkers === update.workers) {
+          this.logger.warn({ serviceId, workers: update.workers }, 'No change in the number of workers for service')
+        } else {
+          workers = update.workers
+        }
       }
 
-      if (currentWorkers < workers) {
-        await this.#updateWorkerCount(serviceId, workers)
+      let maxHeapTotal, maxYoungGeneration
+      if (update.health) {
+        if (update.health.maxHeapTotal !== undefined) {
+          if (typeof update.health.maxHeapTotal === 'string') {
+            try {
+              maxHeapTotal = parseMemorySize(update.health.maxHeapTotal)
+            } catch {
+              throw new errors.InvalidArgumentError('maxHeapTotal', 'must be a valid memory size')
+            }
+          } else if (typeof update.health.maxHeapTotal === 'number') {
+            maxHeapTotal = update.health.maxHeapTotal
+            if (update.health.maxHeapTotal <= 0) {
+              throw new errors.InvalidArgumentError('maxHeapTotal', 'must be greater than 0')
+            }
+          } else {
+            throw new errors.InvalidArgumentError('maxHeapTotal', 'must be a number or a string representing a memory size')
+          }
+
+          if (currentHealth.maxHeapTotal === maxHeapTotal) {
+            this.logger.warn({ serviceId, maxHeapTotal }, 'No change in the max heap total for service')
+            maxHeapTotal = undefined
+          }
+        }
+
+        if (update.health.maxYoungGeneration !== undefined) {
+          if (typeof update.health.maxYoungGeneration === 'string') {
+            try {
+              maxYoungGeneration = parseMemorySize(update.health.maxYoungGeneration)
+            } catch {
+              throw new errors.InvalidArgumentError('maxYoungGeneration', 'must be a valid memory size')
+            }
+          } else if (typeof update.health.maxYoungGeneration === 'number') {
+            maxYoungGeneration = update.health.maxYoungGeneration
+            if (update.health.maxYoungGeneration <= 0) {
+              throw new errors.InvalidArgumentError('maxYoungGeneration', 'must be greater than 0')
+            }
+          } else {
+            throw new errors.InvalidArgumentError('maxYoungGeneration', 'must be a number or a string representing a memory size')
+          }
+
+          if (currentHealth.maxYoungGeneration && currentHealth.maxYoungGeneration === maxYoungGeneration) {
+            this.logger.warn({ serviceId, maxYoungGeneration }, 'No change in the max young generation for service')
+            maxYoungGeneration = undefined
+          }
+        }
+      }
+
+      if (workers || maxHeapTotal || maxYoungGeneration) {
+        let health
+        if (maxHeapTotal || maxYoungGeneration) {
+          health = { }
+          if (maxHeapTotal) {
+            health.maxHeapTotal = maxHeapTotal
+          }
+          if (maxYoungGeneration) {
+            health.maxYoungGeneration = maxYoungGeneration
+          }
+        }
+        validatedUpdates.push({ serviceId, config: serviceConfig, workers, health, currentWorkers, currentHealth })
+      }
+    }
+
+    return validatedUpdates
+  }
+
+  async #updateServiceWorkersAndHealth (serviceId, config, serviceConfig, workers, health, currentWorkers, currentHealth) {
+    if (currentWorkers > workers) {
+      // stop workers
+      const reportWorkers = await this.#updateServiceWorkers(serviceId, config, serviceConfig, workers, currentWorkers)
+      // update heap for current workers
+      const reportHealth = await this.#updateServiceHealth(serviceId, config, serviceConfig, workers, currentHealth, health)
+
+      return { workers: reportWorkers, health: reportHealth }
+    } else {
+      // update service heap
+      await this.#updateServiceConfigHealth(serviceId, health)
+      // start new workers with new heap
+      const reportWorkers = await this.#updateServiceWorkers(serviceId, config, serviceConfig, workers, currentWorkers)
+      // update heap for current workers
+      const reportHealth = await this.#updateServiceHealth(serviceId, config, serviceConfig, currentWorkers, currentHealth, health, false)
+
+      return { workers: reportWorkers, health: reportHealth }
+    }
+  }
+
+  async #updateServiceHealth (serviceId, config, serviceConfig, currentWorkers, currentHealth, health, updateConfig = true) {
+    const report = {
+      current: currentHealth,
+      new: health,
+      updated: []
+    }
+    try {
+      if (updateConfig) {
+        await this.#updateServiceConfigHealth(serviceId, health)
+      }
+
+      for (let i = 0; i < currentWorkers; i++) {
+        this.logger.info({ health: { current: currentHealth, new: health } }, `Restarting service "${serviceId}" worker ${i} to update config health heap...`)
+
+        const worker = await this.#getWorkerById(serviceId, i)
+        if (health.maxHeapTotal) { worker[kConfig].health.maxHeapTotal = health.maxHeapTotal }
+        if (health.maxYoungGeneration) { worker[kConfig].health.maxYoungGeneration = health.maxYoungGeneration }
+
+        await this.#replaceWorker(config, serviceConfig, currentWorkers, serviceId, i, worker)
+        report.updated.push(i)
+        this.logger.info({ health: { current: currentHealth, new: health } }, `Restarted service "${serviceId}" worker ${i}`)
+      }
+      report.success = true
+    } catch (err) {
+      if (report.updated.length < 1) {
+        this.logger.error({ err }, 'Cannot update service health heap, no worker updated')
+        await this.#updateServiceConfigHealth(serviceId, currentHealth)
+      } else {
+        this.logger.error({ err }, `Cannot update service health heap, updated workers: ${report.updated.length} out of ${currentWorkers}`)
+      }
+      report.success = false
+    }
+    return report
+  }
+
+  async #updateServiceWorkers (serviceId, config, serviceConfig, workers, currentWorkers) {
+    const report = {
+      current: currentWorkers,
+      new: workers
+    }
+    if (currentWorkers < workers) {
+      report.started = []
+      try {
+        await this.#updateServiceConfigWorkers(serviceId, workers)
         for (let i = currentWorkers; i < workers; i++) {
           await this.#setupWorker(config, serviceConfig, workers, serviceId, i)
           await this.#startWorker(config, serviceConfig, workers, serviceId, i, false, 0)
+          report.started.push(i)
         }
-      } else {
+        report.success = true
+      } catch (err) {
+        if (report.started.length < 1) {
+          this.logger.error({ err }, 'Cannot start service workers, no worker started')
+          await this.#updateServiceConfigWorkers(serviceId, currentWorkers)
+        } else {
+          this.logger.error({ err }, `Cannot start service workers, started workers: ${report.started.length} out of ${workers}`)
+          await this.#updateServiceConfigWorkers(serviceId, currentWorkers + report.started.length)
+        }
+        report.success = false
+      }
+    } else {
+      // keep the current workers count until all the service workers are all stopped
+      report.stopped = []
+      try {
         for (let i = currentWorkers - 1; i >= workers; i--) {
-          // keep the current workers count until the workers are stopped
-          await this.#stopWorker(currentWorkers, serviceId, i, false)
+          const worker = await this.#getWorkerById(serviceId, i, false, false)
+          await sendViaITC(worker, 'removeFromMesh')
+          await this.#stopWorker(currentWorkers, serviceId, i, false, worker)
+          report.stopped.push(i)
+        }
+        await this.#updateServiceConfigWorkers(serviceId, workers)
+        report.success = true
+      } catch (err) {
+        if (report.stopped.length < 1) {
+          this.logger.error({ err }, 'Cannot stop service workers, no worker stopped')
+        } else {
+          this.logger.error({ err }, `Cannot stop service workers, stopped workers: ${report.stopped.length} out of ${workers}`)
+          await this.#updateServiceConfigWorkers(serviceId, currentWorkers - report.stopped)
         }
-        await this.#updateWorkerCount(serviceId, workers)
+        report.success = false
       }
     }
+    return report
   }
 }