@platformatic/runtime 2.7.1-alpha.2 → 2.8.0-alpha.2

package/lib/runtime.js

@@ -16,16 +16,27 @@ const errors = require('./errors')
 const { createLogger } = require('./logger')
 const { startManagementApi } = require('./management-api')
 const { startPrometheusServer } = require('./prom-server')
-const { createSharedStore } = require('./shared-http-cache')
 const { getRuntimeTmpDir } = require('./utils')
 const { sendViaITC, waitEventFromITC } = require('./worker/itc')
-const { kId, kITC, kConfig } = require('./worker/symbols')
-
-const Fastify = require('fastify')
+const { RoundRobinMap } = require('./worker/round-robin-map.js')
+const {
+  kId,
+  kServiceId,
+  kWorkerId,
+  kITC,
+  kConfig,
+  kLoggerDestination,
+  kLoggingPort,
+  kWorkerStatus
+} = require('./worker/symbols')
+
+const fastify = require('fastify')
 
 const platformaticVersion = require('../package.json').version
 const kWorkerFile = join(__dirname, 'worker/main.js')
 
+const kInspectorOptions = Symbol('plt.runtime.worker.inspectorOptions')
+
 const MAX_LISTENERS_COUNT = 100
 const MAX_METRICS_QUEUE_LENGTH = 5 * 60 // 5 minutes in seconds
 const COLLECT_METRICS_TIMEOUT = 1000
@@ -34,12 +45,11 @@ const MAX_BOOTSTRAP_ATTEMPTS = 5
 
 class Runtime extends EventEmitter {
   #configManager
+  #isProduction
   #runtimeTmpDir
   #runtimeLogsDir
   #env
-  #services
   #servicesIds
-  #entrypoint
   #entrypointId
   #url
   #loggerDestination
@@ -49,12 +59,9 @@ class Runtime extends EventEmitter {
   #interceptor
   #managementApi
   #prometheusServer
-  #startedServices
-  #restartPromises
-  #bootstrapAttempts
-  #inspectors
   #inspectorServer
-  #sharedHttpCache
+  #workers
+  #restartingWorkers
 
   constructor (configManager, runtimeLogsDir, env) {
     super()
@@ -64,17 +71,13 @@ class Runtime extends EventEmitter {
     this.#runtimeTmpDir = getRuntimeTmpDir(configManager.dirname)
     this.#runtimeLogsDir = runtimeLogsDir
     this.#env = env
-    this.#services = new Map()
+    this.#workers = new RoundRobinMap()
     this.#servicesIds = []
     this.#url = undefined
     // Note: nothing hits the main thread so there is no reason to set the globalDispatcher here
     this.#interceptor = createThreadInterceptor({ domain: '.plt.local', timeout: true })
     this.#status = undefined
-    this.#startedServices = new Map()
-    this.#restartPromises = new Map()
-    this.#bootstrapAttempts = new Map()
-    this.#inspectors = []
-    this.#sharedHttpCache = null
+    this.#restartingWorkers = new Map()
   }
 
   async init () {
@@ -97,19 +100,21 @@ class Runtime extends EventEmitter {
     this.logger = logger
     this.#loggerDestination = destination
 
+    this.#isProduction = this.#configManager.args?.production ?? false
+    this.#servicesIds = config.services.map(service => service.id)
+    this.#workers.configure(config.services, this.#configManager.current.workers, this.#isProduction)
+
     // Create all services, each in is own worker thread
     for (const serviceConfig of config.services) {
-      // Setup forwarding of logs from the worker threads to the main thread
       await this.#setupService(serviceConfig)
     }
 
     try {
       // Make sure the list exists before computing the dependencies, otherwise some services might not be stopped
-      this.#servicesIds = config.services.map(service => service.id)
 
       if (autoloadEnabled) {
         checkDependencies(config.services)
-        this.#services = topologicalSort(this.#services, config)
+        this.#workers = topologicalSort(this.#workers, config)
       }
 
       // Recompute the list of services after sorting
@@ -119,15 +124,10 @@ class Runtime extends EventEmitter {
       throw e
     }
 
-    this.#sharedHttpCache = createSharedStore(
-      this.#configManager.dirname,
-      config.httpCache
-    )
-
     this.#updateStatus('init')
   }
 
-  async start () {
+  async start (silent = false) {
     if (typeof this.#configManager.current.entrypoint === 'undefined') {
       throw new errors.MissingEntrypointError()
     }
@@ -136,33 +136,41 @@ class Runtime extends EventEmitter {
     // Important: do not use Promise.all here since it won't properly manage dependencies
     try {
       for (const service of this.#servicesIds) {
-        await this.startService(service)
+        await this.startService(service, silent)
       }
 
       if (this.#configManager.current.inspectorOptions) {
         const { port } = this.#configManager.current.inspectorOptions
 
-        const server = Fastify({
+        const server = fastify({
           loggerInstance: this.logger.child({ name: 'inspector' }, { level: 'warn' })
         })
 
-        const version = await fetch(`http://127.0.0.1:${this.#configManager.current.inspectorOptions.port + 1}/json/version`).then((res) => res.json())
-
-        const data = (await Promise.all(this.#inspectors.map(async (data) => {
-          const res = await fetch(`http://127.0.0.1:${data.port}/json/list`)
-          const details = await res.json()
-          return {
-            ...details[0],
-            title: data.id
-          }
-        })))
+        const version = await fetch(
+          `http://127.0.0.1:${this.#configManager.current.inspectorOptions.port + 1}/json/version`
+        ).then(res => res.json())
+
+        const data = await Promise.all(
+          Array.from(this.#workers.values()).map(async worker => {
+            const data = worker[kInspectorOptions]
+
+            const res = await fetch(`http://127.0.0.1:${data.port}/json/list`)
+            const details = await res.json()
+            return {
+              ...details[0],
+              title: data.id
+            }
+          })
+        )
 
         server.get('/json/list', () => data)
         server.get('/json', () => data)
         server.get('/json/version', () => version)
 
         await server.listen({ port })
-        this.logger.info('The inspector server is now listening for all services. Open `chrome://inspect` in Google Chrome to connect.')
+        this.logger.info(
+          'The inspector server is now listening for all services. Open `chrome://inspect` in Google Chrome to connect.'
+        )
         this.#inspectorServer = server
       }
     } catch (error) {
@@ -188,13 +196,14 @@ class Runtime extends EventEmitter {
     }
 
     this.#updateStatus('stopping')
-    this.#startedServices.clear()
 
     if (this.#inspectorServer) {
       await this.#inspectorServer.close()
     }
 
-    await Promise.all(this.#servicesIds.map(service => this._stopService(service, silent)))
+    for (const service of this.#servicesIds) {
+      await this.stopService(service, silent)
+    }
 
     this.#updateStatus('stopped')
   }
@@ -243,109 +252,48 @@ class Runtime extends EventEmitter {
       this.#loggerDestination = null
     }
 
-    if (this.#sharedHttpCache?.close) {
-      await this.#sharedHttpCache.close()
-    }
-
     this.#updateStatus('closed')
   }
 
-  async startService (id) {
-    if (this.#startedServices.get(id)) {
+  async startService (id, silent) {
+    // Since when a service is stopped the worker is deleted, we consider a service start if its first service
+    // is no longer in the init phase
+    const firstWorker = this.#workers.get(`${id}:0`)
+    if (firstWorker && firstWorker[kWorkerStatus] !== 'boot' && firstWorker[kWorkerStatus] !== 'init') {
       throw new errors.ApplicationAlreadyStartedError()
     }
 
-    // This is set here so that if the service fails while starting we track the status
-    this.#startedServices.set(id, true)
-
-    let service = await this.#getServiceById(id, false, false)
-
-    // The service was stopped, recreate the thread
-    if (!service) {
-      const config = this.#configManager.current
-      const serviceConfig = config.services.find(s => s.id === id)
+    const config = this.#configManager.current
+    const serviceConfig = config.services.find(s => s.id === id)
 
-      await this.#setupService(serviceConfig)
-      service = await this.#getServiceById(id)
+    if (!serviceConfig) {
+      throw new errors.ServiceNotFoundError(id, Array.from(this.#servicesIds).join(', '))
     }
 
-    try {
-      const serviceUrl = await sendViaITC(service, 'start')
-      if (serviceUrl) {
-        this.#url = serviceUrl
-      }
-      this.#bootstrapAttempts.set(id, 0)
-    } catch (error) {
-      // TODO: handle port allocation error here
-      if (error.code === 'EADDRINUSE') throw error
-
-      this.logger.error({ err: ensureLoggableError(error) }, `Failed to start service "${id}".`)
-
-      const config = this.#configManager.current
-      const restartOnError = config.restartOnError
-
-      if (!restartOnError) {
-        this.logger.error(`Failed to start service "${id}".`)
-        throw error
-      }
-
-      let bootstrapAttempt = this.#bootstrapAttempts.get(id)
-      if (bootstrapAttempt++ >= MAX_BOOTSTRAP_ATTEMPTS || restartOnError === 0) {
-        this.logger.error(`Failed to start service "${id}" after ${MAX_BOOTSTRAP_ATTEMPTS} attempts.`)
-        throw error
-      }
-
-      this.logger.warn(
-        `Starting a service "${id}" in ${restartOnError}ms. ` +
-          `Attempt ${bootstrapAttempt} of ${MAX_BOOTSTRAP_ATTEMPTS}...`
-      )
+    const workersCount = await this.#workers.getCount(serviceConfig.id)
 
-      this.#bootstrapAttempts.set(id, bootstrapAttempt)
-      await this.#restartCrashedService(id)
+    for (let i = 0; i < workersCount; i++) {
+      await this.#startWorker(config, serviceConfig, workersCount, id, i, silent)
     }
   }
 
-  // Do not rename to #stopService as this is used in tests
-  async _stopService (id, silent) {
-    const service = await this.#getServiceById(id, false, false)
-
-    if (!service) {
-      return
-    }
-
-    this.#startedServices.set(id, false)
-
-    if (!silent) {
-      this.logger?.info(`Stopping service "${id}"...`)
-    }
+  async stopService (id, silent) {
+    const config = this.#configManager.current
+    const serviceConfig = config.services.find(s => s.id === id)
 
-    // Always send the stop message, it will shut down workers that only had ITC and interceptors setup
-    try {
-      await executeWithTimeout(sendViaITC(service, 'stop'), 10000)
-    } catch (error) {
-      this.logger?.info(
-        { error: ensureLoggableError(error) },
-        `Failed to stop service "${id}". Killing a worker thread.`
-      )
-    } finally {
-      service[kITC].close()
+    if (!serviceConfig) {
+      throw new errors.ServiceNotFoundError(id, Array.from(this.#servicesIds).join(', '))
     }
 
-    // Wait for the worker thread to finish, we're going to create a new one if the service is ever restarted
-    const res = await executeWithTimeout(once(service, 'exit'), 10000)
+    const workersCount = await this.#workers.getCount(serviceConfig.id)
 
-    // If the worker didn't exit in time, kill it
-    if (res === 'timeout') {
-      await service.terminate()
+    for (let i = 0; i < workersCount; i++) {
+      await this.#stopWorker(workersCount, id, i, silent)
     }
   }
 
   async buildService (id) {
-    const service = this.#services.get(id)
-
-    if (!service) {
-      throw new errors.ServiceNotFoundError(id, Array.from(this.#services.keys()).join(', '))
-    }
+    const service = await this.#getServiceById(id)
 
     try {
       return await sendViaITC(service, 'build')
@@ -523,6 +471,7 @@ class Runtime extends EventEmitter {
   async getServices () {
     return {
       entrypoint: this.#entrypointId,
+      production: this.#isProduction,
       services: await Promise.all(this.#servicesIds.map(id => this.getServiceDetails(id)))
     }
   }
@@ -555,6 +504,10 @@ class Runtime extends EventEmitter {
       dependencies
     }
 
+    if (this.#isProduction) {
+      serviceDetails.workers = this.#workers.getCount(id)
+    }
+
     if (entrypoint) {
       serviceDetails.url = status === 'started' ? this.#url : null
     }
@@ -593,16 +546,14 @@ class Runtime extends EventEmitter {
   async getMetrics (format = 'json') {
     let metrics = null
 
-    for (const id of this.#servicesIds) {
+    for (const worker of this.#workers.values()) {
       try {
-        const service = await this.#getServiceById(id, true, false)
-
         // The service might be temporarily unavailable
-        if (!service) {
+        if (worker[kWorkerStatus] !== 'started') {
           continue
         }
 
-        const serviceMetrics = await sendViaITC(service, 'getMetrics', format)
+        const serviceMetrics = await sendViaITC(worker, 'getMetrics', format)
         if (serviceMetrics) {
           if (metrics === null) {
             metrics = format === 'json' ? [] : ''
@@ -711,11 +662,7 @@ class Runtime extends EventEmitter {
   }
 
   async getServiceMeta (id) {
-    const service = this.#services.get(id)
-
-    if (!service) {
-      throw new errors.ServiceNotFoundError(id, Array.from(this.#services.keys()).join(', '))
-    }
+    const service = await this.#getServiceById(id)
 
     try {
       return await sendViaITC(service, 'getServiceMeta')
@@ -767,24 +714,6 @@ class Runtime extends EventEmitter {
     return createReadStream(filePath)
   }
 
-  async getCachedHttpRequests () {
-    return this.#sharedHttpCache.getRoutes()
-  }
-
-  async invalidateHttpCache (options = {}) {
-    const { origin, routes, tags } = options
-
-    if (!this.#sharedHttpCache) return
-
-    if (routes && routes.length > 0) {
-      await this.#sharedHttpCache.deleteRoutes(routes)
-    }
-
-    if (tags && tags.length > 0) {
-      await this.#sharedHttpCache.deleteByCacheTags(origin, tags)
-    }
-  }
-
   #updateStatus (status) {
     this.#status = status
     this.emit(status)
@@ -798,16 +727,21 @@ class Runtime extends EventEmitter {
     if (this.#status === 'stopping' || this.#status === 'closed') return
 
     const config = this.#configManager.current
+    const workersCount = await this.#workers.getCount(serviceConfig.id)
+    const id = serviceConfig.id
+
+    for (let i = 0; i < workersCount; i++) {
+      await this.#setupWorker(config, serviceConfig, workersCount, id, i)
+    }
+  }
+
+  async #setupWorker (config, serviceConfig, workersCount, serviceId, index) {
     const { autoload, restartOnError } = config
+    const workerId = `${serviceId}:${index}`
 
-    const id = serviceConfig.id
     const { port1: loggerDestination, port2: loggingPort } = new MessageChannel()
     loggerDestination.on('message', this.#forwardThreadLog.bind(this))
 
-    if (!this.#bootstrapAttempts.has(id)) {
-      this.#bootstrapAttempts.set(id, 0)
-    }
-
     // Handle inspector
     let inspectorOptions
 
@@ -816,23 +750,20 @@ class Runtime extends EventEmitter {
         ...this.#configManager.current.inspectorOptions
       }
 
-      inspectorOptions.port = inspectorOptions.port + this.#inspectors.length + 1
-
-      const inspectorData = {
-        port: inspectorOptions.port,
-        id,
-        dirname: this.#configManager.dirname
-      }
-
-      this.#inspectors.push(inspectorData)
+      inspectorOptions.port = inspectorOptions.port + this.#workers.size + 1
     }
 
-    const service = new Worker(kWorkerFile, {
+    const worker = new Worker(kWorkerFile, {
       workerData: {
         config,
         serviceConfig: {
           ...serviceConfig,
-          isProduction: this.#configManager.args?.production ?? false
+          isProduction: this.#isProduction
+        },
+        worker: {
+          id: workerId,
+          index,
+          count: workersCount
         },
         inspectorOptions,
         dirname: this.#configManager.dirname,
@@ -854,100 +785,112 @@ class Runtime extends EventEmitter {
     })
 
     // Make sure the listener can handle a lot of API requests at once before raising a warning
-    service.setMaxListeners(1e3)
+    worker.setMaxListeners(1e3)
 
     // Track service exiting
-    service.once('exit', code => {
-      const started = this.#startedServices.get(id)
-      this.#services.delete(id)
-      loggerDestination.close()
-      service[kITC].close()
-      loggingPort.close()
+    worker.once('exit', code => {
+      if (worker[kWorkerStatus] === 'exited') {
+        return
+      }
 
-      if (this.#status === 'stopping') return
+      const started = worker[kWorkerStatus] === 'started'
+      worker[kWorkerStatus] = 'exited'
+
+      this.#cleanupWorker(workerId, worker)
+
+      if (this.#status === 'stopping') {
+        return
+      }
 
       // Wait for the next tick so that crashed from the thread are logged first
       setImmediate(() => {
-        if (!config.watch || code !== 0) {
-          this.logger.warn(`Service "${id}" unexpectedly exited with code ${code}.`)
+        const errorLabel = workersCount > 1 ? `worker ${index} of the service "${serviceId}"` : `service "${serviceId}"`
+
+        if (started && (!config.watch || code !== 0)) {
+          this.logger.warn(`The ${errorLabel} unexpectedly exited with code ${code}.`)
         }
 
         // Restart the service if it was started
         if (started && this.#status === 'started') {
           if (restartOnError > 0) {
-            this.logger.warn(`Restarting a service "${id}" in ${restartOnError}ms...`)
-            this.#restartCrashedService(id).catch(err => {
-              this.logger.error({ err: ensureLoggableError(err) }, `Failed to restart service "${id}".`)
+            this.logger.warn(`The ${errorLabel} will be restarted in ${restartOnError}ms...`)
+            this.#restartCrashedWorker(config, serviceConfig, workersCount, serviceId, index, false, 0).catch(err => {
+              this.logger.error({ err: ensureLoggableError(err) }, `${errorLabel} could not be restarted.`)
             })
           } else {
-            this.logger.warn(`The "${id}" service is no longer available.`)
+            this.logger.warn(`The ${errorLabel} is no longer available.`)
           }
         }
       })
     })
 
-    service[kId] = id
-    service[kConfig] = serviceConfig
+    worker[kId] = workersCount > 1 ? workerId : serviceId
+    worker[kServiceId] = serviceId
+    worker[kWorkerId] = workersCount > 1 ? index : undefined
+    worker[kConfig] = serviceConfig
+    worker[kLoggerDestination] = loggerDestination
+    worker[kLoggingPort] = loggingPort
+
+    if (inspectorOptions) {
+      worker[kInspectorOptions] = {
+        port: inspectorOptions.port,
+        id: serviceId,
+        dirname: this.#configManager.dirname
+      }
+    }
 
     // Setup ITC
-    service[kITC] = new ITC({
-      name: id + '-runtime',
-      port: service,
+    worker[kITC] = new ITC({
+      name: workerId + '-runtime',
+      port: worker,
       handlers: {
         getServiceMeta: this.getServiceMeta.bind(this),
-        getServices: this.getServices.bind(this),
-        isHttpCacheFull: () => this.#sharedHttpCache.isFull(),
-        getHttpCacheValue: opts => this.#sharedHttpCache.getValue(opts.request),
-        setHttpCacheValue: opts => this.#sharedHttpCache.setValue(
-          opts.request,
-          opts.response,
-          opts.payload
-        ),
-        deleteHttpCacheValue: opts => this.#sharedHttpCache.deleteByOrigin(
-          opts.origin
-        ),
-        invalidateHttpCache: opts => this.invalidateHttpCache(opts),
+        listServices: () => {
+          return this.#servicesIds
+        }
       }
     })
-    service[kITC].listen()
-
-    // Handle services changes
-    // This is not purposely activated on when this.#configManager.current.watch === true
-    // so that services can eventually manually trigger a restart. This mechanism is current
-    // used by the composer
-    service[kITC].on('changed', async () => {
-      try {
-        const wasStarted = this.#startedServices.get(id)
-
-        await this._stopService(id)
+    worker[kITC].listen()
+
+    // Only activate watch for the first instance
+    if (index === 0) {
+      // Handle services changes
+      // This is not purposely activated on when this.#configManager.current.watch === true
+      // so that services can eventually manually trigger a restart. This mechanism is current
+      // used by the composer.
+      worker[kITC].on('changed', async () => {
+        try {
+          const wasStarted = worker[kWorkerStatus].startsWith('start')
+          await this.stopService(serviceId)
 
-        if (wasStarted) {
-          await this.startService(id)
-        }
+          if (wasStarted) {
+            await this.startService(serviceId)
+          }
 
-        this.logger?.info(`Service ${id} has been successfully reloaded ...`)
+          this.logger?.info(`Service "${serviceId}" has been successfully reloaded ...`)
 
-        if (serviceConfig.entrypoint) {
-          this.#showUrl()
+          if (serviceConfig.entrypoint) {
+            this.#showUrl()
+          }
+        } catch (e) {
+          this.logger?.error(e)
         }
-      } catch (e) {
-        this.logger?.error(e)
-      }
-    })
+      })
+    }
 
     // Store locally
-    this.#services.set(id, service)
+    this.#workers.set(workerId, worker)
 
     if (serviceConfig.entrypoint) {
-      this.#entrypoint = service
-      this.#entrypointId = id
+      this.#entrypointId = serviceId
     }
 
     // Setup the interceptor
-    this.#interceptor.route(id, service)
+    this.#interceptor.route(serviceId, worker)
 
     // Store dependencies
-    const [{ dependencies }] = await waitEventFromITC(service, 'init')
+    const [{ dependencies }] = await waitEventFromITC(worker, 'init')
+    worker[kWorkerStatus] = 'boot'
 
     if (autoload) {
       serviceConfig.dependencies = dependencies
@@ -959,11 +902,122 @@ class Runtime extends EventEmitter {
     }
   }
 
-  async #restartCrashedService (id) {
-    const config = this.#configManager.current
-    const serviceConfig = config.services.find(s => s.id === id)
+  async #startWorker (config, serviceConfig, workersCount, id, index, silent, bootstrapAttempt = 0) {
+    const workerId = `${id}:${index}`
+    const label = workersCount > 1 ? `worker ${index} of the service "${id}"` : `service "${id}"`
+
+    if (!silent) {
+      this.logger?.info(`Starting the ${label}...`)
+    }
+
+    let worker = await this.#getWorkerById(id, index, false, false)
+
+    // The service was stopped, recreate the thread
+    if (!worker) {
+      await this.#setupService(serviceConfig, index)
+      worker = await this.#getWorkerById(id, index)
+    }
+
+    worker[kWorkerStatus] = 'starting'
 
-    let restartPromise = this.#restartPromises.get(id)
+    try {
+      const workerUrl = await sendViaITC(worker, 'start')
+      if (workerUrl) {
+        this.#url = workerUrl
+      }
+
+      worker[kWorkerStatus] = 'started'
+
+      if (!silent) {
+        this.logger?.info(`Started the ${label}...`)
+      }
+    } catch (error) {
+      // TODO: handle port allocation error here
+      if (error.code === 'EADDRINUSE') throw error
+
+      this.#cleanupWorker(workerId, worker)
+
+      if (worker[kWorkerStatus] !== 'exited') {
+        // This prevent the exit handler to restart service
+        worker[kWorkerStatus] = 'exited'
+        await worker.terminate()
+      }
+
+      this.logger.error({ err: ensureLoggableError(error) }, `Failed to start ${label}.`)
+
+      const restartOnError = config.restartOnError
+
+      if (!restartOnError) {
+        throw error
+      }
+
+      if (bootstrapAttempt++ >= MAX_BOOTSTRAP_ATTEMPTS || restartOnError === 0) {
+        this.logger.error(`Failed to start ${label} after ${MAX_BOOTSTRAP_ATTEMPTS} attempts.`)
+        throw error
+      }
+
+      this.logger.warn(
+        `Attempt ${bootstrapAttempt} of ${MAX_BOOTSTRAP_ATTEMPTS} to start the ${label} again will be performed in ${restartOnError}ms ...`
+      )
+
+      await this.#restartCrashedWorker(config, serviceConfig, workersCount, id, index, silent, bootstrapAttempt)
+    }
+  }
+
+  async #stopWorker (workersCount, id, index, silent) {
+    const worker = await this.#getWorkerById(id, index, false, false)
+
+    if (!worker) {
+      return
+    }
+
+    worker[kWorkerStatus] = 'stopping'
+
+    const label = workersCount > 1 ? `worker ${index} of the service "${id}"` : `service "${id}"`
+
+    if (!silent) {
+      this.logger?.info(`Stopping the ${label}...`)
+    }
+
+    const exitTimeout = this.#configManager.current.gracefulShutdown.runtime
+    const exitPromise = once(worker, 'exit')
+
+    // Always send the stop message, it will shut down workers that only had ITC and interceptors setup
+    try {
+      await executeWithTimeout(sendViaITC(worker, 'stop'), exitTimeout)
+    } catch (error) {
+      this.logger?.info({ error: ensureLoggableError(error) }, `Failed to stop ${label}. Killing a worker thread.`)
+    } finally {
+      worker[kITC].close()
+    }
+
+    if (!silent) {
+      this.logger?.info(`Stopped the ${label}...`)
+    }
+
+    // Wait for the worker thread to finish, we're going to create a new one if the service is ever restarted
+    const res = await executeWithTimeout(exitPromise, exitTimeout)
+
+    // If the worker didn't exit in time, kill it
+    if (res === 'timeout') {
+      await worker.terminate()
+    }
+
+    worker[kWorkerStatus] = 'stopped'
+  }
+
+  #cleanupWorker (workerId, worker) {
+    this.#workers.delete(workerId)
+
+    worker[kITC].close()
+    worker[kLoggerDestination].close()
+    worker[kLoggingPort].close()
+  }
+
+  async #restartCrashedWorker (config, serviceConfig, workersCount, id, index, silent, bootstrapAttempt) {
+    const workerId = `${id}:${index}`
+
+    let restartPromise = this.#restartingWorkers.get(workerId)
     if (restartPromise) {
       await restartPromise
       return
@@ -971,48 +1025,67 @@ class Runtime extends EventEmitter {
 
     restartPromise = new Promise((resolve, reject) => {
       setTimeout(async () => {
-        this.#restartPromises.delete(id)
+        this.#restartingWorkers.delete(workerId)
 
         try {
-          await this.#setupService(serviceConfig)
-
-          const started = this.#startedServices.get(id)
-          if (started) {
-            this.#startedServices.set(id, false)
-            await this.startService(id)
-          }
+          await this.#setupWorker(config, serviceConfig, workersCount, id, index)
+          await this.#startWorker(config, serviceConfig, workersCount, id, index, silent, bootstrapAttempt)
 
           resolve()
         } catch (err) {
+          // The runtime was stopped while the restart was happening, ignore any error.
+          if (!this.#status.startsWith('start')) {
+            resolve()
+          }
+
           reject(err)
         }
       }, config.restartOnError)
     })
 
-    this.#restartPromises.set(id, restartPromise)
+    this.#restartingWorkers.set(workerId, restartPromise)
     await restartPromise
   }
 
-  async #getServiceById (id, ensureStarted = false, mustExist = true) {
-    const service = this.#services.get(id)
+  async #getServiceById (serviceId, ensureStarted = false, mustExist = true) {
+    // If the serviceId includes the worker, properly split
+    let workerId
+    const matched = serviceId.match(/^(.+):(\d+)$/)
+
+    if (matched) {
+      serviceId = matched[1]
+      workerId = matched[2]
+    }
+
+    return this.#getWorkerById(serviceId, workerId, ensureStarted, mustExist)
+  }
+
+  async #getWorkerById (serviceId, workerId, ensureStarted = false, mustExist = true) {
+    let worker
+
+    if (typeof workerId !== 'undefined') {
+      worker = this.#workers.get(`${serviceId}:${workerId}`)
+    } else {
+      worker = this.#workers.next(serviceId)
+    }
 
-    if (!service) {
-      if (!mustExist && this.#servicesIds.includes(id)) {
+    if (!worker) {
+      if (!mustExist && this.#servicesIds.includes(serviceId)) {
         return null
       }
 
-      throw new errors.ServiceNotFoundError(id, Array.from(this.#services.keys()).join(', '))
+      throw new errors.ServiceNotFoundError(serviceId, Array.from(this.#servicesIds).join(', '))
     }
 
     if (ensureStarted) {
-      const serviceStatus = await sendViaITC(service, 'getStatus')
+      const serviceStatus = await sendViaITC(worker, 'getStatus')
 
       if (serviceStatus !== 'started') {
-        throw new errors.ServiceNotStartedError(id)
+        throw new errors.ServiceNotStartedError(serviceId)
       }
     }
 
-    return service
+    return worker
   }
 
   async #getRuntimePackageJson () {