@platformatic/runtime 3.61.0 → 3.62.0

package/lib/runtime.js

@@ -23,7 +23,7 @@ import { readFile } from 'node:fs/promises'
 import { STATUS_CODES } from 'node:http'
 import { createRequire } from 'node:module'
 import { availableParallelism } from 'node:os'
-import { dirname, isAbsolute, join } from 'node:path'
+import { basename, dirname, isAbsolute, join } from 'node:path'
 import { Readable } from 'node:stream'
 import { finished } from 'node:stream/promises'
 import { setImmediate as immediate, setTimeout as sleep } from 'node:timers/promises'
@@ -39,9 +39,14 @@ import {
   ApplicationNotStartedError,
   ApplicationStartTimeoutError,
   CannotRemoveEntrypointError,
+  DuplicateITCHandlerNameError,
+  FailedToLoadExtensionError,
   InvalidArgumentError,
+  InvalidExtensionError,
+  LastProfileTimeoutError,
   MessagingError,
   MissingPprofCapture,
+  ReservedITCHandlerNameError,
   RuntimeAbortedError,
   WorkerInterceptorJoinTimeoutError,
   WorkerNotFoundError
@@ -82,7 +87,25 @@ const kWorkerFile = join(import.meta.dirname, 'worker/main.js')
 const kInspectorOptions = Symbol('plt.runtime.worker.inspectorOptions')
 const kHeapCheckCounter = Symbol('plt.runtime.worker.heapCheckCounter')
 const kLastHeapStats = Symbol('plt.runtime.worker.lastHeapStats')
+const kProfilingELUGates = Symbol('plt.runtime.worker.profilingELUGates')
 const kHealthITCTimeoutMs = 5000
+const kProfilingELUHysteresis = 0.1
+const kLastProfileTimeoutMs = 10000
+
+// getApplicationLastProfile falls back to the preserved overload profile
+// whenever the worker cannot currently provide one: it is gone, profiling was
+// not (re)started, no window has completed yet (e.g. profiling was just
+// restarted on a replacement worker) or the profiler is paused with no recent
+// window. The returned timestamp lets consumers judge freshness. Any other
+// error is rethrown.
+const kLastProfileFallbackCodes = new Set([
+  'PLT_RUNTIME_APPLICATION_NOT_FOUND',
+  'PLT_RUNTIME_WORKER_NOT_FOUND',
+  'PLT_RUNTIME_APPLICATION_NOT_STARTED',
+  'PLT_PPROF_PROFILING_NOT_STARTED',
+  'PLT_PPROF_NO_PROFILE_AVAILABLE',
+  'PLT_PPROF_NOT_ENOUGH_ELU'
+])
 const kApplicationRestartsMetricName = 'platformatic_application_restarts_total'
 const kApplicationRestartsMetricHelp = 'Total number of restarts triggered by the runtime for an application.'
 
@@ -103,11 +126,22 @@ function parseOrigins (origins) {
   })
 }
 
+function formatMetricValue (value) {
+  if (Number.isNaN(value)) {
+    return 'NaN'
+  } else if (!Number.isFinite(value)) {
+    return value < 0 ? '-Inf' : '+Inf'
+  }
+
+  return `${value}`
+}
+
 // Always run operations in parallel to avoid deadlocks when services have dependencies
 const DEFAULT_CONCURRENCY = availableParallelism() * 2
 const MAX_BOOTSTRAP_ATTEMPTS = 5
 const IMMEDIATE_RESTART_MAX_THRESHOLD = 10
 const MAX_WORKERS = 100
+const DEFAULT_RESTART_ON_ERROR_DELAY = 5000
 
 export class Runtime extends EventEmitter {
   logger
@@ -134,6 +168,7 @@ export class Runtime extends EventEmitter {
   #entrypointPort
 
   #healthMetricsTimer
+  #healthMetricsCollectionActive
 
   #meshInterceptor
   #dispatcher
@@ -151,10 +186,15 @@ export class Runtime extends EventEmitter {
   #workers
   #workersBroadcastChannel
   #workerITCHandlers
+  #reservedITCHandlerNames
+  #extensions
+  #extensionsWantHealthMetrics
+  #lastOverloadProfiles
   #restartingApplications
   #restartingWorkers
   #dynamicWorkersScaler
   #nextWorkerIndex
+  #workerPortOffsets
 
   #sharedHttpCache
   #scheduler
@@ -196,6 +236,7 @@ export class Runtime extends EventEmitter {
     this.#restartingApplications = new Set()
     this.#restartingWorkers = new Map()
     this.#nextWorkerIndex = new Map()
+    this.#workerPortOffsets = new Map() // fullWorkerId => portOffset
     this.#sharedHttpCache = null
     this.#applicationsConfigsPatches = new Map()
 
@@ -220,6 +261,12 @@ export class Runtime extends EventEmitter {
       getSharedContext: this.getSharedContext.bind(this),
       sendHealthSignals: this.#processHealthSignals.bind(this)
     }
+    this.#reservedITCHandlerNames = new Set(Object.keys(this.#workerITCHandlers))
+    // Registered per-worker in #setupWorker, reserved so extensions cannot clobber it
+    this.#reservedITCHandlerNames.add('profiling:started')
+    this.#extensions = []
+    this.#extensionsWantHealthMetrics = false
+    this.#lastOverloadProfiles = new Map()
     this.#sharedContext = {}
 
     if (this.#isProduction) {
@@ -284,6 +331,10 @@ export class Runtime extends EventEmitter {
       }
     }
 
+    // Load extensions before creating any worker so that custom ITC handlers
+    // registered by the extensions are available to all workers.
+    await this.#loadExtensions()
+
     await this.addApplications(this.#config.applications)
     await this.#setDispatcher(config.undici)
 
@@ -410,10 +461,14 @@ export class Runtime extends EventEmitter {
 
   async close (silent = false) {
     clearTimeout(this.#healthMetricsTimer)
+    this.#healthMetricsCollectionActive = false
+    this.#lastOverloadProfiles.clear()
 
     await this.stop(silent)
     this.#updateStatus('closing')
 
+    await this.#closeExtensions()
+
     // The management API autocloses by itself via event in management-api.js.
     // This is needed to let management API stop endpoint to reply.
 
@@ -544,12 +599,19 @@ export class Runtime extends EventEmitter {
 
   async addApplications (applications, start = false) {
     const setupInvocations = []
+    // Per-worker ports do not need SO_REUSEPORT because each worker binds a different port.
+    const usesPerWorkerPorts = this.#config.server?.portAssignment === 'perWorkerIncrement'
 
     const toStart = []
     for (const application of applications) {
       const workers = application.workers
 
-      if ((workers.static > 1 || workers.minimum > 1) && application.entrypoint && !features.node.reusePort) {
+      if (
+        (workers.static > 1 || workers.minimum > 1) &&
+        application.entrypoint &&
+        !features.node.reusePort &&
+        !usesPerWorkerPorts
+      ) {
         this.logger.warn(
           `"${application.id}" is set as the entrypoint, but reusePort is not available in your OS; setting workers to 1 instead of ${workers.static}`
         )
@@ -809,6 +871,91 @@ export class Runtime extends EventEmitter {
     return sendViaITC(service, 'stopProfiling', options)
   }
 
+  async getApplicationLastProfile (id, options = {}, ensureStarted = true) {
+    this.#validatePprofCapturePreload()
+
+    const type = options.type ?? 'cpu'
+    const timeout = options.timeout ?? kLastProfileTimeoutMs
+    let error
+
+    try {
+      // Bound the whole retrieval with a single timeout budget: resolving the
+      // worker round-trips to it when ensureStarted is set, and the profile
+      // pull does too — both hang if the worker event loop is blocked. Attach
+      // a noop handler so that a late settlement after the timeout does not
+      // surface as an unhandled rejection.
+      const pull = (async () => {
+        const service = await this.#getApplicationById(id, ensureStarted)
+        const result = await sendViaITC(service, 'getLastProfile', { ...options, includeTimestamp: true })
+        return { service, result }
+      })()
+      pull.catch(() => {})
+
+      const outcome = await executeWithTimeout(pull, timeout, kTimeout)
+
+      if (outcome !== kTimeout) {
+        const { service, result } = outcome
+
+        // An older capture module which does not support includeTimestamp
+        // returns the raw profile.
+        const value = result instanceof Uint8Array ? { profile: result, timestamp: null } : result
+
+        // A strictly newer live window supersedes the preserved overload
+        // profile: prune it so the preserved copy naturally expires once the
+        // worker is healthy again and its profiles are being consumed.
+        if (value.timestamp != null) {
+          const key = `${service[kApplicationId]}:${service[kWorkerId]}:${type}`
+          const preserved = this.#lastOverloadProfiles.get(key)
+
+          if (preserved && preserved.timestamp < value.timestamp) {
+            this.#lastOverloadProfiles.delete(key)
+          }
+        }
+
+        return { ...value, preserved: false }
+      }
+
+      // The worker event loop is not responding (e.g. it is hard-blocked).
+      error = new LastProfileTimeoutError(id)
+    } catch (e) {
+      if (!kLastProfileFallbackCodes.has(e.code)) {
+        throw e
+      }
+
+      error = e
+    }
+
+    const preserved = this.#getPreservedOverloadProfile(id, type)
+
+    if (preserved) {
+      // The preserved flag lets consumers distinguish post-mortem evidence
+      // from a live window and judge it together with the timestamp.
+      return { profile: preserved.profile, timestamp: preserved.timestamp, preserved: true }
+    }
+
+    throw error
+  }
+
+  // The final profile of an overload pause is pushed by the worker and
+  // preserved in the main thread (see the profile:overload listener), so that
+  // the evidence of what saturated a worker survives the worker being blocked
+  // or replaced.
+  #getPreservedOverloadProfile (id, type) {
+    if (id.includes(':')) {
+      return this.#lastOverloadProfiles.get(`${id}:${type}`)
+    }
+
+    // Application-level id: return the most recent profile among its workers
+    let latest = null
+    for (const [key, entry] of this.#lastOverloadProfiles) {
+      if (key.startsWith(`${id}:`) && key.endsWith(`:${type}`) && (!latest || entry.timestamp > latest.timestamp)) {
+        latest = entry
+      }
+    }
+
+    return latest
+  }
+
   async takeApplicationHeapSnapshot (id, ensureStarted = true) {
     const service = await this.#getApplicationById(id, ensureStarted)
 
@@ -1242,9 +1389,7 @@ export class Runtime extends EventEmitter {
 
     let metrics = null
 
-    const applicationRestartMetrics = format === 'json'
-      ? this.#getApplicationRestartMetricsJson()
-      : this.#formatApplicationRestartMetricsText()
+    const applicationRestartMetrics = this.#getApplicationRestartMetricsJson()
 
     // Get process-level metrics once from main thread registry (if available)
     let processMetricsJson = null
@@ -1259,16 +1404,16 @@ export class Runtime extends EventEmitter {
           continue
         }
 
-        // Get thread-specific metrics from worker
+        // Get thread-specific metrics from worker. Always collect JSON so that
+        // the text format can be serialized once with a single HELP/TYPE block
+        // per metric family, as required by the Prometheus exposition format.
         const applicationMetrics = await executeWithTimeout(
-          sendViaITC(worker, 'getMetrics', format),
+          sendViaITC(worker, 'getMetrics', 'json'),
           this.#config.metrics?.timeout ?? 10000
         )
 
         if (applicationMetrics && applicationMetrics !== kTimeout) {
-          if (metrics === null) {
-            metrics = format === 'json' ? [] : ''
-          }
+          metrics ??= []
 
           // Build worker labels including custom labels from metrics config
           const workerLabels = {
@@ -1280,21 +1425,13 @@ export class Runtime extends EventEmitter {
             workerLabels.workerId = workerId
           }
 
-          if (format === 'json') {
-            // Duplicate process metrics with worker labels and add to output
-            if (processMetricsJson) {
-              this.#applyLabelsToMetrics(processMetricsJson, workerLabels, metrics)
-            }
-            // Add worker's thread-specific metrics
-            for (let i = 0; i < applicationMetrics.length; i++) {
-              metrics.push(applicationMetrics[i])
-            }
-          } else {
-            // Text format: format process metrics with worker labels
-            if (processMetricsJson) {
-              metrics += this.#formatProcessMetricsText(processMetricsJson, workerLabels)
-            }
-            metrics += applicationMetrics
+          // Duplicate process metrics with worker labels and add to output
+          if (processMetricsJson) {
+            this.#applyLabelsToMetrics(processMetricsJson, workerLabels, metrics)
+          }
+          // Add worker's thread-specific metrics
+          for (let i = 0; i < applicationMetrics.length; i++) {
+            metrics.push(applicationMetrics[i])
           }
         }
       } catch (e) {
@@ -1312,9 +1449,11 @@ export class Runtime extends EventEmitter {
     }
 
     if (metrics !== null && applicationRestartMetrics.length > 0) {
-      metrics = format === 'json'
-        ? [...applicationRestartMetrics, ...metrics]
-        : applicationRestartMetrics + metrics
+      metrics = [...applicationRestartMetrics, ...metrics]
+    }
+
+    if (metrics !== null && format !== 'json') {
+      metrics = this.#formatMetricsAsText(metrics)
     }
 
     return { metrics }
@@ -1351,28 +1490,6 @@ export class Runtime extends EventEmitter {
     return metrics
   }
 
-  #formatApplicationRestartMetricsText () {
-    let output = ''
-
-    for (const applicationId of this.#applications.keys()) {
-      const labelParts = []
-      const labels = this.#getApplicationRestartMetricLabels(applicationId)
-
-      for (const [key, val] of Object.entries(labels)) {
-        const escapedVal = String(val).replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, '\\n')
-        labelParts.push(`${key}="${escapedVal}"`)
-      }
-
-      const labelStr = labelParts.length > 0 ? `{${labelParts.join(',')}}` : ''
-
-      output += `# HELP ${kApplicationRestartsMetricName} ${kApplicationRestartsMetricHelp}\n`
-      output += `# TYPE ${kApplicationRestartsMetricName} counter\n`
-      output += `${kApplicationRestartsMetricName}${labelStr} ${this.#applicationRestartCounts.get(applicationId) ?? 0}\n`
-    }
-
-    return output
-  }
-
   // Apply labels to process metrics and push to output array (for JSON format)
   #applyLabelsToMetrics (processMetrics, labels, outputArray) {
     for (let i = 0; i < processMetrics.length; i++) {
@@ -1397,27 +1514,33 @@ export class Runtime extends EventEmitter {
     }
   }
 
-  // Format process metrics as Prometheus text format with labels
-  #formatProcessMetricsText (processMetricsJson, labels) {
-    let output = ''
+  // Serialize JSON metrics to the Prometheus text exposition format.
+  // Samples are grouped by metric family so each family is emitted as a single
+  // block with one HELP/TYPE header: the format forbids repeating them, and
+  // strict parsers (e.g. OpenMetrics-based ones like Dynatrace) would otherwise
+  // only ingest the first block for each metric name.
+  #formatMetricsAsText (metricsJson) {
+    const families = new Map()
 
-    for (let i = 0; i < processMetricsJson.length; i++) {
-      const metric = processMetricsJson[i]
-      const name = metric.name
-      const help = metric.help
-      const type = metric.type
+    for (const metric of metricsJson) {
+      let family = families.get(metric.name)
+      if (!family) {
+        family = { help: metric.help, type: metric.type, values: [] }
+        families.set(metric.name, family)
+      }
+      family.values.push(...metric.values)
+    }
 
-      // Add HELP and TYPE lines
-      output += `# HELP ${name} ${help}\n`
-      output += `# TYPE ${name} ${type}\n`
+    let output = ''
+    for (const [name, family] of families) {
+      const escapedHelp = String(family.help).replace(/\\/g, '\\\\').replace(/\n/g, '\\n')
+      output += `# HELP ${name} ${escapedHelp}\n`
+      output += `# TYPE ${name} ${family.type}\n`
 
-      const values = metric.values
-      for (let j = 0; j < values.length; j++) {
-        const v = values[j]
-        const combinedLabels = { ...labels, ...v.labels }
+      for (const v of family.values) {
         const labelParts = []
 
-        for (const [key, val] of Object.entries(combinedLabels)) {
+        for (const [key, val] of Object.entries(v.labels ?? {})) {
           // Escape label values for Prometheus format
           const escapedVal = String(val).replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, '\\n')
           labelParts.push(`${key}="${escapedVal}"`)
@@ -1425,7 +1548,7 @@ export class Runtime extends EventEmitter {
 
         const labelStr = labelParts.length > 0 ? `{${labelParts.join(',')}}` : ''
         const metricName = v.metricName || name
-        output += `${metricName}${labelStr} ${v.value}\n`
+        output += `${metricName}${labelStr} ${formatMetricValue(v.value)}\n`
       }
     }
 
@@ -1833,6 +1956,7 @@ export class Runtime extends EventEmitter {
     const setupInvocations = []
 
     for (let i = 0; i < workers; i++) {
+      this.#workerPortOffsets.set(`${id}:${i}`, i)
       setupInvocations.push([config, applicationConfig, workers, id, i])
     }
 
@@ -1846,7 +1970,7 @@ export class Runtime extends EventEmitter {
   }
 
   async #setupWorker (config, applicationConfig, workersCount, applicationId, index, enabled = true, attempt = 0) {
-    const { restartOnError } = config
+    const restartOnError = this.#getApplicationRestartOnError(config, applicationConfig)
     const workerId = `${applicationId}:${index}`
 
     // Handle inspector
@@ -1860,6 +1984,12 @@ export class Runtime extends EventEmitter {
       inspectorOptions.port = inspectorOptions.port + this.#workers.size + 1
     }
 
+    let serverConfigOverride
+    if (this.#config.server?.portAssignment === 'perWorkerIncrement') {
+      const portOffset = this.#workerPortOffsets.get(workerId)
+      serverConfigOverride = { port: Number(this.#config.server.port) + portOffset }
+    }
+
     if (config.telemetry) {
       applicationConfig.telemetry = {
         ...config.telemetry,
@@ -1923,12 +2053,21 @@ export class Runtime extends EventEmitter {
     const maxYoungGenerationSizeMb = maxYoungGeneration ? Math.floor(maxYoungGeneration / (1024 * 1024)) : undefined
     const codeRangeSizeMb = codeRangeSize ? Math.floor(codeRangeSize / (1024 * 1024)) : undefined
 
+    const workerConfig = {
+      ...config,
+      preload
+    }
+
+    if (config.server && serverConfigOverride) {
+      workerConfig.server = {
+        ...config.server,
+        ...serverConfigOverride
+      }
+    }
+
     const worker = new Worker(kWorkerFile, {
       workerData: {
-        config: {
-          ...config,
-          preload
-        },
+        config: workerConfig,
         applicationConfig: {
           ...applicationConfig,
           isProduction: this.#isProduction,
@@ -1977,6 +2116,7 @@ export class Runtime extends EventEmitter {
       worker[kWorkerStatus] = 'exited'
       this.emitAndNotify('application:worker:exited', eventPayload)
 
+      const portOffset = this.#workerPortOffsets.get(worker[kFullId]) ?? index
       this.#cleanupWorker(worker)
 
       if (this.#status === 'stopping') {
@@ -2001,7 +2141,7 @@ export class Runtime extends EventEmitter {
               this.logger.warn(`The ${errorLabel} will be restarted in ${restartOnError}ms ...`)
             }
 
-            this.#restartCrashedWorker(config, applicationConfig, workersCount, applicationId, index, false, 0).catch(
+            this.#restartCrashedWorker(config, applicationConfig, workersCount, applicationId, index, false, 0, portOffset).catch(
               err => {
                 this.logger.error({ err: ensureLoggableError(err) }, `${errorLabel} could not be restarted.`)
               }
@@ -2076,6 +2216,120 @@ export class Runtime extends EventEmitter {
       this.#opentelemetryMetricsForwarder?.collect(resourceMetrics)
     })
 
+    // The continuous profiler notifies us when a profile window is completed.
+    // The event only carries metadata: the profile can be retrieved on demand
+    // via getApplicationLastProfile. We use emit instead of emitAndNotify since
+    // other workers are not interested in this event.
+    worker[kITC].on('profile:captured', ({ type, timestamp }) => {
+      // A strictly newer completed window supersedes the preserved overload
+      // profile: once the worker is past the overload and producing windows
+      // again, its old evidence must not be served anymore.
+      const preservedKey = `${workerId}:${type}`
+      const preservedEntry = this.#lastOverloadProfiles.get(preservedKey)
+
+      if (preservedEntry && preservedEntry.timestamp < timestamp) {
+        this.#lastOverloadProfiles.delete(preservedKey)
+      }
+
+      this.emit('application:worker:profile:captured', {
+        id: workerId,
+        application: applicationId,
+        worker: index,
+        type,
+        timestamp
+      })
+    })
+
+    // The continuous profiler registers its gating needs when profiling starts.
+    // The main thread drives it based on the ELU measured by the health
+    // metrics cycle (see #applyProfilingELUGates): when an ELU threshold is
+    // set the profiler starts paused and only runs while the ELU is above it,
+    // and continuous profiling is paused while the worker ELU is above the
+    // maxELU cutoff so that profiling does not add overhead to an already
+    // overloaded worker. This is a request handler rather than a notification
+    // listener on purpose: the capture module can detect a runtime without
+    // the driver (PLT_ITC_HANDLER_NOT_FOUND) and fall back to ungated
+    // profiling instead of starting paused forever.
+    worker[kITC].handle('profiling:started', ({ type, eluThreshold, maxELU, continuous }) => {
+      // Resolve the overload cutoff: the maxELU profiling option overrides it
+      // (false disables it), otherwise continuous profiling defaults to the
+      // worker health.maxELU.
+      let overloadELU = null
+      if (typeof maxELU === 'number') {
+        overloadELU = maxELU
+      } else if (maxELU !== false && continuous) {
+        const healthConfig = worker[kConfig]?.health
+        const configMaxELU = Number(healthConfig?.maxELU)
+
+        if (healthConfig?.enabled !== false && Number.isFinite(configMaxELU)) {
+          overloadELU = configMaxELU
+        }
+      }
+
+      if (eluThreshold == null && overloadELU == null) {
+        worker[kProfilingELUGates]?.delete(type)
+        return true
+      }
+
+      worker[kProfilingELUGates] ??= new Map()
+      worker[kProfilingELUGates].set(type, {
+        eluThreshold: eluThreshold ?? null,
+        maxELU: overloadELU,
+        // Hysteresis memories: `wanted` tracks the eluThreshold demand,
+        // `overloaded` tracks the maxELU cutoff. `running` is the last state
+        // commanded to the worker: the profiler starts paused when an ELU
+        // threshold is set and running otherwise.
+        wanted: eluThreshold == null,
+        overloaded: false,
+        running: eluThreshold == null
+      })
+      this.#startHealthMetricsCollectionIfNeeded()
+
+      return true
+    })
+
+    worker[kITC].on('profiling:stopped', ({ type }) => {
+      worker[kProfilingELUGates]?.delete(type)
+      this.#lastOverloadProfiles.delete(`${workerId}:${type}`)
+    })
+
+    // When an overload pause is applied, the worker pushes the encoded final
+    // profile. It is preserved here so that the evidence of what saturated
+    // the worker can be retrieved (see getApplicationLastProfile) even while
+    // the worker event loop is blocked, and it survives the worker being
+    // replaced by the health checks.
+    worker[kITC].on('profile:overload', ({ type, timestamp, profile }) => {
+      this.#lastOverloadProfiles.set(`${workerId}:${type}`, { profile, timestamp })
+    })
+
+    // Preserved overload profiles outlive their worker only for a grace
+    // period of twice the runtime graceful shutdown timeout: post-mortem
+    // collectors (alert or health-event driven) have time to fetch the
+    // evidence, but a long-dead worker's profile is not served forever and
+    // entries cannot pile up across replacements (replacement workers get
+    // fresh indices, so their keys are never reused).
+    worker.on('exit', () => {
+      for (const type of ['cpu', 'heap']) {
+        const key = `${workerId}:${type}`
+        const entry = this.#lastOverloadProfiles.get(key)
+
+        if (!entry) {
+          continue
+        }
+
+        const gracefulShutdown = Number(this.#config?.gracefulShutdown?.runtime)
+        const grace = Number.isFinite(gracefulShutdown) && gracefulShutdown > 0 ? gracefulShutdown * 2 : 20_000
+
+        setTimeout(() => {
+          // Only delete the exact entry scheduled here: a successor worker
+          // reusing the index may have preserved a newer profile meanwhile.
+          if (this.#lastOverloadProfiles.get(key) === entry) {
+            this.#lastOverloadProfiles.delete(key)
+          }
+        }, grace).unref()
+      }
+    })
+
     worker[kITC].on('request:restart', async () => {
       // Do not restart applications that are not fully started yet or when the runtime is still starting.
       // The gateway sends request:restart when it receives application:added events,
@@ -2164,15 +2418,24 @@ export class Runtime extends EventEmitter {
   }
 
   #startHealthMetricsCollectionIfNeeded () {
-    // Need health metrics if dynamic workers scaler exists (for vertical scaling)
-    // or if any worker has health checks enabled
-    let needsHealthMetrics = !!this.#dynamicWorkersScaler
+    if (this.#healthMetricsCollectionActive || this.#status !== 'started') {
+      return
+    }
+
+    // Need health metrics if dynamic workers scaler exists (for vertical scaling),
+    // if an extension subscribed to them, if any worker has health checks enabled
+    // or if any worker runs ELU-gated continuous profiling
+    let needsHealthMetrics = !!this.#dynamicWorkersScaler || this.#extensionsWantHealthMetrics
 
     if (!needsHealthMetrics) {
-      // Check if any worker has health checks enabled
       for (const worker of this.#workers.values()) {
         const healthConfig = worker[kConfig]?.health
-        if (healthConfig?.enabled && this.#config.restartOnError > 0) {
+        if (healthConfig?.enabled && this.#getApplicationRestartOnError(this.#config, worker[kConfig]) > 0) {
+          needsHealthMetrics = true
+          break
+        }
+
+        if (worker[kProfilingELUGates]?.size > 0) {
           needsHealthMetrics = true
           break
         }
@@ -2185,8 +2448,11 @@ export class Runtime extends EventEmitter {
   }
 
   #startHealthMetricsCollection () {
+    this.#healthMetricsCollectionActive = true
+
     const collectHealthMetrics = async () => {
       if (this.#status !== 'started') {
+        this.#healthMetricsCollectionActive = false
         return
       }
 
@@ -2202,12 +2468,11 @@ export class Runtime extends EventEmitter {
           const id = worker[kApplicationId]
           const index = worker[kWorkerId]
           const errorLabel = this.#workerExtendedLabel(id, index, worker[kConfig].workers)
+          const previousELU = worker[kLastHealthCheckELU]
 
           let health = null
           try {
-            health = await this.getWorkerHealth(worker, {
-              previousELU: worker[kLastHealthCheckELU]
-            })
+            health = await this.getWorkerHealth(worker, { previousELU })
           } catch (err) {
             this.logger.error({ err }, `Failed to get health for ${errorLabel}.`)
           } finally {
@@ -2226,6 +2491,13 @@ export class Runtime extends EventEmitter {
             currentHealth: health,
             healthSignals
           })
+
+          // Drive the ELU gating of the continuous profiler. The first sample
+          // is skipped as it reports the utilization since the thread started
+          // rather than over the last collection interval.
+          if (health && previousELU != null) {
+            this.#applyProfilingELUGates(worker, health.elu)
+          }
         })())
       }
 
@@ -2240,6 +2512,56 @@ export class Runtime extends EventEmitter {
     this.#healthMetricsTimer = setTimeout(collectHealthMetrics, 1000).unref()
   }
 
+  // Drive the ELU gating of the continuous profiler from the main thread.
+  // The health metrics cycle measures the worker ELU (without depending on the
+  // worker's event loop being responsive, and consistently with health checks)
+  // and resumes/pauses the in-worker profiler accordingly. The profiler runs
+  // while the ELU is above the eluThreshold demand (if one is set) and below
+  // the maxELU overload cutoff (if one is set): crossing the cutoff captures
+  // one last profile and pauses profiling until the worker recovers, so that
+  // profiling does not add overhead to an already overloaded worker.
+  #applyProfilingELUGates (worker, elu) {
+    const gates = worker[kProfilingELUGates]
+
+    if (!gates?.size || typeof elu !== 'number') {
+      return
+    }
+
+    for (const [type, gate] of gates) {
+      // Hysteresis on both bounds to prevent rapid toggling: each state only
+      // flips back once the ELU moves kProfilingELUHysteresis past the bound.
+      if (gate.eluThreshold != null) {
+        gate.wanted = gate.wanted ? elu >= gate.eluThreshold - kProfilingELUHysteresis : elu > gate.eluThreshold
+      }
+
+      if (gate.maxELU != null) {
+        gate.overloaded = gate.overloaded ? elu >= gate.maxELU - kProfilingELUHysteresis : elu > gate.maxELU
+      }
+
+      const shouldRun = gate.wanted && !gate.overloaded
+
+      if (shouldRun === gate.running) {
+        continue
+      }
+
+      gate.running = shouldRun
+
+      try {
+        if (shouldRun) {
+          worker[kITC].notify('resumeProfiling', { type })
+        } else {
+          // The reason matters to the worker: when pausing for overload it
+          // keeps the final profile available for the whole pause, so that
+          // consumers can still retrieve the evidence of what saturated the
+          // worker.
+          worker[kITC].notify('pauseProfiling', { type, reason: gate.overloaded ? 'overload' : 'threshold' })
+        }
+      } catch (err) {
+        this.logger.error({ err }, 'Failed to toggle the continuous profiler')
+      }
+    }
+  }
+
   #setupHealthCheck (config, applicationConfig, workersCount, id, index, worker, errorLabel) {
     let healthMetricsListener = null
 
@@ -2253,12 +2575,19 @@ export class Runtime extends EventEmitter {
 
     const healthConfig = worker[kConfig].health
 
-    let { maxELU, maxHeapUsed, maxHeapTotal, maxUnhealthyChecks, interval } = worker[kConfig].health
+    let { maxELU, maxHeapUsed, maxHeapTotal, maxUnhealthyChecks, interval, maxEventLoopDelay, maxEventLoopDelayP99 } =
+      worker[kConfig].health
 
     if (typeof maxHeapTotal === 'string') {
       maxHeapTotal = parseMemorySize(maxHeapTotal)
     }
 
+    maxEventLoopDelay = Number(maxEventLoopDelay)
+    maxEventLoopDelayP99 = Number(maxEventLoopDelayP99)
+    const eventLoopDelayEnabled = Number.isFinite(maxEventLoopDelay) && maxEventLoopDelay > 0
+    const eventLoopDelayP99Enabled = Number.isFinite(maxEventLoopDelayP99) && maxEventLoopDelayP99 > 0
+    const eventLoopDelayMonitored = eventLoopDelayEnabled || eventLoopDelayP99Enabled
+
     if (interval < 1000) {
       interval = 1000
       this.logger.warn(
@@ -2269,9 +2598,32 @@ export class Runtime extends EventEmitter {
 
     let lastHealthMetrics = null
 
+    // Health metrics arrive every second while the check runs every
+    // `interval`: track the maximum event loop delay (and the worst reported
+    // per-second p99) across the whole check window, so that stalls between
+    // checks are not missed.
+    let maxObservedEventLoopDelay = 0
+    let maxObservedEventLoopDelayP99 = 0
+
     healthMetricsListener = healthCheck => {
       if (healthCheck.id === worker[kId]) {
         lastHealthMetrics = healthCheck
+
+        if (eventLoopDelayMonitored) {
+          for (const signal of healthCheck.healthSignals) {
+            if (signal.type !== 'eventLoopDelay') {
+              continue
+            }
+
+            if (signal.max > maxObservedEventLoopDelay) {
+              maxObservedEventLoopDelay = signal.max
+            }
+
+            if (signal.p99 > maxObservedEventLoopDelayP99) {
+              maxObservedEventLoopDelayP99 = signal.p99
+            }
+          }
+        }
       }
     }
 
@@ -2334,13 +2686,22 @@ export class Runtime extends EventEmitter {
       }
 
       const memoryUsage = health.heapUsed != null ? health.heapUsed / maxHeapTotal : 0
-      const unhealthy = health.elu > maxELU || memoryUsage > maxHeapUsed
+      const eventLoopDelay = maxObservedEventLoopDelay
+      const eventLoopDelayP99 = maxObservedEventLoopDelayP99
+      maxObservedEventLoopDelay = 0
+      maxObservedEventLoopDelayP99 = 0
+      const eventLoopDelayExceeded = eventLoopDelayEnabled && eventLoopDelay > maxEventLoopDelay
+      const eventLoopDelayP99Exceeded = eventLoopDelayP99Enabled && eventLoopDelayP99 > maxEventLoopDelayP99
+      const unhealthy =
+        health.elu > maxELU || memoryUsage > maxHeapUsed || eventLoopDelayExceeded || eventLoopDelayP99Exceeded
 
       this.emit('application:worker:health', {
         id: worker[kId],
         application: id,
         worker: index,
         currentHealth: health,
+        eventLoopDelay: eventLoopDelayMonitored ? eventLoopDelay : undefined,
+        eventLoopDelayP99: eventLoopDelayMonitored ? eventLoopDelayP99 : undefined,
         unhealthy,
         healthConfig
       })
@@ -2359,6 +2720,20 @@ export class Runtime extends EventEmitter {
         )
       }
 
+      if (eventLoopDelayExceeded) {
+        this.logger.error(
+          `The ${errorLabel} had a maximum event loop delay of ${eventLoopDelay.toFixed(2)} ms, ` +
+            `above the maximum allowed delay of ${maxEventLoopDelay} ms.`
+        )
+      }
+
+      if (eventLoopDelayP99Exceeded) {
+        this.logger.error(
+          `The ${errorLabel} had a p99 event loop delay of ${eventLoopDelayP99.toFixed(2)} ms, ` +
+            `above the maximum allowed p99 delay of ${maxEventLoopDelayP99} ms.`
+        )
+      }
+
       if (unhealthy) {
         unhealthyChecks++
       } else {
@@ -2370,7 +2745,16 @@ export class Runtime extends EventEmitter {
           this.emitAndNotify('application:worker:unhealthy', { application: id, worker: index })
 
           this.logger.error(
-            { elu: health.elu, maxELU, memoryUsage: health.heapUsed, maxMemoryUsage: maxHeapUsed },
+            {
+              elu: health.elu,
+              maxELU,
+              memoryUsage: health.heapUsed,
+              maxMemoryUsage: maxHeapUsed,
+              eventLoopDelay,
+              maxEventLoopDelay,
+              eventLoopDelayP99,
+              maxEventLoopDelayP99
+            },
             `The ${errorLabel} is unhealthy. Replacing it ...`
           )
 
@@ -2473,7 +2857,7 @@ export class Runtime extends EventEmitter {
       }
 
       const { enabled, gracePeriod } = worker[kConfig].health
-      if (enabled && config.restartOnError > 0) {
+      if (enabled && this.#getApplicationRestartOnError(config, applicationConfig) > 0) {
         // if gracePeriod is 0, it will be set to 1 to start health checks immediately
         // however, the health event will start when the worker is started
         setTimeout(
@@ -2486,6 +2870,7 @@ export class Runtime extends EventEmitter {
     } catch (err) {
       const error = ensureError(err)
       worker[kITC].notify('application:worker:start:processed')
+      const portOffset = this.#workerPortOffsets.get(worker[kFullId]) ?? index
 
       // TODO: handle port allocation error here
       if (error.code === 'EADDRINUSE' || error.code === 'EACCES') throw error
@@ -2510,7 +2895,7 @@ export class Runtime extends EventEmitter {
         this.logger.error({ err: ensureLoggableError(error) }, `Failed to start ${label}: ${error.message}`)
       }
 
-      const restartOnError = config.restartOnError
+      const restartOnError = this.#getApplicationRestartOnError(config, applicationConfig)
 
       if (disableRestartAttempts || !restartOnError) {
         throw error
@@ -2532,7 +2917,7 @@ export class Runtime extends EventEmitter {
         )
       }
 
-      await this.#restartCrashedWorker(config, applicationConfig, workersCount, id, index, silent, bootstrapAttempt)
+      await this.#restartCrashedWorker(config, applicationConfig, workersCount, id, index, silent, bootstrapAttempt, portOffset)
     }
   }
 
@@ -2601,6 +2986,7 @@ export class Runtime extends EventEmitter {
 
   #cleanupWorker (worker) {
     clearTimeout(worker[kHealthCheckTimer])
+    this.#workerPortOffsets.delete(worker[kFullId])
 
     const currentWorker = this.#workers.get(worker[kFullId])
 
@@ -2629,7 +3015,25 @@ export class Runtime extends EventEmitter {
     return index
   }
 
-  async #restartCrashedWorker (config, applicationConfig, workersCount, id, oldIndex, silent, bootstrapAttempt) {
+  // Returns the effective restartOnError value for an application: the application-level
+  // value, when defined, takes precedence over the runtime-level one.
+  // The value is normalized to a number: false becomes 0 (never restart),
+  // true becomes the default delay.
+  #getApplicationRestartOnError (config, applicationConfig) {
+    let restartOnError = applicationConfig?.restartOnError ?? config.restartOnError
+
+    if (restartOnError === true) {
+      restartOnError = DEFAULT_RESTART_ON_ERROR_DELAY
+    } else if (restartOnError === false || restartOnError < 0) {
+      restartOnError = 0
+    }
+
+    return restartOnError
+  }
+
+  async #restartCrashedWorker (config, applicationConfig, workersCount, id, oldIndex, silent, bootstrapAttempt, portOffset) {
+    const restartOnError = this.#getApplicationRestartOnError(config, applicationConfig)
+
     // Use oldIndex for tracking to prevent duplicate restarts of the same crashed worker
     const restartKey = `${id}:${oldIndex}`
 
@@ -2650,18 +3054,22 @@ export class Runtime extends EventEmitter {
         }
 
         // Get a new unique index for the restarted worker
-        const index = this.#getNextWorkerIndex(id)
+        const newIndex = this.#getNextWorkerIndex(id)
+        const newWorkerId = `${id}:${newIndex}`
+        this.#workerPortOffsets.set(newWorkerId, portOffset)
 
         try {
-          await this.#setupWorker(config, applicationConfig, workersCount, id, index)
-          await this.#startWorker(config, applicationConfig, workersCount, id, index, silent, bootstrapAttempt)
+          await this.#setupWorker(config, applicationConfig, workersCount, id, newIndex)
+          await this.#startWorker(config, applicationConfig, workersCount, id, newIndex, silent, bootstrapAttempt)
           this.#incrementApplicationRestartCount(id)
 
           this.logger.info(
-            `The ${this.#workerExtendedLabel(id, index, workersCount)} has been successfully restarted ...`
+            `The ${this.#workerExtendedLabel(id, newIndex, workersCount)} has been successfully restarted ...`
           )
           resolve()
         } catch (err) {
+          this.#workerPortOffsets.delete(newWorkerId)
+
           // The runtime was stopped while the restart was happening, ignore any error.
           if (!this.#status.startsWith('start')) {
             resolve()
@@ -2671,10 +3079,10 @@ export class Runtime extends EventEmitter {
         }
       }
 
-      if (config.restartOnError < IMMEDIATE_RESTART_MAX_THRESHOLD) {
+      if (restartOnError < IMMEDIATE_RESTART_MAX_THRESHOLD) {
         process.nextTick(restart.bind(this))
       } else {
-        setTimeout(restart.bind(this), config.restartOnError)
+        setTimeout(restart.bind(this), restartOnError)
       }
     })
 
@@ -2690,6 +3098,8 @@ export class Runtime extends EventEmitter {
     const newIndex = this.#getNextWorkerIndex(applicationId)
     const newWorkerId = `${applicationId}:${newIndex}`
     const newLabel = this.#workerExtendedLabel(applicationId, newIndex, workersCount)
+    const portOffset = this.#workerPortOffsets.get(`${applicationId}:${oldIndex}`)
+    this.#workerPortOffsets.set(newWorkerId, portOffset)
 
     const stopBeforeStart =
       applicationConfig.entrypoint &&
@@ -2705,15 +3115,15 @@ export class Runtime extends EventEmitter {
       configForNewWorker = { ...config, server: { ...config.server, port: this.#entrypointPort } }
     }
 
-    if (stopBeforeStart) {
-      await this.#removeWorker(workersCount, applicationId, oldIndex, worker, silent, oldLabel)
-    }
-
     try {
       if (!silent) {
         this.logger.debug(`Preparing to start ${newLabel} as replacement for ${oldLabel} ...`)
       }
 
+      if (stopBeforeStart) {
+        await this.#removeWorker(workersCount, applicationId, oldIndex, worker, silent, oldLabel)
+      }
+
       // Create a new worker with a new index, preserving any pinned port when stopBeforeStart is required.
       newWorker = await this.#setupWorker(configForNewWorker, applicationConfig, workersCount, applicationId, newIndex, false)
 
@@ -2722,6 +3132,11 @@ export class Runtime extends EventEmitter {
         return this.#discardWorker(newWorker)
       }
 
+      // Register the worker before starting it, like in the regular startup flow,
+      // so that it is addressable when the application:worker:started event is emitted.
+      // The discard paths below remove it from the map via #cleanupWorker.
+      this.#workers.set(newWorkerId, newWorker)
+
       // Add the worker to the mesh
       await this.#startWorker(configForNewWorker, applicationConfig, workersCount, applicationId, newIndex, false, 0, newWorker, true)
 
@@ -2729,9 +3144,13 @@ export class Runtime extends EventEmitter {
       if (this.#status !== 'started') {
         return this.#discardWorker(newWorker)
       }
-
-      this.#workers.set(newWorkerId, newWorker)
     } catch (e) {
+      this.#workerPortOffsets.delete(newWorkerId)
+
+      if (this.#workers.get(newWorkerId) === newWorker) {
+        this.#workers.delete(newWorkerId)
+      }
+
       newWorker?.terminate?.()
       throw e
     }
@@ -3286,16 +3705,25 @@ export class Runtime extends EventEmitter {
 
     if (currentWorkers < workers) {
       report.started = []
+      let pendingWorkerId
+
       try {
         for (let i = currentWorkers; i < workers; i++) {
           const newIndex = this.#getNextWorkerIndex(applicationId)
+          pendingWorkerId = `${applicationId}:${newIndex}`
+          this.#workerPortOffsets.set(pendingWorkerId, i)
+
           await this.#setupWorker(config, applicationConfig, workers, applicationId, newIndex)
           await this.#startWorker(config, applicationConfig, workers, applicationId, newIndex, false, 0)
+
+          pendingWorkerId = undefined
           report.started.push(newIndex)
           startedWorkersCount++
         }
         report.success = true
       } catch (err) {
+        pendingWorkerId && this.#workerPortOffsets.delete(pendingWorkerId)
+
         if (startedWorkersCount < 1) {
           this.logger.error({ err }, 'Cannot start application workers, no worker started')
         } else {
@@ -3310,21 +3738,32 @@ export class Runtime extends EventEmitter {
       // keep the current workers count until all the application workers are all stopped
       report.stopped = []
       try {
-        // Get actual worker keys and sort by index descending to stop most recent first (snapshot to avoid mutation during iteration)
-        const workerKeys = [...this.#workers.getKeys(applicationId)]
-        const workerIndices = workerKeys
+        const workersToStop = currentWorkers - workers
+        const allInOnePort = this.#config.server?.portAssignment !== 'perWorkerIncrement'
+        const workerIdsToStop = this.#workers
+          .getKeys(applicationId)
           .map(key => parseInt(key.split(':')[1], 10))
-          .sort((a, b) => b - a) // descending order
 
-        const workersToStop = currentWorkers - workers
-        for (let i = 0; i < workersToStop && i < workerIndices.length; i++) {
-          const workerIndex = workerIndices[i]
+        if (allInOnePort) {
+          // Stop most recent workers first.
+          workerIdsToStop.sort((a, b) => b - a)
+        } else {
+          // When port increment is enabled, disable the workers with the highest ports.
+          workerIdsToStop.sort((a, b) => {
+            const offsetA = this.#workerPortOffsets.get(`${applicationId}:${a}`) ?? a
+            const offsetB = this.#workerPortOffsets.get(`${applicationId}:${b}`) ?? b
+            return offsetB - offsetA
+          })
+        }
+
+        for (const workerIndex of workerIdsToStop.splice(0, workersToStop)) {
           const worker = this.#workers.get(`${applicationId}:${workerIndex}`)
           await sendViaITC(worker, 'removeFromMesh')
           await this.#stopWorker(currentWorkers, applicationId, workerIndex, false, worker, [])
           report.stopped.push(workerIndex)
           stoppedWorkersCount++
         }
+
         report.success = true
       } catch (err) {
         if (stoppedWorkersCount < 1) {
@@ -3419,6 +3858,115 @@ export class Runtime extends EventEmitter {
     worker[kWorkerHealthSignals].add(signals)
   }
 
+  async #loadExtensions () {
+    let extensions = this.#config.extensions
+
+    // Do not load extensions when building applications
+    if (!extensions || this.#context.build) {
+      return
+    }
+
+    if (!Array.isArray(extensions)) {
+      extensions = [extensions]
+    }
+
+    for (const extension of extensions) {
+      const { path, options } = typeof extension === 'string' ? { path: extension } : extension
+
+      let setup
+      try {
+        setup = (await import(pathToFileURL(path))).default
+      } catch (e) {
+        throw new FailedToLoadExtensionError(path, e.message, { cause: e })
+      }
+
+      if (typeof setup !== 'function') {
+        throw new InvalidExtensionError(path)
+      }
+
+      const logger = this.logger.child({ name: `extension:${basename(path)}` })
+
+      try {
+        const instance = await setup({
+          runtime: this,
+          itc: this.#createExtensionITC(),
+          logger,
+          options: options ?? {},
+          root: this.#root
+        })
+
+        this.#extensions.push({ path, instance })
+      } catch (e) {
+        throw new FailedToLoadExtensionError(path, e.message, { cause: e })
+      }
+    }
+
+    // If any extension subscribed to health metrics during its setup, make sure
+    // the health metrics collection is started even if no health check or
+    // dynamic workers scaler is enabled. Note that this is evaluated here on
+    // purpose, before any worker health check listener is registered.
+    this.#extensionsWantHealthMetrics = this.listenerCount('application:worker:health:metrics') > 0
+  }
+
+  async #closeExtensions () {
+    // Close in reverse order, so that extensions loaded later, which might depend
+    // on earlier ones, are closed first.
+    const extensions = this.#extensions.splice(0).reverse()
+
+    for (const { path, instance } of extensions) {
+      try {
+        await instance?.close?.()
+      } catch (e) {
+        this.logger.error({ err: ensureLoggableError(e) }, `Failed to close the extension "${path}".`)
+      }
+    }
+  }
+
+  #createExtensionITC () {
+    return {
+      handle: (name, handler) => {
+        if (this.#reservedITCHandlerNames.has(name)) {
+          throw new ReservedITCHandlerNameError(name)
+        }
+
+        if (name in this.#workerITCHandlers) {
+          throw new DuplicateITCHandlerNameError(name)
+        }
+
+        this.#workerITCHandlers[name] = handler
+
+        // Workers copy the handlers when their ITC is created, so also register
+        // the handler on all running workers.
+        for (const worker of this.#workers.values()) {
+          worker[kITC]?.handle(name, handler)
+        }
+      },
+      send: async (target, name, payload) => {
+        const worker = await this.#getApplicationById(target)
+        return sendViaITC(worker, name, payload)
+      },
+      notify: async (target, name, payload) => {
+        const matched = target.match(/^(.+):(\d+)$/)
+
+        if (matched) {
+          const worker = await this.#getWorkerByIdOrNext(matched[1], matched[2])
+          worker[kITC].notify(name, payload)
+          return
+        }
+
+        if (!this.#applications.has(target)) {
+          throw new ApplicationNotFoundError(target, this.getApplicationsIds().join(', '))
+        }
+
+        for (const worker of this.#workers.values()) {
+          if (worker[kApplicationId] === target && worker[kWorkerStatus] === 'started') {
+            worker[kITC].notify(name, payload)
+          }
+        }
+      }
+    }
+  }
+
   #updateLoggingPrefixes () {
     if (!this.#loggerContext) {
       return