@platformatic/runtime 3.63.0-alpha.3 → 3.64.0

package/lib/runtime.js

@@ -39,12 +39,17 @@ import {
   ApplicationNotStartedError,
   ApplicationStartTimeoutError,
   CannotRemoveEntrypointError,
+  DuplicateExtensionHealthCheckError,
   DuplicateITCHandlerNameError,
+  ExtensionHealthRoutesUnavailableError,
   FailedToLoadExtensionError,
+  FailedToStartExtensionError,
+  FailedToStopExtensionError,
   InvalidArgumentError,
   InvalidExtensionError,
   LastProfileTimeoutError,
   MessagingError,
+  MetricFamilyCollisionError,
   MissingPprofCapture,
   ReservedITCHandlerNameError,
   RuntimeAbortedError,
@@ -112,6 +117,10 @@ const kApplicationRestartsMetricHelp = 'Total number of restarts triggered by th
 
 const MAX_LISTENERS_COUNT = 100
 
+function hasWorkerIndex (applicationId) {
+  return /^.+:\d+$/.test(applicationId)
+}
+
 function parseOrigins (origins) {
   if (!origins) return undefined
 
@@ -190,6 +199,9 @@ export class Runtime extends EventEmitter {
   #reservedITCHandlerNames
   #extensions
   #extensionsWantHealthMetrics
+  #extensionReadinessChecks
+  #extensionLivenessChecks
+  #extensionHealthRoutes
   #lastOverloadProfiles
   #restartingApplications
   #restartingWorkers
@@ -267,6 +279,9 @@ export class Runtime extends EventEmitter {
     this.#reservedITCHandlerNames.add('profiling:started')
     this.#extensions = []
     this.#extensionsWantHealthMetrics = false
+    this.#extensionReadinessChecks = new Map()
+    this.#extensionLivenessChecks = new Map()
+    this.#extensionHealthRoutes = []
     this.#lastOverloadProfiles = new Map()
     this.#sharedContext = {}
 
@@ -298,12 +313,6 @@ export class Runtime extends EventEmitter {
       this.#metricsLabelName = 'applicationId'
     }
 
-    if (config.metrics || (typeof config.healthProbes === 'object' && config.healthProbes !== null)) {
-      this.#prometheusServer = await startPrometheusServer(this, config.metrics ?? false, config.healthProbes)
-    }
-
-    this.#healthProbesServer = await startHealthProbesServer(this, config.metrics, config.healthProbes)
-
     // Initialize process-level metrics registry in the main thread if metrics or management API is enabled
     // These metrics are the same across all workers and only need to be collected once
     // We need this for management API as it can request metrics even without explicit metrics config
@@ -312,7 +321,7 @@ export class Runtime extends EventEmitter {
       collectProcessMetrics(this.#processMetricsRegistry)
     }
 
-    // Create the logger
+    // Create the logger before extensions and health/metrics servers so that both can use it.
     const [logger, destination, context] = await createLogger(config)
     this.logger = logger
     this.#loggerDestination = destination
@@ -333,9 +342,18 @@ export class Runtime extends EventEmitter {
     }
 
     // Load extensions before creating any worker so that custom ITC handlers
-    // registered by the extensions are available to all workers.
+    // registered by the extensions are available to all workers. Also load them
+    // before starting the health/metrics servers so extensions can register
+    // readiness/liveness checks and probe routes before Fastify starts listening.
     await this.#loadExtensions()
 
+    if (config.metrics || (typeof config.healthProbes === 'object' && config.healthProbes !== null)) {
+      this.#prometheusServer = await startPrometheusServer(this, config.metrics ?? false, config.healthProbes)
+    }
+
+    this.#healthProbesServer = await startHealthProbesServer(this, config.metrics, config.healthProbes)
+    this.#assertExtensionHealthRoutesApplied()
+
     await this.addApplications(this.#config.applications)
     await this.#setDispatcher(config.undici)
 
@@ -355,10 +373,16 @@ export class Runtime extends EventEmitter {
     this.#createWorkersBroadcastChannel()
 
     try {
-      const applications = this.getApplicationsIds()
-      await this.startApplications(applications, silent)
+      // Snapshot originally configured application IDs before extension start hooks.
+      // Dynamic applications started by an extension are excluded from the normal startup pass.
+      const configuredApplications = this.getApplicationsIds()
 
-      if (applications.length === 0) {
+      await this.#startExtensions()
+
+      const applicationsToStart = configuredApplications.filter(id => !this.#isApplicationStarted(id))
+      await this.startApplications(applicationsToStart, silent)
+
+      if (this.getApplicationsIds().length === 0) {
         this.#updateStatus('started')
         await this.close(silent)
         return
@@ -419,6 +443,15 @@ export class Runtime extends EventEmitter {
       await once(this, 'started')
     }
 
+    if (this.#status === 'stopping') {
+      await once(this, 'stopped')
+      return
+    }
+
+    if (this.#status === 'stopped' || this.#status === 'closing' || this.#status === 'closed') {
+      return
+    }
+
     this.#updateStatus('stopping')
 
     if (this.#scheduler) {
@@ -436,6 +469,10 @@ export class Runtime extends EventEmitter {
       await this.stopApplication(this.#entrypointId, silent)
     }
 
+    // Await extension stop hooks before stopping remaining applications so that
+    // control-plane extensions can settle work and hand off state first.
+    await this.#stopExtensions()
+
     await this.stopApplications(this.getApplicationsIds(), silent)
 
     await this.#meshInterceptor.close()
@@ -461,6 +498,15 @@ export class Runtime extends EventEmitter {
   }
 
   async close (silent = false) {
+    if (this.#status === 'closing') {
+      await once(this, 'closed')
+      return
+    }
+
+    if (this.#status === 'closed') {
+      return
+    }
+
     clearTimeout(this.#healthMetricsTimer)
     this.#healthMetricsCollectionActive = false
     this.#lastOverloadProfiles.clear()
@@ -652,7 +698,13 @@ export class Runtime extends EventEmitter {
 
     const removed = []
     for (const application of applications) {
-      const details = await this.getApplicationDetails(application)
+      if (!this.#applications.has(application)) {
+        throw new ApplicationNotFoundError(application, this.getApplicationsIds().join(', '))
+      }
+
+      // Use allowUnloaded so that applications without a live worker
+      // (stopped or crashed with restartOnError: 0) can still be removed.
+      const details = await this.getApplicationDetails(application, true)
       details.status = 'removed'
       removed.push(details)
     }
@@ -773,6 +825,7 @@ export class Runtime extends EventEmitter {
     await this.#registerApplicationSchedulerJobs(id)
 
     this.emitAndNotify('application:started', id)
+    await this.#dynamicWorkersScaler?.applyPendingUpdate(id)
   }
 
   async stopApplication (id, silent = false, dependents = []) {
@@ -864,17 +917,68 @@ export class Runtime extends EventEmitter {
   }
 
   async startApplicationProfiling (id, options = {}, ensureStarted = true) {
-    const service = await this.#getApplicationById(id, ensureStarted)
     this.#validatePprofCapturePreload()
 
-    return sendViaITC(service, 'startProfiling', options)
+    const { allWorkers, ...profilingOptions } = options
+
+    if (!allWorkers || hasWorkerIndex(id)) {
+      const service = await this.#getApplicationWorkerForProfiling(id, ensureStarted)
+      return sendViaITC(service, 'startProfiling', profilingOptions)
+    }
+
+    const started = []
+    const alreadyProfiling = []
+    let firstError
+
+    for (const { workerIndex, worker } of await this.#getApplicationWorkersForProfiling(id, ensureStarted)) {
+      try {
+        await sendViaITC(worker, 'startProfiling', profilingOptions)
+        started.push(workerIndex)
+      } catch (error) {
+        // A worker which is already being profiled is considered covered, but
+        // if no other worker could be started the error is still reported.
+        if (error.code === 'PLT_PPROF_PROFILING_ALREADY_STARTED') {
+          alreadyProfiling.push(workerIndex)
+        }
+
+        firstError ??= error
+      }
+    }
+
+    if (started.length === 0 && firstError) {
+      throw firstError
+    }
+
+    return { workers: started.concat(alreadyProfiling).sort((a, b) => a - b) }
   }
 
   async stopApplicationProfiling (id, options = {}, ensureStarted = true) {
-    const service = await this.#getApplicationById(id, ensureStarted)
     this.#validatePprofCapturePreload()
 
-    return sendViaITC(service, 'stopProfiling', options)
+    const { allWorkers, ...profilingOptions } = options
+
+    if (!allWorkers || hasWorkerIndex(id)) {
+      const service = await this.#getApplicationWorkerForProfiling(id, ensureStarted)
+      return sendViaITC(service, 'stopProfiling', profilingOptions)
+    }
+
+    const profiles = []
+    let firstError
+
+    for (const { workerIndex, worker } of await this.#getApplicationWorkersForProfiling(id, ensureStarted)) {
+      try {
+        const profile = await sendViaITC(worker, 'stopProfiling', profilingOptions)
+        profiles.push({ workerIndex, profile })
+      } catch (error) {
+        firstError ??= error
+      }
+    }
+
+    if (profiles.length === 0 && firstError) {
+      throw firstError
+    }
+
+    return profiles
   }
 
   async getApplicationLastProfile (id, options = {}, ensureStarted = true) {
@@ -892,7 +996,11 @@ export class Runtime extends EventEmitter {
       // surface as an unhandled rejection.
       const pull = (async () => {
         const service = await this.#getApplicationById(id, ensureStarted)
-        const result = await sendViaITC(service, 'getLastProfile', { ...options, includeTimestamp: true })
+        const result = await sendViaITC(service, 'getLastProfile', {
+          ...options,
+          includeTimestamp: true,
+          includeSampleCount: true
+        })
         return { service, result }
       })()
       pull.catch(() => {})
@@ -904,7 +1012,9 @@ export class Runtime extends EventEmitter {
 
         // An older capture module which does not support includeTimestamp
         // returns the raw profile.
-        const value = result instanceof Uint8Array ? { profile: result, timestamp: null } : result
+        const value = result instanceof Uint8Array
+          ? { profile: result, timestamp: null, sampleCount: null }
+          : { sampleCount: null, ...result }
 
         // A strictly newer live window supersedes the preserved overload
         // profile: prune it so the preserved copy naturally expires once the
@@ -936,7 +1046,12 @@ export class Runtime extends EventEmitter {
     if (preserved) {
       // The preserved flag lets consumers distinguish post-mortem evidence
       // from a live window and judge it together with the timestamp.
-      return { profile: preserved.profile, timestamp: preserved.timestamp, preserved: true }
+      return {
+        profile: preserved.profile,
+        timestamp: preserved.timestamp,
+        sampleCount: preserved.sampleCount,
+        preserved: true
+      }
     }
 
     throw error
@@ -1053,9 +1168,15 @@ export class Runtime extends EventEmitter {
     this.#config.metrics = metricsConfig
     this.#metricsLabelName = metricsConfig?.applicationLabel || 'applicationId'
 
+    // Allow extension health routes to be re-applied on the restarted servers.
+    for (const entry of this.#extensionHealthRoutes) {
+      entry.applied = false
+    }
+
     this.#prometheusServer = await startPrometheusServer(this, metricsConfig, this.#config.healthProbes)
 
     this.#healthProbesServer = await startHealthProbesServer(this, metricsConfig, this.#config.healthProbes)
+    this.#assertExtensionHealthRoutesApplied()
 
     await this.#startOpenTelemetryMetricsForwarder(metricsConfig?.opentelemetry)
 
@@ -1362,7 +1483,7 @@ export class Runtime extends EventEmitter {
       undefined,
       [],
       this.#concurrency,
-      this.#config.metrics.healthChecksTimeout,
+      this.#getHealthChecksTimeout(),
       {}
     )
   }
@@ -1383,11 +1504,23 @@ export class Runtime extends EventEmitter {
       undefined,
       [],
       this.#concurrency,
-      this.#config.metrics.healthChecksTimeout,
+      this.#getHealthChecksTimeout(),
       {}
     )
   }
 
+  getExtensionHealthRoutes () {
+    return this.#extensionHealthRoutes
+  }
+
+  async runExtensionReadinessChecks () {
+    return this.#runExtensionHealthChecks(this.#extensionReadinessChecks, 'readiness')
+  }
+
+  async runExtensionLivenessChecks () {
+    return this.#runExtensionHealthChecks(this.#extensionLivenessChecks, 'liveness')
+  }
+
   async getMetrics (format = 'json') {
     if (this.#config.metrics === false || this.#config.metrics?.enabled === false) {
       throw new Error('Metrics are disabled')
@@ -1403,6 +1536,15 @@ export class Runtime extends EventEmitter {
       processMetricsJson = await this.#processMetricsRegistry.getMetricsAsJSON()
     }
 
+    // Collect main-thread extension metrics once. Each extension has its own
+    // registry so metric registration and cleanup stay isolated. Collisions
+    // with other extensions, process metrics, restart metrics, or worker
+    // metrics fail with a coded error identifying the extension and family.
+    const extensionMetrics = await this.#getExtensionMetricsJson({
+      processMetricsJson,
+      applicationRestartMetrics
+    })
+
     for (const worker of this.#workers.values()) {
       try {
         // The application might be temporarily unavailable
@@ -1421,20 +1563,6 @@ export class Runtime extends EventEmitter {
         if (applicationMetrics && applicationMetrics !== kTimeout) {
           metrics ??= []
 
-          // Build worker labels including custom labels from metrics config
-          const workerLabels = {
-            ...this.#config.metrics?.labels,
-            [this.#metricsLabelName]: worker[kApplicationId]
-          }
-          const workerId = worker[kWorkerId]
-          if (workerId >= 0) {
-            workerLabels.workerId = workerId
-          }
-
-          // Duplicate process metrics with worker labels and add to output
-          if (processMetricsJson) {
-            this.#applyLabelsToMetrics(processMetricsJson, workerLabels, metrics)
-          }
           // Add worker's thread-specific metrics
           for (let i = 0; i < applicationMetrics.length; i++) {
             metrics.push(applicationMetrics[i])
@@ -1454,6 +1582,49 @@ export class Runtime extends EventEmitter {
       }
     }
 
+    // Extension metrics must not share a family name with any worker metric.
+    if (metrics !== null && extensionMetrics.length > 0) {
+      const workerMetricNames = new Set()
+      for (let i = 0; i < metrics.length; i++) {
+        workerMetricNames.add(metrics[i].name)
+      }
+
+      for (const { path, metricNames } of extensionMetrics) {
+        for (const name of metricNames) {
+          if (workerMetricNames.has(name)) {
+            throw new MetricFamilyCollisionError(path, name, 'application worker metrics')
+          }
+        }
+      }
+    }
+
+    if (extensionMetrics.length > 0) {
+      metrics ??= []
+      const extensionMetricsJson = []
+      for (const { metrics: extensionMetricList } of extensionMetrics) {
+        extensionMetricsJson.push(...extensionMetricList)
+      }
+      metrics = [...extensionMetricsJson, ...metrics]
+    }
+
+    // Report process-level metrics (e.g. process_resident_memory_bytes) only once:
+    // they describe the whole runtime process, so replicating them for each
+    // application running in a worker thread would just duplicate the same value.
+    // Applications running as separate OS processes report their own process-level
+    // metrics, with their own labels, as part of their thread metrics above.
+    // See https://github.com/platformatic/platformatic/issues/3332.
+    if (metrics !== null && processMetricsJson) {
+      const processMetrics = []
+      // Drop any configured custom label that shares the name of the application
+      // label (a config can set both `applicationLabel: 'serviceId'` and a static
+      // `serviceId` label): keeping it would make these runtime-wide metrics look
+      // like they belong to an application, both here and in getFormattedMetrics().
+      const processLabels = { ...this.#config.metrics?.labels }
+      delete processLabels[this.#metricsLabelName]
+      this.#applyLabelsToMetrics(processMetricsJson, processLabels, processMetrics)
+      metrics = [...processMetrics, ...metrics]
+    }
+
     if (metrics !== null && applicationRestartMetrics.length > 0) {
       metrics = [...applicationRestartMetrics, ...metrics]
     }
@@ -1465,6 +1636,57 @@ export class Runtime extends EventEmitter {
     return { metrics }
   }
 
+  async #getExtensionMetricsJson ({ processMetricsJson, applicationRestartMetrics }) {
+    const results = []
+    const metricSources = new Map()
+
+    // Static labels from metrics config apply, but Runtime never invents a
+    // worker ID or application ID for main-thread extension metrics.
+    const extensionLabels = typeof this.#config.metrics === 'object' && this.#config.metrics
+      ? { ...this.#config.metrics.labels }
+      : {}
+    delete extensionLabels[this.#metricsLabelName]
+
+    const processMetricNames = new Set((processMetricsJson ?? []).map(metric => metric.name))
+    const restartMetricNames = new Set((applicationRestartMetrics ?? []).map(metric => metric.name))
+
+    for (const { path, registry } of this.#extensions) {
+      if (!registry) {
+        continue
+      }
+
+      const registryMetrics = await registry.getMetricsAsJSON()
+      if (!registryMetrics || registryMetrics.length === 0) {
+        continue
+      }
+
+      const metricNames = []
+      for (const metric of registryMetrics) {
+        const existing = metricSources.get(metric.name)
+        if (existing) {
+          throw new MetricFamilyCollisionError(path, metric.name, `extension "${existing}"`)
+        }
+
+        if (processMetricNames.has(metric.name)) {
+          throw new MetricFamilyCollisionError(path, metric.name, 'runtime process metrics')
+        }
+
+        if (restartMetricNames.has(metric.name)) {
+          throw new MetricFamilyCollisionError(path, metric.name, 'runtime restart metrics')
+        }
+
+        metricSources.set(metric.name, path)
+        metricNames.push(metric.name)
+      }
+
+      const labeledMetrics = []
+      this.#applyLabelsToMetrics(registryMetrics, extensionLabels, labeledMetrics)
+      results.push({ path, metrics: labeledMetrics, metricNames })
+    }
+
+    return results
+  }
+
   #incrementApplicationRestartCount (applicationId) {
     this.#applicationRestartCounts.set(applicationId, (this.#applicationRestartCounts.get(applicationId) ?? 0) + 1)
   }
@@ -1581,6 +1803,13 @@ export class Runtime extends EventEmitter {
 
       const applicationsMetrics = {}
 
+      // Process-level metrics are reported only once for the whole runtime, without
+      // an application label, since they are shared by all the applications running
+      // in worker threads (see issue #3332). Applications running as separate OS
+      // processes report their own labeled values, which take precedence below.
+      const runtimeProcessMetrics = {}
+      const applicationProcessMetrics = new Set()
+
       for (const metric of metrics) {
         const { name, values } = metric
 
@@ -1592,9 +1821,22 @@ export class Runtime extends EventEmitter {
         const applicationId = labels?.[this.#metricsLabelName]
 
         if (!applicationId) {
+          if (name === 'process_cpu_percent_usage') {
+            runtimeProcessMetrics.cpu = values[0].value
+            continue
+          }
+          if (name === 'process_resident_memory_bytes') {
+            runtimeProcessMetrics.rss = values[0].value
+            continue
+          }
+
           throw new Error(`Missing ${this.#metricsLabelName} label in metrics`)
         }
 
+        if (name === 'process_cpu_percent_usage' || name === 'process_resident_memory_bytes') {
+          applicationProcessMetrics.add(`${applicationId}:${name}`)
+        }
+
         let applicationMetrics = applicationsMetrics[applicationId]
         if (!applicationMetrics) {
           applicationMetrics = {
@@ -1618,6 +1860,24 @@ export class Runtime extends EventEmitter {
         parsePromMetric(applicationMetrics, metric)
       }
 
+      // Apply the runtime-wide process-level values to every application that did
+      // not report its own (i.e. every application running in a worker thread).
+      for (const [applicationId, applicationMetrics] of Object.entries(applicationsMetrics)) {
+        if (
+          runtimeProcessMetrics.cpu !== undefined &&
+          !applicationProcessMetrics.has(`${applicationId}:process_cpu_percent_usage`)
+        ) {
+          applicationMetrics.cpu = runtimeProcessMetrics.cpu
+        }
+
+        if (
+          runtimeProcessMetrics.rss !== undefined &&
+          !applicationProcessMetrics.has(`${applicationId}:process_resident_memory_bytes`)
+        ) {
+          applicationMetrics.rss = runtimeProcessMetrics.rss
+        }
+      }
+
       function parsePromMetric (applicationMetrics, promMetric) {
         const { name } = promMetric
 
@@ -1700,17 +1960,27 @@ export class Runtime extends EventEmitter {
   }
 
   async getApplicationMeta (id) {
-    const application = await this.#getApplicationById(id)
+    const hasWorkerId = /^.+:\d+$/.test(id)
+    const attempts = hasWorkerId ? 1 : Math.max(1, this.#workers.getKeys(id).length)
 
-    try {
-      return await sendViaITC(application, 'getApplicationMeta')
-    } catch (e) {
-      // The application exports no meta, return an empty object
-      if (e.code === 'PLT_ITC_HANDLER_NOT_FOUND') {
-        return {}
-      }
+    for (let attempt = 0; attempt < attempts; attempt++) {
+      const application = await this.#getApplicationById(id)
 
-      throw e
+      try {
+        return await sendViaITC(application, 'getApplicationMeta')
+      } catch (e) {
+        // The application exports no meta, return an empty object
+        if (e.code === 'PLT_ITC_HANDLER_NOT_FOUND') {
+          return {}
+        }
+
+        // A parallel restart can stop the selected worker while metadata is
+        // being retrieved. Retry another worker unless one was requested
+        // explicitly or every worker available at the start has been tried.
+        if (e.code !== 'PLT_RUNTIME_APPLICATION_WORKER_EXIT' || attempt === attempts - 1) {
+          throw e
+        }
+      }
     }
   }
 
@@ -1797,7 +2067,7 @@ export class Runtime extends EventEmitter {
     return sendViaITC(application, 'runApplicationScheduledTasks', { scheduleId, scheduledTime })
   }
 
-  getScheduler () {
+  getSchedulerJobs () {
     return this.#scheduler?.getJobs() ?? []
   }
 
@@ -2299,7 +2569,7 @@ export class Runtime extends EventEmitter {
     // The event only carries metadata: the profile can be retrieved on demand
     // via getApplicationLastProfile. We use emit instead of emitAndNotify since
     // other workers are not interested in this event.
-    worker[kITC].on('profile:captured', ({ type, timestamp }) => {
+    worker[kITC].on('profile:captured', ({ type, timestamp, sampleCount }) => {
       // A strictly newer completed window supersedes the preserved overload
       // profile: once the worker is past the overload and producing windows
       // again, its old evidence must not be served anymore.
@@ -2315,7 +2585,8 @@ export class Runtime extends EventEmitter {
         application: applicationId,
         worker: index,
         type,
-        timestamp
+        timestamp,
+        sampleCount: sampleCount ?? null
       })
     })
 
@@ -2377,8 +2648,12 @@ export class Runtime extends EventEmitter {
     // the worker can be retrieved (see getApplicationLastProfile) even while
     // the worker event loop is blocked, and it survives the worker being
     // replaced by the health checks.
-    worker[kITC].on('profile:overload', ({ type, timestamp, profile }) => {
-      this.#lastOverloadProfiles.set(`${workerId}:${type}`, { profile, timestamp })
+    worker[kITC].on('profile:overload', ({ type, timestamp, profile, sampleCount }) => {
+      this.#lastOverloadProfiles.set(`${workerId}:${type}`, {
+        profile,
+        timestamp,
+        sampleCount: sampleCount ?? null
+      })
     })
 
     // Preserved overload profiles outlive their worker only for a grace
@@ -3033,10 +3308,15 @@ export class Runtime extends EventEmitter {
 
     // Always send the stop message, it will shut down workers that only had ITC and interceptors setup
     try {
-      await executeWithTimeout(sendViaITC(worker, 'stop', { force: !!this.error, dependents }), exitTimeout)
+      const res = await executeWithTimeout(sendViaITC(worker, 'stop', { force: !!this.error, dependents }), exitTimeout)
+
+      if (res === kTimeout) {
+        this.emitAndNotify('application:worker:stop:timeout', eventPayload)
+        this.logger.error(`Timeout while stopping ${label}. Killing a worker thread.`)
+      }
     } catch (error) {
       this.emitAndNotify('application:worker:stop:error', eventPayload)
-      this.logger.info({ error: ensureLoggableError(error) }, `Failed to stop ${label}. Killing a worker thread.`)
+      this.logger.error({ err: ensureLoggableError(error) }, `Failed to stop ${label}. Killing a worker thread.`)
     } finally {
       worker[kITC].notify('application:worker:stop:processed')
       // Wait for the processed message to be received
@@ -3055,6 +3335,7 @@ export class Runtime extends EventEmitter {
     // If the worker didn't exit in time, kill it
     if (res === kTimeout) {
       this.emitAndNotify('application:worker:exit:timeout', eventPayload)
+      this.logger.error(`Timeout while waiting for ${label} to exit. Killing a worker thread.`)
       await worker.terminate()
     }
 
@@ -3279,6 +3560,37 @@ export class Runtime extends EventEmitter {
     return this.#getWorkerByIdOrNext(applicationId, workerId, ensureStarted, mustExist)
   }
 
+  // Profiling start and stop must address the same worker: resolve an id
+  // without an explicit worker index to the first worker deterministically,
+  // as the round-robin used by #getApplicationById would rotate to a
+  // different worker between the two calls.
+  async #getApplicationWorkerForProfiling (applicationId, ensureStarted) {
+    if (hasWorkerIndex(applicationId)) {
+      return this.#getApplicationById(applicationId, ensureStarted)
+    }
+
+    if (!this.#applications.has(applicationId)) {
+      throw new ApplicationNotFoundError(applicationId, this.getApplicationsIds().join(', '))
+    }
+
+    const [firstWorker] = this.#workers.getKeys(applicationId)
+    return this.#getWorkerByIdOrNext(applicationId, firstWorker?.split(':')[1], ensureStarted)
+  }
+
+  async #getApplicationWorkersForProfiling (applicationId, ensureStarted) {
+    if (!this.#applications.has(applicationId)) {
+      throw new ApplicationNotFoundError(applicationId, this.getApplicationsIds().join(', '))
+    }
+
+    const workers = []
+    for (const key of this.#workers.getKeys(applicationId)) {
+      const workerIndex = parseInt(key.split(':')[1], 10)
+      workers.push({ workerIndex, worker: await this.#getWorkerByIdOrNext(applicationId, workerIndex, ensureStarted) })
+    }
+
+    return workers
+  }
+
   // This method can work in two modes: when workerId is provided, it will return the specific worker
   // otherwise it will return the next available worker for the application.
   async #getWorkerByIdOrNext (applicationId, workerId, ensureStarted = false, mustExist = true) {
@@ -3976,18 +4288,30 @@ export class Runtime extends EventEmitter {
       }
 
       const logger = this.logger.child({ name: `extension:${basename(path)}` })
+      const health = this.#createExtensionHealth(path, logger)
+
+      // One registry per extension so metric conflicts and cleanup are isolated.
+      // Extension metrics are main-thread only: Runtime never invents a worker ID.
+      const registry = new metricsClient.Registry()
+      const metrics = { client: metricsClient, registry }
 
       try {
         const instance = await setup({
           runtime: this,
           itc: this.#createExtensionITC(),
+          sharedContext: this.#createExtensionSharedContext(),
           logger,
           options: options ?? {},
-          root: this.#root
+          root: this.#root,
+          metrics,
+          health
         })
 
-        this.#extensions.push({ path, instance })
+        this.#extensions.push({ path, instance, registry, health, started: false, stopped: false, closed: false })
       } catch (e) {
+        registry.clear()
+        // Drop any health contributions from a failed extension setup.
+        health.cleanup()
         throw new FailedToLoadExtensionError(path, e.message, { cause: e })
       }
     }
@@ -3999,18 +4323,301 @@ export class Runtime extends EventEmitter {
     this.#extensionsWantHealthMetrics = this.listenerCount('application:worker:health:metrics') > 0
   }
 
+  #isApplicationStarted (id) {
+    const applicationConfig = this.#applications.get(id)
+    if (!applicationConfig) {
+      return false
+    }
+
+    const workers = applicationConfig.workers.static
+    for (let i = 0; i < workers; i++) {
+      const worker = this.#workers.get(`${id}:${i}`)
+      const status = worker?.[kWorkerStatus]
+
+      // Match startApplication(): anything past boot/init means already started.
+      if (status && status !== 'boot' && status !== 'init') {
+        return true
+      }
+    }
+
+    return false
+  }
+
+  async #startExtensions () {
+    for (const extension of this.#extensions) {
+      if (extension.started) {
+        continue
+      }
+
+      try {
+        await extension.instance?.start?.()
+        extension.started = true
+      } catch (e) {
+        throw new FailedToStartExtensionError(extension.path, e.message, { cause: e })
+      }
+    }
+  }
+
+  async #stopExtensions () {
+    // Stop in reverse order, so that extensions loaded later, which might depend
+    // on earlier ones, are stopped first. Only extensions that completed start
+    // (including those without a start hook) are stopped, and at most once.
+    for (const extension of [...this.#extensions].reverse()) {
+      if (!extension.started || extension.stopped) {
+        continue
+      }
+
+      // Mark before invoking so repeated stop is idempotent even if stop throws.
+      extension.stopped = true
+
+      try {
+        await extension.instance?.stop?.()
+      } catch (e) {
+        const err = new FailedToStopExtensionError(extension.path, e.message, { cause: e })
+        this.logger.error({ err: ensureLoggableError(err) }, `Failed to stop the extension "${extension.path}".`)
+      }
+    }
+  }
+
   async #closeExtensions () {
     // Close in reverse order, so that extensions loaded later, which might depend
-    // on earlier ones, are closed first.
+    // on earlier ones, are closed first. Close is invoked at most once per extension.
     const extensions = this.#extensions.splice(0).reverse()
 
-    for (const { path, instance } of extensions) {
+    for (const extension of extensions) {
+      if (extension.closed) {
+        continue
+      }
+
+      extension.closed = true
+
+      try {
+        await extension.instance?.close?.()
+      } catch (e) {
+        this.logger.error(
+          { err: ensureLoggableError(e) },
+          `Failed to close the extension "${extension.path}".`
+        )
+      } finally {
+        // Always drop health contributions with the extension, even if close fails.
+        extension.health?.cleanup?.()
+      }
+
+      // Drop the extension registry after close so its metrics stop appearing in
+      // getMetrics()/exporters and collectors cannot keep running in the background.
       try {
-        await instance?.close?.()
+        extension.registry?.clear()
       } catch (e) {
-        this.logger.error({ err: ensureLoggableError(e) }, `Failed to close the extension "${path}".`)
+        this.logger.error(
+          { err: ensureLoggableError(e) },
+          `Failed to clear metrics registry for extension "${extension.path}".`
+        )
+      }
+    }
+  }
+
+  #createExtensionSharedContext () {
+    return {
+      // Synchronous on the main thread. Return an isolated snapshot so an
+      // extension cannot mutate the store without going through update(),
+      // which broadcasts changes to workers.
+      get: () => structuredClone(this.getSharedContext()),
+      update: async (update, options = {}) => {
+        // Keep the positional update authoritative, matching the worker API.
+        await this.updateSharedContext({ ...options, context: update })
+      }
+    }
+  }
+
+  #getHealthChecksTimeout () {
+    const metrics = this.#config.metrics
+    if (typeof metrics !== 'object' || metrics === null) {
+      return 5000
+    }
+
+    // Prefer the schema property; keep the historical misspelled key as a fallback.
+    return metrics.healthChecksTimeouts ?? metrics.healthChecksTimeout ?? 5000
+  }
+
+  #assertExtensionHealthRoutesApplied () {
+    const pending = this.#extensionHealthRoutes.filter(entry => entry.active && !entry.applied)
+    if (pending.length > 0) {
+      throw new ExtensionHealthRoutesUnavailableError()
+    }
+  }
+
+  async #runExtensionHealthChecks (checks, kind) {
+    if (checks.size === 0) {
+      return { status: true }
+    }
+
+    const timeout = this.#getHealthChecksTimeout()
+    let response
+
+    for (const [name, entry] of checks) {
+      const result = await this.#runExtensionHealthCheck(name, entry, kind, timeout)
+
+      if (typeof result === 'object' && result !== null) {
+        response = result
+      }
+
+      if (!this.#isExtensionHealthCheckSuccessful(result)) {
+        this.logger.error(
+          { extension: entry.extensionPath, check: name, kind },
+          `Extension ${kind} check "${name}" failed.`
+        )
+        return { status: false, response }
+      }
+    }
+
+    return { status: true, response }
+  }
+
+  async #runExtensionHealthCheck (name, entry, kind, timeout) {
+    try {
+      const result = await executeWithTimeout(
+        Promise.resolve().then(() => entry.check()),
+        timeout,
+        kTimeout
+      )
+
+      if (result === kTimeout) {
+        this.logger.error(
+          { extension: entry.extensionPath, check: name, kind, timeout },
+          `Extension ${kind} check "${name}" timed out.`
+        )
+        return false
+      }
+
+      if (typeof result === 'boolean') {
+        return result
+      }
+
+      if (typeof result === 'object' && result !== null && typeof result.status === 'boolean') {
+        return result
+      }
+
+      this.logger.error(
+        { extension: entry.extensionPath, check: name, kind, result },
+        `Extension ${kind} check "${name}" returned a malformed result.`
+      )
+      return false
+    } catch (err) {
+      this.logger.error(
+        { err: ensureLoggableError(err), extension: entry.extensionPath, check: name, kind },
+        `Extension ${kind} check "${name}" rejected.`
+      )
+      return false
+    }
+  }
+
+  #isExtensionHealthCheckSuccessful (result) {
+    if (typeof result === 'boolean') {
+      return result
+    }
+
+    if (typeof result === 'object' && result !== null) {
+      return !!result.status
+    }
+
+    return false
+  }
+
+  #createExtensionHealth (extensionPath, logger) {
+    const readinessNames = new Set()
+    const livenessNames = new Set()
+    const routeEntries = []
+
+    const registerCheck = (map, names, kind, name, check) => {
+      if (typeof name !== 'string' || name.length === 0) {
+        throw new InvalidArgumentError(`${kind} check name must be a non-empty string`)
+      }
+
+      if (typeof check !== 'function') {
+        throw new InvalidArgumentError(`${kind} check "${name}" must be a function`)
+      }
+
+      const existing = map.get(name)
+      if (existing) {
+        throw new DuplicateExtensionHealthCheckError(kind, name, existing.extensionPath)
+      }
+
+      const entry = { extensionPath, check }
+      map.set(name, entry)
+      names.add(name)
+
+      logger.debug({ check: name, kind }, `Registered extension ${kind} check "${name}"`)
+
+      return () => {
+        const current = map.get(name)
+        if (current === entry) {
+          map.delete(name)
+        }
+        names.delete(name)
+      }
+    }
+
+    const api = {
+      registerReadinessCheck: (name, check) => {
+        return registerCheck(this.#extensionReadinessChecks, readinessNames, 'readiness', name, check)
+      },
+      registerLivenessCheck: (name, check) => {
+        return registerCheck(this.#extensionLivenessChecks, livenessNames, 'liveness', name, check)
+      },
+      registerRoutes: plugin => {
+        if (typeof plugin !== 'function') {
+          throw new InvalidArgumentError('health route plugin must be a function')
+        }
+
+        const entry = {
+          extensionPath,
+          plugin,
+          active: true,
+          applied: false
+        }
+
+        this.#extensionHealthRoutes.push(entry)
+        routeEntries.push(entry)
+
+        logger.debug('Registered extension health routes plugin')
+
+        return () => {
+          entry.active = false
+          const index = this.#extensionHealthRoutes.indexOf(entry)
+          if (index !== -1) {
+            this.#extensionHealthRoutes.splice(index, 1)
+          }
+        }
+      },
+      cleanup: () => {
+        for (const name of readinessNames) {
+          const current = this.#extensionReadinessChecks.get(name)
+          if (current?.extensionPath === extensionPath) {
+            this.#extensionReadinessChecks.delete(name)
+          }
+        }
+        readinessNames.clear()
+
+        for (const name of livenessNames) {
+          const current = this.#extensionLivenessChecks.get(name)
+          if (current?.extensionPath === extensionPath) {
+            this.#extensionLivenessChecks.delete(name)
+          }
+        }
+        livenessNames.clear()
+
+        for (const entry of routeEntries) {
+          entry.active = false
+          const index = this.#extensionHealthRoutes.indexOf(entry)
+          if (index !== -1) {
+            this.#extensionHealthRoutes.splice(index, 1)
+          }
+        }
+        routeEntries.length = 0
       }
     }
+
+    return api
   }
 
   #createExtensionITC () {