@platformatic/runtime 3.63.0-alpha.2 → 3.63.0

package/lib/runtime.js

@@ -39,12 +39,17 @@ import {
   ApplicationNotStartedError,
   ApplicationStartTimeoutError,
   CannotRemoveEntrypointError,
+  DuplicateExtensionHealthCheckError,
   DuplicateITCHandlerNameError,
+  ExtensionHealthRoutesUnavailableError,
   FailedToLoadExtensionError,
+  FailedToStartExtensionError,
+  FailedToStopExtensionError,
   InvalidArgumentError,
   InvalidExtensionError,
   LastProfileTimeoutError,
   MessagingError,
+  MetricFamilyCollisionError,
   MissingPprofCapture,
   ReservedITCHandlerNameError,
   RuntimeAbortedError,
@@ -88,7 +93,6 @@ const kInspectorOptions = Symbol('plt.runtime.worker.inspectorOptions')
 const kHeapCheckCounter = Symbol('plt.runtime.worker.heapCheckCounter')
 const kLastHeapStats = Symbol('plt.runtime.worker.lastHeapStats')
 const kProfilingELUGates = Symbol('plt.runtime.worker.profilingELUGates')
-const kWorkerScheduledTasks = Symbol('plt.runtime.worker.scheduledTasks')
 const kHealthITCTimeoutMs = 5000
 const kProfilingELUHysteresis = 0.1
 const kLastProfileTimeoutMs = 10000
@@ -190,6 +194,9 @@ export class Runtime extends EventEmitter {
   #reservedITCHandlerNames
   #extensions
   #extensionsWantHealthMetrics
+  #extensionReadinessChecks
+  #extensionLivenessChecks
+  #extensionHealthRoutes
   #lastOverloadProfiles
   #restartingApplications
   #restartingWorkers
@@ -267,6 +274,9 @@ export class Runtime extends EventEmitter {
     this.#reservedITCHandlerNames.add('profiling:started')
     this.#extensions = []
     this.#extensionsWantHealthMetrics = false
+    this.#extensionReadinessChecks = new Map()
+    this.#extensionLivenessChecks = new Map()
+    this.#extensionHealthRoutes = []
     this.#lastOverloadProfiles = new Map()
     this.#sharedContext = {}
 
@@ -298,12 +308,6 @@ export class Runtime extends EventEmitter {
       this.#metricsLabelName = 'applicationId'
     }
 
-    if (config.metrics || (typeof config.healthProbes === 'object' && config.healthProbes !== null)) {
-      this.#prometheusServer = await startPrometheusServer(this, config.metrics ?? false, config.healthProbes)
-    }
-
-    this.#healthProbesServer = await startHealthProbesServer(this, config.metrics, config.healthProbes)
-
     // Initialize process-level metrics registry in the main thread if metrics or management API is enabled
     // These metrics are the same across all workers and only need to be collected once
     // We need this for management API as it can request metrics even without explicit metrics config
@@ -312,7 +316,7 @@ export class Runtime extends EventEmitter {
       collectProcessMetrics(this.#processMetricsRegistry)
     }
 
-    // Create the logger
+    // Create the logger before extensions and health/metrics servers so that both can use it.
     const [logger, destination, context] = await createLogger(config)
     this.logger = logger
     this.#loggerDestination = destination
@@ -333,14 +337,23 @@ export class Runtime extends EventEmitter {
     }
 
     // Load extensions before creating any worker so that custom ITC handlers
-    // registered by the extensions are available to all workers.
+    // registered by the extensions are available to all workers. Also load them
+    // before starting the health/metrics servers so extensions can register
+    // readiness/liveness checks and probe routes before Fastify starts listening.
     await this.#loadExtensions()
 
+    if (config.metrics || (typeof config.healthProbes === 'object' && config.healthProbes !== null)) {
+      this.#prometheusServer = await startPrometheusServer(this, config.metrics ?? false, config.healthProbes)
+    }
+
+    this.#healthProbesServer = await startHealthProbesServer(this, config.metrics, config.healthProbes)
+    this.#assertExtensionHealthRoutesApplied()
+
     await this.addApplications(this.#config.applications)
     await this.#setDispatcher(config.undici)
 
-    if (!this.#context.build) {
-      this.#scheduler = startScheduler(config.scheduler ?? [], this.#dispatcher, logger)
+    if (config.scheduler && !this.#context.build) {
+      this.#scheduler = startScheduler(config.scheduler, this.#dispatcher, logger)
     }
 
     this.#updateStatus('init')
@@ -355,10 +368,16 @@ export class Runtime extends EventEmitter {
     this.#createWorkersBroadcastChannel()
 
     try {
-      const applications = this.getApplicationsIds()
-      await this.startApplications(applications, silent)
+      // Snapshot originally configured application IDs before extension start hooks.
+      // Dynamic applications started by an extension are excluded from the normal startup pass.
+      const configuredApplications = this.getApplicationsIds()
+
+      await this.#startExtensions()
 
-      if (applications.length === 0) {
+      const applicationsToStart = configuredApplications.filter(id => !this.#isApplicationStarted(id))
+      await this.startApplications(applicationsToStart, silent)
+
+      if (this.getApplicationsIds().length === 0) {
         this.#updateStatus('started')
         await this.close(silent)
         return
@@ -419,6 +438,15 @@ export class Runtime extends EventEmitter {
       await once(this, 'started')
     }
 
+    if (this.#status === 'stopping') {
+      await once(this, 'stopped')
+      return
+    }
+
+    if (this.#status === 'stopped' || this.#status === 'closing' || this.#status === 'closed') {
+      return
+    }
+
     this.#updateStatus('stopping')
 
     if (this.#scheduler) {
@@ -436,6 +464,10 @@ export class Runtime extends EventEmitter {
       await this.stopApplication(this.#entrypointId, silent)
     }
 
+    // Await extension stop hooks before stopping remaining applications so that
+    // control-plane extensions can settle work and hand off state first.
+    await this.#stopExtensions()
+
     await this.stopApplications(this.getApplicationsIds(), silent)
 
     await this.#meshInterceptor.close()
@@ -461,6 +493,15 @@ export class Runtime extends EventEmitter {
   }
 
   async close (silent = false) {
+    if (this.#status === 'closing') {
+      await once(this, 'closed')
+      return
+    }
+
+    if (this.#status === 'closed') {
+      return
+    }
+
     clearTimeout(this.#healthMetricsTimer)
     this.#healthMetricsCollectionActive = false
     this.#lastOverloadProfiles.clear()
@@ -652,7 +693,13 @@ export class Runtime extends EventEmitter {
 
     const removed = []
     for (const application of applications) {
-      const details = await this.getApplicationDetails(application)
+      if (!this.#applications.has(application)) {
+        throw new ApplicationNotFoundError(application, this.getApplicationsIds().join(', '))
+      }
+
+      // Use allowUnloaded so that applications without a live worker
+      // (stopped or crashed with restartOnError: 0) can still be removed.
+      const details = await this.getApplicationDetails(application, true)
       details.status = 'removed'
       removed.push(details)
     }
@@ -661,7 +708,6 @@ export class Runtime extends EventEmitter {
 
     for (const application of applications) {
       this.#dynamicWorkersScaler?.remove(application)
-      await this.#scheduler?.removeApplicationJobs(application)
       this.#applications.delete(application)
       this.#applicationRestartCounts.delete(application)
     }
@@ -770,9 +816,8 @@ export class Runtime extends EventEmitter {
       await this.#startWorker(config, applicationConfig, workers, id, i, silent)
     }
 
-    await this.#registerApplicationSchedulerJobs(id)
-
     this.emitAndNotify('application:started', id)
+    await this.#dynamicWorkersScaler?.applyPendingUpdate(id)
   }
 
   async stopApplication (id, silent = false, dependents = []) {
@@ -784,7 +829,6 @@ export class Runtime extends EventEmitter {
     const workersCount = workersIds.length
 
     this.emitAndNotify('application:stopping', id)
-    await this.#scheduler?.stopApplicationJobs(id)
 
     if (typeof workersCount === 'number') {
       const stopInvocations = []
@@ -838,7 +882,6 @@ export class Runtime extends EventEmitter {
         await this.#replaceWorker(config, applicationConfig, workersCount, id, workerIndex, worker, true)
       }
 
-      await this.#registerApplicationSchedulerJobs(id)
       this.#incrementApplicationRestartCount(id)
       this.emitAndNotify('application:restarted', id)
     } finally {
@@ -1053,9 +1096,15 @@ export class Runtime extends EventEmitter {
     this.#config.metrics = metricsConfig
     this.#metricsLabelName = metricsConfig?.applicationLabel || 'applicationId'
 
+    // Allow extension health routes to be re-applied on the restarted servers.
+    for (const entry of this.#extensionHealthRoutes) {
+      entry.applied = false
+    }
+
     this.#prometheusServer = await startPrometheusServer(this, metricsConfig, this.#config.healthProbes)
 
     this.#healthProbesServer = await startHealthProbesServer(this, metricsConfig, this.#config.healthProbes)
+    this.#assertExtensionHealthRoutesApplied()
 
     await this.#startOpenTelemetryMetricsForwarder(metricsConfig?.opentelemetry)
 
@@ -1362,7 +1411,7 @@ export class Runtime extends EventEmitter {
       undefined,
       [],
       this.#concurrency,
-      this.#config.metrics.healthChecksTimeout,
+      this.#getHealthChecksTimeout(),
       {}
     )
   }
@@ -1383,11 +1432,23 @@ export class Runtime extends EventEmitter {
       undefined,
       [],
       this.#concurrency,
-      this.#config.metrics.healthChecksTimeout,
+      this.#getHealthChecksTimeout(),
       {}
     )
   }
 
+  getExtensionHealthRoutes () {
+    return this.#extensionHealthRoutes
+  }
+
+  async runExtensionReadinessChecks () {
+    return this.#runExtensionHealthChecks(this.#extensionReadinessChecks, 'readiness')
+  }
+
+  async runExtensionLivenessChecks () {
+    return this.#runExtensionHealthChecks(this.#extensionLivenessChecks, 'liveness')
+  }
+
   async getMetrics (format = 'json') {
     if (this.#config.metrics === false || this.#config.metrics?.enabled === false) {
       throw new Error('Metrics are disabled')
@@ -1403,6 +1464,15 @@ export class Runtime extends EventEmitter {
       processMetricsJson = await this.#processMetricsRegistry.getMetricsAsJSON()
     }
 
+    // Collect main-thread extension metrics once. Each extension has its own
+    // registry so metric registration and cleanup stay isolated. Collisions
+    // with other extensions, process metrics, restart metrics, or worker
+    // metrics fail with a coded error identifying the extension and family.
+    const extensionMetrics = await this.#getExtensionMetricsJson({
+      processMetricsJson,
+      applicationRestartMetrics
+    })
+
     for (const worker of this.#workers.values()) {
       try {
         // The application might be temporarily unavailable
@@ -1421,20 +1491,6 @@ export class Runtime extends EventEmitter {
         if (applicationMetrics && applicationMetrics !== kTimeout) {
           metrics ??= []
 
-          // Build worker labels including custom labels from metrics config
-          const workerLabels = {
-            ...this.#config.metrics?.labels,
-            [this.#metricsLabelName]: worker[kApplicationId]
-          }
-          const workerId = worker[kWorkerId]
-          if (workerId >= 0) {
-            workerLabels.workerId = workerId
-          }
-
-          // Duplicate process metrics with worker labels and add to output
-          if (processMetricsJson) {
-            this.#applyLabelsToMetrics(processMetricsJson, workerLabels, metrics)
-          }
           // Add worker's thread-specific metrics
           for (let i = 0; i < applicationMetrics.length; i++) {
             metrics.push(applicationMetrics[i])
@@ -1454,6 +1510,49 @@ export class Runtime extends EventEmitter {
       }
     }
 
+    // Extension metrics must not share a family name with any worker metric.
+    if (metrics !== null && extensionMetrics.length > 0) {
+      const workerMetricNames = new Set()
+      for (let i = 0; i < metrics.length; i++) {
+        workerMetricNames.add(metrics[i].name)
+      }
+
+      for (const { path, metricNames } of extensionMetrics) {
+        for (const name of metricNames) {
+          if (workerMetricNames.has(name)) {
+            throw new MetricFamilyCollisionError(path, name, 'application worker metrics')
+          }
+        }
+      }
+    }
+
+    if (extensionMetrics.length > 0) {
+      metrics ??= []
+      const extensionMetricsJson = []
+      for (const { metrics: extensionMetricList } of extensionMetrics) {
+        extensionMetricsJson.push(...extensionMetricList)
+      }
+      metrics = [...extensionMetricsJson, ...metrics]
+    }
+
+    // Report process-level metrics (e.g. process_resident_memory_bytes) only once:
+    // they describe the whole runtime process, so replicating them for each
+    // application running in a worker thread would just duplicate the same value.
+    // Applications running as separate OS processes report their own process-level
+    // metrics, with their own labels, as part of their thread metrics above.
+    // See https://github.com/platformatic/platformatic/issues/3332.
+    if (metrics !== null && processMetricsJson) {
+      const processMetrics = []
+      // Drop any configured custom label that shares the name of the application
+      // label (a config can set both `applicationLabel: 'serviceId'` and a static
+      // `serviceId` label): keeping it would make these runtime-wide metrics look
+      // like they belong to an application, both here and in getFormattedMetrics().
+      const processLabels = { ...this.#config.metrics?.labels }
+      delete processLabels[this.#metricsLabelName]
+      this.#applyLabelsToMetrics(processMetricsJson, processLabels, processMetrics)
+      metrics = [...processMetrics, ...metrics]
+    }
+
     if (metrics !== null && applicationRestartMetrics.length > 0) {
       metrics = [...applicationRestartMetrics, ...metrics]
     }
@@ -1465,6 +1564,57 @@ export class Runtime extends EventEmitter {
     return { metrics }
   }
 
+  async #getExtensionMetricsJson ({ processMetricsJson, applicationRestartMetrics }) {
+    const results = []
+    const metricSources = new Map()
+
+    // Static labels from metrics config apply, but Runtime never invents a
+    // worker ID or application ID for main-thread extension metrics.
+    const extensionLabels = typeof this.#config.metrics === 'object' && this.#config.metrics
+      ? { ...this.#config.metrics.labels }
+      : {}
+    delete extensionLabels[this.#metricsLabelName]
+
+    const processMetricNames = new Set((processMetricsJson ?? []).map(metric => metric.name))
+    const restartMetricNames = new Set((applicationRestartMetrics ?? []).map(metric => metric.name))
+
+    for (const { path, registry } of this.#extensions) {
+      if (!registry) {
+        continue
+      }
+
+      const registryMetrics = await registry.getMetricsAsJSON()
+      if (!registryMetrics || registryMetrics.length === 0) {
+        continue
+      }
+
+      const metricNames = []
+      for (const metric of registryMetrics) {
+        const existing = metricSources.get(metric.name)
+        if (existing) {
+          throw new MetricFamilyCollisionError(path, metric.name, `extension "${existing}"`)
+        }
+
+        if (processMetricNames.has(metric.name)) {
+          throw new MetricFamilyCollisionError(path, metric.name, 'runtime process metrics')
+        }
+
+        if (restartMetricNames.has(metric.name)) {
+          throw new MetricFamilyCollisionError(path, metric.name, 'runtime restart metrics')
+        }
+
+        metricSources.set(metric.name, path)
+        metricNames.push(metric.name)
+      }
+
+      const labeledMetrics = []
+      this.#applyLabelsToMetrics(registryMetrics, extensionLabels, labeledMetrics)
+      results.push({ path, metrics: labeledMetrics, metricNames })
+    }
+
+    return results
+  }
+
   #incrementApplicationRestartCount (applicationId) {
     this.#applicationRestartCounts.set(applicationId, (this.#applicationRestartCounts.get(applicationId) ?? 0) + 1)
   }
@@ -1581,6 +1731,13 @@ export class Runtime extends EventEmitter {
 
       const applicationsMetrics = {}
 
+      // Process-level metrics are reported only once for the whole runtime, without
+      // an application label, since they are shared by all the applications running
+      // in worker threads (see issue #3332). Applications running as separate OS
+      // processes report their own labeled values, which take precedence below.
+      const runtimeProcessMetrics = {}
+      const applicationProcessMetrics = new Set()
+
       for (const metric of metrics) {
         const { name, values } = metric
 
@@ -1592,9 +1749,22 @@ export class Runtime extends EventEmitter {
         const applicationId = labels?.[this.#metricsLabelName]
 
         if (!applicationId) {
+          if (name === 'process_cpu_percent_usage') {
+            runtimeProcessMetrics.cpu = values[0].value
+            continue
+          }
+          if (name === 'process_resident_memory_bytes') {
+            runtimeProcessMetrics.rss = values[0].value
+            continue
+          }
+
           throw new Error(`Missing ${this.#metricsLabelName} label in metrics`)
         }
 
+        if (name === 'process_cpu_percent_usage' || name === 'process_resident_memory_bytes') {
+          applicationProcessMetrics.add(`${applicationId}:${name}`)
+        }
+
         let applicationMetrics = applicationsMetrics[applicationId]
         if (!applicationMetrics) {
           applicationMetrics = {
@@ -1618,6 +1788,24 @@ export class Runtime extends EventEmitter {
         parsePromMetric(applicationMetrics, metric)
       }
 
+      // Apply the runtime-wide process-level values to every application that did
+      // not report its own (i.e. every application running in a worker thread).
+      for (const [applicationId, applicationMetrics] of Object.entries(applicationsMetrics)) {
+        if (
+          runtimeProcessMetrics.cpu !== undefined &&
+          !applicationProcessMetrics.has(`${applicationId}:process_cpu_percent_usage`)
+        ) {
+          applicationMetrics.cpu = runtimeProcessMetrics.cpu
+        }
+
+        if (
+          runtimeProcessMetrics.rss !== undefined &&
+          !applicationProcessMetrics.has(`${applicationId}:process_resident_memory_bytes`)
+        ) {
+          applicationMetrics.rss = runtimeProcessMetrics.rss
+        }
+      }
+
       function parsePromMetric (applicationMetrics, promMetric) {
         const { name } = promMetric
 
@@ -1785,79 +1973,6 @@ export class Runtime extends EventEmitter {
     return sendViaITC(application, 'getApplicationGraphQLSchema')
   }
 
-  async getApplicationScheduledTasks (id) {
-    const application = await this.#getApplicationById(id, true)
-
-    return sendViaITC(application, 'getApplicationScheduledTasks')
-  }
-
-  async runApplicationScheduledTasks (id, scheduleId, scheduledTime) {
-    const application = await this.#getApplicationById(id, true)
-
-    return sendViaITC(application, 'runApplicationScheduledTasks', { scheduleId, scheduledTime })
-  }
-
-  getScheduler () {
-    return this.#scheduler?.getJobs() ?? []
-  }
-
-  pauseSchedulerJob (name) {
-    return this.#schedulerOrThrow().pauseJob(name)
-  }
-
-  resumeSchedulerJob (name) {
-    return this.#schedulerOrThrow().resumeJob(name)
-  }
-
-  runSchedulerJob (name) {
-    return this.#schedulerOrThrow().runJob(name)
-  }
-
-  #schedulerOrThrow () {
-    if (!this.#scheduler) {
-      throw new Error('The scheduler is not configured')
-    }
-
-    return this.#scheduler
-  }
-
-  async #registerApplicationSchedulerJobs (id) {
-    if (!this.#scheduler) {
-      return
-    }
-
-    const pausedJobs = new Set(
-      this.#scheduler
-        .getJobs()
-        .filter(job => job.applicationId === id && job.paused)
-        .map(job => job.name)
-    )
-
-    await this.#scheduler.removeApplicationJobs(id)
-
-    const workerId = this.#workers.getKeys(id)[0]
-    const schedules = this.#workers.get(workerId)?.[kWorkerScheduledTasks] ?? []
-    for (const schedule of schedules) {
-      const name = `${id}:${schedule.id}`
-      this.#scheduler.addJob(
-        {
-          name,
-          cron: schedule.cron,
-          source: 'application',
-          applicationId: id,
-          scheduleId: schedule.id,
-          tasks: schedule.tasks,
-          maxRetries: 3
-        },
-        ({ scheduledTime }) => this.runApplicationScheduledTasks(id, schedule.id, scheduledTime)
-      )
-
-      if (pausedJobs.has(name)) {
-        await this.#scheduler.pauseJob(name)
-      }
-    }
-  }
-
   async getWorkers (includeRaw = false) {
     const status = {}
 
@@ -2885,24 +3000,22 @@ export class Runtime extends EventEmitter {
     this.emitAndNotify('application:worker:starting', eventPayload)
 
     try {
-      let workerStartResult
+      let workerUrl
       if (config.startTimeout > 0) {
-        workerStartResult = await executeWithTimeout(sendViaITC(worker, 'start'), config.startTimeout)
+        workerUrl = await executeWithTimeout(sendViaITC(worker, 'start'), config.startTimeout)
 
-        if (workerStartResult === kTimeout) {
+        if (workerUrl === kTimeout) {
           this.emitAndNotify('application:worker:startTimeout', eventPayload)
           this.logger.error(`The ${label} failed to start in ${config.startTimeout}ms. Forcefully killing the thread.`)
           worker.terminate()
           throw new ApplicationStartTimeoutError(id, config.startTimeout)
         }
       } else {
-        workerStartResult = await sendViaITC(worker, 'start')
+        workerUrl = await sendViaITC(worker, 'start')
       }
 
       await this.#avoidOutOfOrderThreadLogs()
 
-      const { url: workerUrl, scheduledTasks } = workerStartResult
-      worker[kWorkerScheduledTasks] = scheduledTasks
       if (workerUrl) {
         this.#url = workerUrl
 
@@ -3033,10 +3146,15 @@ export class Runtime extends EventEmitter {
 
     // Always send the stop message, it will shut down workers that only had ITC and interceptors setup
     try {
-      await executeWithTimeout(sendViaITC(worker, 'stop', { force: !!this.error, dependents }), exitTimeout)
+      const res = await executeWithTimeout(sendViaITC(worker, 'stop', { force: !!this.error, dependents }), exitTimeout)
+
+      if (res === kTimeout) {
+        this.emitAndNotify('application:worker:stop:timeout', eventPayload)
+        this.logger.error(`Timeout while stopping ${label}. Killing a worker thread.`)
+      }
     } catch (error) {
       this.emitAndNotify('application:worker:stop:error', eventPayload)
-      this.logger.info({ error: ensureLoggableError(error) }, `Failed to stop ${label}. Killing a worker thread.`)
+      this.logger.error({ err: ensureLoggableError(error) }, `Failed to stop ${label}. Killing a worker thread.`)
     } finally {
       worker[kITC].notify('application:worker:stop:processed')
       // Wait for the processed message to be received
@@ -3055,6 +3173,7 @@ export class Runtime extends EventEmitter {
     // If the worker didn't exit in time, kill it
     if (res === kTimeout) {
       this.emitAndNotify('application:worker:exit:timeout', eventPayload)
+      this.logger.error(`Timeout while waiting for ${label} to exit. Killing a worker thread.`)
       await worker.terminate()
     }
 
@@ -3219,17 +3338,7 @@ export class Runtime extends EventEmitter {
       this.#workers.set(newWorkerId, newWorker)
 
       // Add the worker to the mesh
-      await this.#startWorker(
-        configForNewWorker,
-        applicationConfig,
-        workersCount,
-        applicationId,
-        newIndex,
-        false,
-        0,
-        newWorker,
-        true
-      )
+      await this.#startWorker(configForNewWorker, applicationConfig, workersCount, applicationId, newIndex, false, 0, newWorker, true)
 
       // Make sure the runtime hasn't been stopped in the meanwhile
       if (this.#status !== 'started') {
@@ -3976,18 +4085,30 @@ export class Runtime extends EventEmitter {
       }
 
       const logger = this.logger.child({ name: `extension:${basename(path)}` })
+      const health = this.#createExtensionHealth(path, logger)
+
+      // One registry per extension so metric conflicts and cleanup are isolated.
+      // Extension metrics are main-thread only: Runtime never invents a worker ID.
+      const registry = new metricsClient.Registry()
+      const metrics = { client: metricsClient, registry }
 
       try {
         const instance = await setup({
           runtime: this,
           itc: this.#createExtensionITC(),
+          sharedContext: this.#createExtensionSharedContext(),
           logger,
           options: options ?? {},
-          root: this.#root
+          root: this.#root,
+          metrics,
+          health
         })
 
-        this.#extensions.push({ path, instance })
+        this.#extensions.push({ path, instance, registry, health, started: false, stopped: false, closed: false })
       } catch (e) {
+        registry.clear()
+        // Drop any health contributions from a failed extension setup.
+        health.cleanup()
         throw new FailedToLoadExtensionError(path, e.message, { cause: e })
       }
     }
@@ -3999,18 +4120,301 @@ export class Runtime extends EventEmitter {
     this.#extensionsWantHealthMetrics = this.listenerCount('application:worker:health:metrics') > 0
   }
 
+  #isApplicationStarted (id) {
+    const applicationConfig = this.#applications.get(id)
+    if (!applicationConfig) {
+      return false
+    }
+
+    const workers = applicationConfig.workers.static
+    for (let i = 0; i < workers; i++) {
+      const worker = this.#workers.get(`${id}:${i}`)
+      const status = worker?.[kWorkerStatus]
+
+      // Match startApplication(): anything past boot/init means already started.
+      if (status && status !== 'boot' && status !== 'init') {
+        return true
+      }
+    }
+
+    return false
+  }
+
+  async #startExtensions () {
+    for (const extension of this.#extensions) {
+      if (extension.started) {
+        continue
+      }
+
+      try {
+        await extension.instance?.start?.()
+        extension.started = true
+      } catch (e) {
+        throw new FailedToStartExtensionError(extension.path, e.message, { cause: e })
+      }
+    }
+  }
+
+  async #stopExtensions () {
+    // Stop in reverse order, so that extensions loaded later, which might depend
+    // on earlier ones, are stopped first. Only extensions that completed start
+    // (including those without a start hook) are stopped, and at most once.
+    for (const extension of [...this.#extensions].reverse()) {
+      if (!extension.started || extension.stopped) {
+        continue
+      }
+
+      // Mark before invoking so repeated stop is idempotent even if stop throws.
+      extension.stopped = true
+
+      try {
+        await extension.instance?.stop?.()
+      } catch (e) {
+        const err = new FailedToStopExtensionError(extension.path, e.message, { cause: e })
+        this.logger.error({ err: ensureLoggableError(err) }, `Failed to stop the extension "${extension.path}".`)
+      }
+    }
+  }
+
   async #closeExtensions () {
     // Close in reverse order, so that extensions loaded later, which might depend
-    // on earlier ones, are closed first.
+    // on earlier ones, are closed first. Close is invoked at most once per extension.
     const extensions = this.#extensions.splice(0).reverse()
 
-    for (const { path, instance } of extensions) {
+    for (const extension of extensions) {
+      if (extension.closed) {
+        continue
+      }
+
+      extension.closed = true
+
       try {
-        await instance?.close?.()
+        await extension.instance?.close?.()
       } catch (e) {
-        this.logger.error({ err: ensureLoggableError(e) }, `Failed to close the extension "${path}".`)
+        this.logger.error(
+          { err: ensureLoggableError(e) },
+          `Failed to close the extension "${extension.path}".`
+        )
+      } finally {
+        // Always drop health contributions with the extension, even if close fails.
+        extension.health?.cleanup?.()
+      }
+
+      // Drop the extension registry after close so its metrics stop appearing in
+      // getMetrics()/exporters and collectors cannot keep running in the background.
+      try {
+        extension.registry?.clear()
+      } catch (e) {
+        this.logger.error(
+          { err: ensureLoggableError(e) },
+          `Failed to clear metrics registry for extension "${extension.path}".`
+        )
+      }
+    }
+  }
+
+  #createExtensionSharedContext () {
+    return {
+      // Synchronous on the main thread. Return an isolated snapshot so an
+      // extension cannot mutate the store without going through update(),
+      // which broadcasts changes to workers.
+      get: () => structuredClone(this.getSharedContext()),
+      update: async (update, options = {}) => {
+        // Keep the positional update authoritative, matching the worker API.
+        await this.updateSharedContext({ ...options, context: update })
+      }
+    }
+  }
+
+  #getHealthChecksTimeout () {
+    const metrics = this.#config.metrics
+    if (typeof metrics !== 'object' || metrics === null) {
+      return 5000
+    }
+
+    // Prefer the schema property; keep the historical misspelled key as a fallback.
+    return metrics.healthChecksTimeouts ?? metrics.healthChecksTimeout ?? 5000
+  }
+
+  #assertExtensionHealthRoutesApplied () {
+    const pending = this.#extensionHealthRoutes.filter(entry => entry.active && !entry.applied)
+    if (pending.length > 0) {
+      throw new ExtensionHealthRoutesUnavailableError()
+    }
+  }
+
+  async #runExtensionHealthChecks (checks, kind) {
+    if (checks.size === 0) {
+      return { status: true }
+    }
+
+    const timeout = this.#getHealthChecksTimeout()
+    let response
+
+    for (const [name, entry] of checks) {
+      const result = await this.#runExtensionHealthCheck(name, entry, kind, timeout)
+
+      if (typeof result === 'object' && result !== null) {
+        response = result
+      }
+
+      if (!this.#isExtensionHealthCheckSuccessful(result)) {
+        this.logger.error(
+          { extension: entry.extensionPath, check: name, kind },
+          `Extension ${kind} check "${name}" failed.`
+        )
+        return { status: false, response }
+      }
+    }
+
+    return { status: true, response }
+  }
+
+  async #runExtensionHealthCheck (name, entry, kind, timeout) {
+    try {
+      const result = await executeWithTimeout(
+        Promise.resolve().then(() => entry.check()),
+        timeout,
+        kTimeout
+      )
+
+      if (result === kTimeout) {
+        this.logger.error(
+          { extension: entry.extensionPath, check: name, kind, timeout },
+          `Extension ${kind} check "${name}" timed out.`
+        )
+        return false
+      }
+
+      if (typeof result === 'boolean') {
+        return result
+      }
+
+      if (typeof result === 'object' && result !== null && typeof result.status === 'boolean') {
+        return result
+      }
+
+      this.logger.error(
+        { extension: entry.extensionPath, check: name, kind, result },
+        `Extension ${kind} check "${name}" returned a malformed result.`
+      )
+      return false
+    } catch (err) {
+      this.logger.error(
+        { err: ensureLoggableError(err), extension: entry.extensionPath, check: name, kind },
+        `Extension ${kind} check "${name}" rejected.`
+      )
+      return false
+    }
+  }
+
+  #isExtensionHealthCheckSuccessful (result) {
+    if (typeof result === 'boolean') {
+      return result
+    }
+
+    if (typeof result === 'object' && result !== null) {
+      return !!result.status
+    }
+
+    return false
+  }
+
+  #createExtensionHealth (extensionPath, logger) {
+    const readinessNames = new Set()
+    const livenessNames = new Set()
+    const routeEntries = []
+
+    const registerCheck = (map, names, kind, name, check) => {
+      if (typeof name !== 'string' || name.length === 0) {
+        throw new InvalidArgumentError(`${kind} check name must be a non-empty string`)
+      }
+
+      if (typeof check !== 'function') {
+        throw new InvalidArgumentError(`${kind} check "${name}" must be a function`)
+      }
+
+      const existing = map.get(name)
+      if (existing) {
+        throw new DuplicateExtensionHealthCheckError(kind, name, existing.extensionPath)
+      }
+
+      const entry = { extensionPath, check }
+      map.set(name, entry)
+      names.add(name)
+
+      logger.debug({ check: name, kind }, `Registered extension ${kind} check "${name}"`)
+
+      return () => {
+        const current = map.get(name)
+        if (current === entry) {
+          map.delete(name)
+        }
+        names.delete(name)
       }
     }
+
+    const api = {
+      registerReadinessCheck: (name, check) => {
+        return registerCheck(this.#extensionReadinessChecks, readinessNames, 'readiness', name, check)
+      },
+      registerLivenessCheck: (name, check) => {
+        return registerCheck(this.#extensionLivenessChecks, livenessNames, 'liveness', name, check)
+      },
+      registerRoutes: plugin => {
+        if (typeof plugin !== 'function') {
+          throw new InvalidArgumentError('health route plugin must be a function')
+        }
+
+        const entry = {
+          extensionPath,
+          plugin,
+          active: true,
+          applied: false
+        }
+
+        this.#extensionHealthRoutes.push(entry)
+        routeEntries.push(entry)
+
+        logger.debug('Registered extension health routes plugin')
+
+        return () => {
+          entry.active = false
+          const index = this.#extensionHealthRoutes.indexOf(entry)
+          if (index !== -1) {
+            this.#extensionHealthRoutes.splice(index, 1)
+          }
+        }
+      },
+      cleanup: () => {
+        for (const name of readinessNames) {
+          const current = this.#extensionReadinessChecks.get(name)
+          if (current?.extensionPath === extensionPath) {
+            this.#extensionReadinessChecks.delete(name)
+          }
+        }
+        readinessNames.clear()
+
+        for (const name of livenessNames) {
+          const current = this.#extensionLivenessChecks.get(name)
+          if (current?.extensionPath === extensionPath) {
+            this.#extensionLivenessChecks.delete(name)
+          }
+        }
+        livenessNames.clear()
+
+        for (const entry of routeEntries) {
+          entry.active = false
+          const index = this.#extensionHealthRoutes.indexOf(entry)
+          if (index !== -1) {
+            this.#extensionHealthRoutes.splice(index, 1)
+          }
+        }
+        routeEntries.length = 0
+      }
+    }
+
+    return api
   }
 
   #createExtensionITC () {