@platformatic/runtime 3.72.0 → 4.0.0-alpha.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/runtime.js CHANGED
@@ -5,7 +5,6 @@ import {
5
5
  executeInParallel,
6
6
  executeWithTimeout,
7
7
  features,
8
- kEnvFileFallbackKeys,
9
8
  kMetadata,
10
9
  kTimeout,
11
10
  parseMemorySize,
@@ -14,11 +13,13 @@ import {
14
13
  import { getExecutable } from '@platformatic/globals'
15
14
  import { ITC } from '@platformatic/itc'
16
15
  import {
16
+ clearRegistry,
17
17
  collectProcessMetrics,
18
18
  client as metricsClient,
19
19
  openTelemetryITCMessage
20
20
  } from '@platformatic/metrics'
21
21
  import fastify from 'fastify'
22
+ import { randomUUID } from 'node:crypto'
22
23
  import { EventEmitter, once } from 'node:events'
23
24
  import { existsSync } from 'node:fs'
24
25
  import { readFile } from 'node:fs/promises'
@@ -30,46 +31,50 @@ import { Readable } from 'node:stream'
30
31
  import { finished } from 'node:stream/promises'
31
32
  import { setImmediate as immediate, setTimeout as sleep } from 'node:timers/promises'
32
33
  import { pathToFileURL } from 'node:url'
34
+ import { isDeepStrictEqual } from 'node:util'
33
35
  import { Worker } from 'node:worker_threads'
34
36
  import SonicBoom from 'sonic-boom'
35
37
  import { Agent, request, interceptors as undiciInterceptors } from 'undici'
36
- import { createThreadInterceptor } from 'undici-thread-interceptor'
38
+ import { createCoordinator, createInterceptor } from 'undici-thread-interceptor'
37
39
  import { pprofCapturePreloadPath } from './config.js'
38
40
  import {
41
+ AddressInUseError,
39
42
  ApplicationAlreadyStartedError,
40
43
  ApplicationNotFoundError,
41
44
  ApplicationNotStartedError,
42
45
  ApplicationStartTimeoutError,
43
- CannotRemoveEntrypointError,
44
46
  DuplicateExtensionHealthCheckError,
45
47
  DuplicateITCHandlerNameError,
46
48
  ExtensionHealthRoutesUnavailableError,
47
49
  FailedToLoadExtensionError,
48
50
  FailedToStartExtensionError,
49
51
  FailedToStopExtensionError,
52
+ FailedToUpdateMetricsConfigError,
50
53
  InvalidArgumentError,
51
54
  InvalidExtensionError,
52
55
  LastProfileTimeoutError,
53
56
  MessagingError,
54
57
  MetricFamilyCollisionError,
58
+ MetricsDisabledError,
55
59
  MissingPprofCapture,
60
+ MixedServingStateError,
56
61
  ReservedITCHandlerNameError,
57
62
  RuntimeAbortedError,
58
63
  RuntimeExtensionBuildAlreadyCalledError,
59
- WorkerInterceptorJoinTimeoutError,
64
+ WorkerAddressInUseError,
60
65
  WorkerNotFoundError
61
66
  } from './errors.js'
62
67
  import { abstractLogger, createLogger } from './logger.js'
63
68
  import { startManagementApi } from './management-api.js'
64
69
  import { createManagementHandlers } from './management-handlers.js'
65
70
  import { OpenTelemetryMetricsForwarder } from './opentelemetry-metrics.js'
66
- import { createChannelCreationHook } from './policies.js'
71
+ import { createChannelCreationHook, createTargetPermissionHook } from './policies.js'
72
+ import { PredictiveWorkersScaler } from './predictive-worker-scaler.js'
67
73
  import { startHealthProbesServer, startPrometheusServer } from './prom-server.js'
68
74
  import { startScheduler } from './scheduler.js'
69
75
  import { createSharedStore } from './shared-http-cache.js'
70
76
  import { topologicalLevels, topologicalSort } from './utils.js'
71
77
  import { version } from './version.js'
72
- import { DynamicWorkersScaler } from './worker-scaler.js'
73
78
  import { HealthSignalsQueue } from './worker/health-signals.js'
74
79
  import { sendMultipleViaITC, sendViaITC, waitEventFromITC } from './worker/itc.js'
75
80
  import { RoundRobinMap } from './worker/round-robin-map.js'
@@ -79,16 +84,21 @@ import {
79
84
  kFullId,
80
85
  kHealthCheckTimer,
81
86
  kId,
82
- kInterceptorReadyPromise,
83
87
  kIsSubprocessHost,
84
88
  kITC,
85
89
  kLastHealthCheckELU,
86
90
  kStderrMarker,
91
+ kSubprocessPid,
92
+ kWorkerExitCode,
87
93
  kWorkerHealthSignals,
88
94
  kWorkerId,
95
+ kWorkerPortOffset,
89
96
  kWorkersBroadcast,
97
+ kWorkerServerOptions,
98
+ kWorkerStartAborted,
90
99
  kWorkerStartTime,
91
- kWorkerStatus
100
+ kWorkerStatus,
101
+ kWorkerUrl
92
102
  } from './worker/symbols.js'
93
103
 
94
104
  const kWorkerFile = join(import.meta.dirname, 'worker/main.js')
@@ -118,6 +128,10 @@ const kLastProfileFallbackCodes = new Set([
118
128
  const kApplicationRestartsMetricName = 'platformatic_application_restarts_total'
119
129
  const kApplicationRestartsMetricHelp = 'Total number of restarts triggered by the runtime for an application.'
120
130
 
131
+ function normalizeCompileCache (value) {
132
+ return typeof value === 'boolean' ? { enabled: value } : value ?? {}
133
+ }
134
+
121
135
  const MAX_LISTENERS_COUNT = 100
122
136
 
123
137
  function hasWorkerIndex (applicationId) {
@@ -174,6 +188,56 @@ const IMMEDIATE_RESTART_MAX_THRESHOLD = 10
174
188
  const MAX_WORKERS = 100
175
189
  const DEFAULT_RESTART_ON_ERROR_DELAY = 5000
176
190
 
191
+ /*
192
+ Both public payloads are built from a snapshot and frozen through. What a consumer could observe
193
+ could once observe was scalars and a file path, so handing out interior state was harmless in
194
+ practice; the payload now nests resolvedConfig -- an entire capability payload -- inside every
195
+ entry, and the getters read
196
+ straight off live state. A consumer mutating what it received would be editing the configuration
197
+ that later restarts and scale-up workers read, silently, and would make worker generations
198
+ disagree about what they are running. setApplicationConfigPatch exists precisely so that changing
199
+ a running application's configuration is explicit, patch-shaped and visible.
200
+
201
+ Only plain objects and arrays are copied. Anything carrying its own prototype -- a class
202
+ instance, a stream, a buffer -- is handed back as it is, because a copy of it would not be the
203
+ thing; the hazard this closes is a consumer editing configuration, not one editing a socket.
204
+ */
205
+ function frozenSnapshot (value, seen = new Map()) {
206
+ if (value === null || typeof value !== 'object') {
207
+ return value
208
+ }
209
+
210
+ if (seen.has(value)) {
211
+ return seen.get(value)
212
+ }
213
+
214
+ if (Array.isArray(value)) {
215
+ const copy = []
216
+ seen.set(value, copy)
217
+
218
+ for (const entry of value) {
219
+ copy.push(frozenSnapshot(entry, seen))
220
+ }
221
+
222
+ return Object.freeze(copy)
223
+ }
224
+
225
+ const prototype = Object.getPrototypeOf(value)
226
+
227
+ if (prototype !== Object.prototype && prototype !== null) {
228
+ return value
229
+ }
230
+
231
+ const copy = {}
232
+ seen.set(value, copy)
233
+
234
+ for (const [key, entry] of Object.entries(value)) {
235
+ copy[key] = frozenSnapshot(entry, seen)
236
+ }
237
+
238
+ return Object.freeze(copy)
239
+ }
240
+
177
241
  export class Runtime extends EventEmitter {
178
242
  logger
179
243
  error
@@ -194,19 +258,22 @@ export class Runtime extends EventEmitter {
194
258
  #sharedContext
195
259
  #isProduction
196
260
  #concurrency
197
- #entrypointId
198
- #url
199
- #entrypointPort
200
261
 
201
262
  #healthMetricsTimer
202
263
  #healthMetricsCollectionActive
203
264
 
204
265
  #meshInterceptor
266
+ #meshCoordinator
267
+ #meshId
205
268
  #dispatcher
206
269
 
207
270
  #managementApi
208
271
  #prometheusServer
209
272
  #healthProbesServer
273
+ #metricsServersInitialized
274
+ #metricsServersStartPromise
275
+ #metricsConfigUpdatePromise
276
+ #metricsConfigUpdateFailed
210
277
  #opentelemetryMetricsForwarder
211
278
  #inspectorServer
212
279
  #metricsLabelName
@@ -224,11 +291,11 @@ export class Runtime extends EventEmitter {
224
291
  #extensionLivenessChecks
225
292
  #extensionHealthRoutes
226
293
  #lastOverloadProfiles
294
+ #servingStates
227
295
  #restartingApplications
228
- #restartingWorkers
296
+ #replicas
229
297
  #dynamicWorkersScaler
230
298
  #nextWorkerIndex
231
- #workerPortOffsets
232
299
 
233
300
  #sharedHttpCache
234
301
  #scheduler
@@ -255,22 +322,13 @@ export class Runtime extends EventEmitter {
255
322
  this.#applications = new Map()
256
323
  this.#applicationRestartCounts = new Map()
257
324
  this.#workers = new RoundRobinMap()
258
- this.#url = undefined
259
- this.#entrypointPort = undefined
260
325
  this.#channelCreationHook = createChannelCreationHook(this.#config)
261
- this.#meshInterceptor = createThreadInterceptor({
262
- domain: '.plt.local',
263
- timeout: this.#config.applicationTimeout,
264
- meshTimeout: this.#context.meshTimeout ?? true,
265
- onChannelCreation: this.#channelCreationHook,
266
- onError: this.#onMeshInterceptorError.bind(this)
267
- })
326
+ this.#meshId = `runtime-${randomUUID()}`
268
327
  this.logger = abstractLogger // This is replaced by the real logger in init() and eventually removed in close()
269
328
  this.#status = undefined
270
329
  this.#restartingApplications = new Set()
271
- this.#restartingWorkers = new Map()
330
+ this.#replicas = new Map()
272
331
  this.#nextWorkerIndex = new Map()
273
- this.#workerPortOffsets = new Map() // fullWorkerId => portOffset
274
332
  this.#sharedHttpCache = null
275
333
  this.#applicationsConfigsPatches = new Map()
276
334
 
@@ -299,20 +357,29 @@ export class Runtime extends EventEmitter {
299
357
  // Registered per-worker in #setupWorker, reserved so extensions cannot clobber it
300
358
  this.#reservedITCHandlerNames.add('profiling:started')
301
359
  this.#extensions = []
360
+ this.#metricsServersInitialized = false
361
+ this.#metricsServersStartPromise = null
362
+ this.#metricsConfigUpdatePromise = Promise.resolve()
363
+ this.#metricsConfigUpdateFailed = false
364
+ this.#processMetricsRegistry = null
302
365
  this.#extensionsWantHealthMetrics = false
303
366
  this.#extensionReadinessChecks = new Map()
304
367
  this.#extensionLivenessChecks = new Map()
305
368
  this.#extensionHealthRoutes = []
306
369
  this.#lastOverloadProfiles = new Map()
370
+ this.#servingStates = new Map()
307
371
  this.#sharedContext = {}
308
372
 
309
- if (this.#isProduction) {
310
- this.#env.PLT_DEV = 'false'
311
- this.#env.PLT_ENVIRONMENT = 'production'
312
- } else {
313
- this.#env.PLT_DEV = 'true'
314
- this.#env.PLT_ENVIRONMENT = 'development'
315
- }
373
+ /*
374
+ `PLT_DEV` and `PLT_ENVIRONMENT` are no longer injected here: an application branches on
375
+ its own variables, or the decision moves into the configuration, where the context carries
376
+ `production` and `mode` with types. `NODE_ENV` is the one the runtime still defaults, at the
377
+ bottom of the env ladder, so anything the project sets outranks it.
378
+
379
+ They were worse than redundant by the end -- the worker environment is the one the loader
380
+ resolved per application, and these were written onto the runtime's own copy, so under
381
+ `wattpm start` an application still read `PLT_ENVIRONMENT=development`.
382
+ */
316
383
  }
317
384
 
318
385
  async init () {
@@ -330,10 +397,9 @@ export class Runtime extends EventEmitter {
330
397
  this.#metricsLabelName = 'applicationId'
331
398
  }
332
399
 
333
- // Initialize process-level metrics registry in the main thread if metrics or management API is enabled
400
+ // Initialize process-level metrics registry only when metrics are enabled.
334
401
  // These metrics are the same across all workers and only need to be collected once
335
- // We need this for management API as it can request metrics even without explicit metrics config
336
- if (config.metrics || config.managementApi) {
402
+ if (config.metrics && config.metrics.enabled !== false) {
337
403
  this.#processMetricsRegistry = new metricsClient.Registry()
338
404
  collectProcessMetrics(this.#processMetricsRegistry)
339
405
  }
@@ -349,18 +415,14 @@ export class Runtime extends EventEmitter {
349
415
  this.#managementApi = await startManagementApi(this, config.managementApi)
350
416
  }
351
417
 
352
- await this.#startOpenTelemetryMetricsForwarder(config.metrics?.opentelemetry)
418
+ if (config.metrics && config.metrics.enabled !== false) {
419
+ await this.#startOpenTelemetryMetricsForwarder(config.metrics.opentelemetry)
420
+ }
353
421
 
354
422
  this.#createWorkersBroadcastChannel()
355
423
 
356
424
  if (this.#config.workers.dynamic) {
357
- if (this.#config.workers.dynamic === false) {
358
- this.logger.warn(
359
- `Worker scaler disabled because the "workers" configuration is set to ${this.#config.workers.static}.`
360
- )
361
- } else {
362
- this.#dynamicWorkersScaler = new DynamicWorkersScaler(this, this.#config.workers)
363
- }
425
+ this.#dynamicWorkersScaler = new PredictiveWorkersScaler(this, this.#config.workers)
364
426
  }
365
427
 
366
428
  // Load extensions before creating any worker so that custom ITC handlers
@@ -369,18 +431,28 @@ export class Runtime extends EventEmitter {
369
431
  // readiness/liveness checks and probe routes before Fastify starts listening.
370
432
  await this.#loadExtensions()
371
433
 
372
- if (config.metrics || (typeof config.healthProbes === 'object' && config.healthProbes !== null)) {
373
- this.#prometheusServer = await startPrometheusServer(this, config.metrics ?? false, config.healthProbes)
434
+ // A runtime without applications exits during start and must not briefly
435
+ // claim the default health-probe port. Applications added later, including
436
+ // by extension start hooks, initialize these servers in addApplications().
437
+ if (config.applications.length > 0) {
438
+ await this.#ensureMetricsServersStarted()
374
439
  }
375
440
 
376
- this.#healthProbesServer = await startHealthProbesServer(this, config.metrics, config.healthProbes)
377
- this.#assertExtensionHealthRoutesApplied()
441
+ this.#meshCoordinator = createCoordinator({ meshId: this.#meshId })
442
+ this.#meshInterceptor = createInterceptor({
443
+ meshId: this.#meshId,
444
+ domain: '.plt.local',
445
+ connectTimeout: this.#config.applicationTimeout,
446
+ bootstrapTimeout: this.#config.applicationTimeout,
447
+ allowTarget: createTargetPermissionHook(this.#config)
448
+ })
449
+ await this.#meshInterceptor.ready
378
450
 
379
451
  await this.addApplications(this.#config.applications)
380
452
  await this.#setDispatcher(config.undici)
381
453
 
382
454
  if (!this.#context.build) {
383
- this.#scheduler = startScheduler(config.scheduler ?? [], this.#dispatcher, logger)
455
+ this.#scheduler = startScheduler(logger)
384
456
  }
385
457
 
386
458
  this.#updateStatus('init')
@@ -407,7 +479,7 @@ export class Runtime extends EventEmitter {
407
479
  if (this.getApplicationsIds().length === 0) {
408
480
  this.#updateStatus('started')
409
481
  await this.close(silent)
410
- return
482
+ return {}
411
483
  }
412
484
 
413
485
  if (this.#config.inspectorOptions) {
@@ -450,19 +522,13 @@ export class Runtime extends EventEmitter {
450
522
 
451
523
  this.#updateStatus('started')
452
524
 
453
- // The CLI enables the module compile cache for this process. Node.js would only write it when
454
- // the process terminates, so flush it now that the boot is complete to not lose it when the
455
- // process is killed abruptly. This is a no-op when the compile cache is not enabled.
456
- scheduleCompileCacheFlush()
457
-
458
525
  // Start the global health metrics timer for all workers if needed
459
526
  this.#startHealthMetricsCollectionIfNeeded()
460
527
 
461
528
  await this.#dynamicWorkersScaler?.start()
462
- if (this.#url) {
463
- this.#showUrl()
464
- }
465
- return this.#url
529
+ this.#showUrls()
530
+ scheduleCompileCacheFlush(this.logger)
531
+ return this.getUrls()
466
532
  }
467
533
 
468
534
  async stop (silent = false) {
@@ -491,18 +557,14 @@ export class Runtime extends EventEmitter {
491
557
 
492
558
  await this.#dynamicWorkersScaler?.stop()
493
559
 
494
- // Stop the entrypoint first so that no new requests are accepted
495
- if (this.#entrypointId) {
496
- await this.stopApplication(this.#entrypointId, silent)
497
- }
498
-
499
560
  // Await extension stop hooks before stopping remaining applications so that
500
561
  // control-plane extensions can settle work and hand off state first.
501
562
  await this.#stopExtensions()
502
563
 
503
564
  await this.stopApplications(this.getApplicationsIds(), silent)
504
565
 
505
- await this.#meshInterceptor.close()
566
+ await this.#meshInterceptor?.close()
567
+ this.#meshCoordinator?.destroy()
506
568
  this.#workersBroadcastChannel?.close()
507
569
 
508
570
  this.#updateStatus('stopped')
@@ -520,8 +582,6 @@ export class Runtime extends EventEmitter {
520
582
  await this.restartApplications(toRestart)
521
583
 
522
584
  this.emitAndNotify('restarted')
523
-
524
- return this.#url
525
585
  }
526
586
 
527
587
  async close (silent = false) {
@@ -537,6 +597,7 @@ export class Runtime extends EventEmitter {
537
597
  clearTimeout(this.#healthMetricsTimer)
538
598
  this.#healthMetricsCollectionActive = false
539
599
  this.#lastOverloadProfiles.clear()
600
+ this.#servingStates.clear()
540
601
 
541
602
  await this.stop(silent)
542
603
  this.#updateStatus('closing')
@@ -561,7 +622,7 @@ export class Runtime extends EventEmitter {
561
622
 
562
623
  // Clean up process metrics registry
563
624
  if (this.#processMetricsRegistry) {
564
- this.#processMetricsRegistry.clear()
625
+ clearRegistry(this.#processMetricsRegistry)
565
626
  this.#processMetricsRegistry = null
566
627
  }
567
628
 
@@ -673,25 +734,8 @@ export class Runtime extends EventEmitter {
673
734
 
674
735
  async addApplications (applications, start = false) {
675
736
  const setupInvocations = []
676
- // Per-worker ports do not need SO_REUSEPORT because each worker binds a different port.
677
- const usesPerWorkerPorts = this.#config.server?.portAssignment === 'perWorkerIncrement'
678
-
679
737
  const toStart = []
680
738
  for (const application of applications) {
681
- const workers = application.workers
682
-
683
- if (
684
- (workers.static > 1 || workers.minimum > 1) &&
685
- application.entrypoint &&
686
- !features.node.reusePort &&
687
- !usesPerWorkerPorts
688
- ) {
689
- this.logger.warn(
690
- `"${application.id}" is set as the entrypoint, but reusePort is not available in your OS; setting workers to 1 instead of ${workers.static}`
691
- )
692
- application.workers = { dynamic: false, static: 1 }
693
- }
694
-
695
739
  this.#applications.set(application.id, application)
696
740
  this.#applicationRestartCounts.set(application.id, this.#applicationRestartCounts.get(application.id) ?? 0)
697
741
  setupInvocations.push([application])
@@ -700,8 +744,12 @@ export class Runtime extends EventEmitter {
700
744
 
701
745
  await executeInParallel(this.#setupApplication.bind(this), setupInvocations, this.#concurrency)
702
746
 
747
+ if (applications.length > 0) {
748
+ await this.#ensureMetricsServersStarted()
749
+ }
750
+
703
751
  for (const application of applications) {
704
- this.logger.debug(`Added application "${application.id}"${application.entrypoint ? ' (entrypoint)' : ''}.`)
752
+ this.logger.debug(`Added application "${application.id}".`)
705
753
  this.emitAndNotify('application:added', application)
706
754
  }
707
755
 
@@ -711,7 +759,7 @@ export class Runtime extends EventEmitter {
711
759
 
712
760
  const created = []
713
761
  for (const { id } of applications) {
714
- created.push(await this.getApplicationDetails(id))
762
+ created.push(await this.#buildApplicationDetails(id))
715
763
  }
716
764
 
717
765
  this.#updateLoggingPrefixes()
@@ -719,10 +767,6 @@ export class Runtime extends EventEmitter {
719
767
  }
720
768
 
721
769
  async removeApplications (applications, silent = false) {
722
- if (applications.includes(this.#entrypointId)) {
723
- throw new CannotRemoveEntrypointError()
724
- }
725
-
726
770
  const removed = []
727
771
  for (const application of applications) {
728
772
  if (!this.#applications.has(application)) {
@@ -731,18 +775,23 @@ export class Runtime extends EventEmitter {
731
775
 
732
776
  // Use allowUnloaded so that applications without a live worker
733
777
  // (stopped or crashed with restartOnError: 0) can still be removed.
734
- const details = await this.getApplicationDetails(application, true)
778
+ const details = await this.#buildApplicationDetails(application, true)
735
779
  details.status = 'removed'
780
+
781
+ // The snapshot is taken while the application is still up, but what it reports is an
782
+ // application about to be gone: it is not making a claim about how it would serve.
783
+ delete details.servingState
784
+
736
785
  removed.push(details)
737
786
  }
738
787
 
739
788
  await this.stopApplications(applications, silent, true)
740
789
 
741
790
  for (const application of applications) {
742
- this.#dynamicWorkersScaler?.remove(application)
743
791
  await this.#scheduler?.removeApplicationJobs(application)
744
792
  this.#applications.delete(application)
745
793
  this.#applicationRestartCounts.delete(application)
794
+ this.#replicas.delete(application)
746
795
  }
747
796
 
748
797
  for (const application of applications) {
@@ -773,8 +822,24 @@ export class Runtime extends EventEmitter {
773
822
  const levels = topologicalLevels(applications, dependencies)
774
823
 
775
824
  for (const level of levels) {
776
- const startInvocations = level.map(app => [app, silent])
777
- await executeInParallel(this.startApplication.bind(this), startInvocations, this.#concurrency)
825
+ const applicationsWithPort = await Promise.all(
826
+ level.map(async applicationId => {
827
+ const worker = await this.#getWorkerByIdOrNext(applicationId)
828
+ const applicationConfig = await sendViaITC(worker, 'getApplicationConfig')
829
+ const port = Number(applicationConfig?.server?.port)
830
+ return { applicationId, hasPort: Number.isInteger(port) && port > 0 }
831
+ })
832
+ )
833
+
834
+ for (const hasPort of [true, false]) {
835
+ const startInvocations = applicationsWithPort
836
+ .filter(application => application.hasPort === hasPort)
837
+ .map(({ applicationId }) => [applicationId, silent])
838
+
839
+ if (startInvocations.length > 0) {
840
+ await executeInParallel(this.startApplication.bind(this), startInvocations, this.#concurrency)
841
+ }
842
+ }
778
843
  }
779
844
  }
780
845
 
@@ -786,16 +851,21 @@ export class Runtime extends EventEmitter {
786
851
 
787
852
  if (!skipDependencies) {
788
853
  try {
789
- const { applications } = await this.getApplications(true)
790
- for (const application of applications) {
791
- for (const dependency of application.dependencies ?? []) {
792
- let applicationDependents = dependents[dependency]
793
- if (!applicationDependents) {
794
- applicationDependents = new Set()
795
- dependents[dependency] = applicationDependents
796
- }
854
+ const details = await executeWithTimeout(
855
+ this.getApplications(true),
856
+ this.#config.gracefulShutdown.application
857
+ )
858
+ if (details !== kTimeout) {
859
+ for (const application of details.applications) {
860
+ for (const dependency of application.dependencies ?? []) {
861
+ let applicationDependents = dependents[dependency]
862
+ if (!applicationDependents) {
863
+ applicationDependents = new Set()
864
+ dependents[dependency] = applicationDependents
865
+ }
797
866
 
798
- applicationDependents.add(application.id)
867
+ applicationDependents.add(application.id)
868
+ }
799
869
  }
800
870
  }
801
871
  } catch (e) {
@@ -804,11 +874,6 @@ export class Runtime extends EventEmitter {
804
874
  }
805
875
 
806
876
  for (const application of applicationsToStop) {
807
- // The entrypoint has been stopped above
808
- if (application === this.#entrypointId) {
809
- continue
810
- }
811
-
812
877
  stopInvocations.push([application, silent, Array.from(dependents[application] ?? [])])
813
878
  }
814
879
 
@@ -816,13 +881,39 @@ export class Runtime extends EventEmitter {
816
881
  }
817
882
 
818
883
  async restartApplications (applicationsToRestart) {
819
- const restartInvocations = []
884
+ const restartInvocations = applicationsToRestart.map(application => [application, true])
885
+ const restarts = await executeInParallel(
886
+ this.restartApplication.bind(this),
887
+ restartInvocations,
888
+ this.#concurrency,
889
+ false
890
+ )
891
+ const failed = restarts.filter(result => result instanceof Error)
892
+ const succeeded = restarts.filter(result => result && !(result instanceof Error))
893
+
894
+ if (failed.length > 0) {
895
+ await Promise.allSettled(succeeded.map(restart => restart.discard()))
896
+ throw failed[0]
897
+ }
820
898
 
821
- for (const application of applicationsToRestart) {
822
- restartInvocations.push([application])
899
+ const results = await executeInParallel(
900
+ retirement => retirement.retire(),
901
+ succeeded.flatMap(({ retirements }) => retirements.map(retirement => [retirement])),
902
+ this.#concurrency,
903
+ false
904
+ )
905
+
906
+ const error = results.find(result => result instanceof Error)
907
+ if (error) {
908
+ for (const restart of succeeded) {
909
+ restart.release()
910
+ }
911
+ throw error
823
912
  }
824
913
 
825
- return executeInParallel(this.restartApplication.bind(this), restartInvocations, this.#concurrency)
914
+ for (const restart of succeeded) {
915
+ await restart.complete()
916
+ }
826
917
  }
827
918
 
828
919
  async startApplication (id, silent = false) {
@@ -833,26 +924,33 @@ export class Runtime extends EventEmitter {
833
924
  throw new ApplicationNotFoundError(id, this.getApplicationsIds().join(', '))
834
925
  }
835
926
 
927
+ if (this.#isApplicationStarted(id)) {
928
+ throw new ApplicationAlreadyStartedError()
929
+ }
930
+
836
931
  const workers = applicationConfig.workers.static
837
- for (let i = 0; i < workers; i++) {
838
- const worker = this.#workers.get(`${id}:${i}`)
839
- const status = worker?.[kWorkerStatus]
932
+ this.emitAndNotify('application:starting', id)
840
933
 
841
- if (status && status !== 'boot' && status !== 'init') {
842
- throw new ApplicationAlreadyStartedError()
843
- }
934
+ const compileCache = {
935
+ ...normalizeCompileCache(config.compileCache),
936
+ ...normalizeCompileCache(applicationConfig.compileCache)
844
937
  }
938
+ const awaitFirstWorker =
939
+ typeof compileCache === 'object' && compileCache.enabled !== false && compileCache.awaitFirstWorker === true
845
940
 
846
- this.emitAndNotify('application:starting', id)
941
+ if (awaitFirstWorker && workers > 0) {
942
+ await this.#startFirstWorkerWithCompileCache(config, applicationConfig, workers, id, silent)
943
+ }
847
944
 
848
- for (let i = 0; i < workers; i++) {
945
+ // Keep the first worker separate so compile-cache startup can be made a barrier without changing this loop later.
946
+ for (let i = awaitFirstWorker ? 1 : 0; i < workers; i++) {
849
947
  await this.#startWorker(config, applicationConfig, workers, id, i, silent)
850
948
  }
851
949
 
852
950
  await this.#registerApplicationSchedulerJobs(id)
951
+ await this.#collectServingState(id)
853
952
 
854
953
  this.emitAndNotify('application:started', id)
855
- await this.#dynamicWorkersScaler?.applyPendingUpdate(id)
856
954
  }
857
955
 
858
956
  async stopApplication (id, silent = false, dependents = []) {
@@ -860,7 +958,17 @@ export class Runtime extends EventEmitter {
860
958
  throw new ApplicationNotFoundError(id, this.getApplicationsIds().join(', '))
861
959
  }
862
960
 
961
+ this.#dynamicWorkersScaler?.remove(id)
962
+
963
+ // Removing the replicas cancels the work in progress for them, i.e. refills of crashed workers and replacements
964
+ const replicas = this.#replicas.get(id) ?? []
965
+ this.#replicas.set(id, [])
966
+ const detached = Promise.all(replicas.map(replica => this.#detachReplica(id, replica)))
967
+
968
+ // Workers being discarded by the detach above are already marked as exited
863
969
  const workersIds = this.#workers.getKeys(id)
970
+ .filter(workerId => this.#workers.get(workerId)[kWorkerStatus] !== 'exited')
971
+
864
972
  const workersCount = workersIds.length
865
973
 
866
974
  this.emitAndNotify('application:stopping', id)
@@ -876,10 +984,17 @@ export class Runtime extends EventEmitter {
876
984
  await executeInParallel(this.#stopWorker.bind(this), stopInvocations, this.#concurrency)
877
985
  }
878
986
 
987
+ await detached
988
+
989
+ // Absent rather than 'inactive': a stopped application is not making a claim about how it
990
+ // would serve, and conflating "not started" with "started and serving nothing" is the exact
991
+ // distinction this field exists to draw.
992
+ this.#servingStates.delete(id)
993
+
879
994
  this.emitAndNotify('application:stopped', id)
880
995
  }
881
996
 
882
- async restartApplication (id) {
997
+ async restartApplication (id, deferOldWorkerRetirement = false) {
883
998
  const applicationConfig = this.#applications.get(id)
884
999
 
885
1000
  if (!applicationConfig) {
@@ -898,31 +1013,75 @@ export class Runtime extends EventEmitter {
898
1013
  }
899
1014
 
900
1015
  this.#restartingApplications.add(id)
1016
+ const retirements = []
1017
+ let deferred = false
901
1018
 
902
1019
  try {
903
1020
  const config = this.#config
904
- const workersIds = await this.#workers.getKeys(id)
905
- const workersCount = workersIds.length
1021
+ // A copy: while the loop awaits, scale-downs remove replicas from the list, which would shift the remaining ones
1022
+ // and skip some. Replicas added in the meanwhile (scale-up) already run fresh workers.
1023
+ const replicas = [...(this.#replicas.get(id) ?? [])]
1024
+ const replicasCount = replicas.length
906
1025
 
907
1026
  this.emitAndNotify('application:restarting', id)
908
1027
 
909
- for (let i = 0; i < workersCount; i++) {
910
- const workerId = workersIds[i]
911
- const worker = this.#workers.get(workerId)
912
- const workerIndex = parseInt(workerId.split(':')[1], 10)
1028
+ for (let i = 0; i < replicasCount; i++) {
1029
+ const replica = replicas[i]
913
1030
 
914
1031
  if (i > 0 && config.workersRestartDelay > 0) {
915
1032
  await sleep(config.workersRestartDelay)
916
1033
  }
917
1034
 
918
- await this.#replaceWorker(config, applicationConfig, workersCount, id, workerIndex, worker, true)
1035
+ // The replica might have been removed while the previous ones were being replaced,
1036
+ // or its worker crashed and the replica is being refilled with a fresh worker.
1037
+ const worker = replica.worker
1038
+ if (replica.abortController.signal.aborted || !worker) {
1039
+ continue
1040
+ }
1041
+
1042
+ const retirement = await this.#replaceWorker(
1043
+ config,
1044
+ applicationConfig,
1045
+ replicasCount,
1046
+ id,
1047
+ worker[kWorkerId],
1048
+ worker,
1049
+ true,
1050
+ deferOldWorkerRetirement
1051
+ )
1052
+ if (retirement) {
1053
+ retirements.push(retirement)
1054
+ }
1055
+ }
1056
+
1057
+ if (deferOldWorkerRetirement) {
1058
+ deferred = true
1059
+ return {
1060
+ retirements,
1061
+ release: () => this.#restartingApplications.delete(id),
1062
+ discard: async () => {
1063
+ await Promise.allSettled(retirements.map(retirement => retirement.discard()))
1064
+ this.#restartingApplications.delete(id)
1065
+ },
1066
+ complete: async () => {
1067
+ try {
1068
+ await this.#registerApplicationSchedulerJobs(id)
1069
+ this.#incrementApplicationRestartCount(id)
1070
+ this.emitAndNotify('application:restarted', id)
1071
+ } finally {
1072
+ this.#restartingApplications.delete(id)
1073
+ }
1074
+ }
1075
+ }
919
1076
  }
920
1077
 
921
1078
  await this.#registerApplicationSchedulerJobs(id)
922
1079
  this.#incrementApplicationRestartCount(id)
923
1080
  this.emitAndNotify('application:restarted', id)
924
1081
  } finally {
925
- this.#restartingApplications.delete(id)
1082
+ if (!deferred) {
1083
+ this.#restartingApplications.delete(id)
1084
+ }
926
1085
  }
927
1086
  }
928
1087
 
@@ -994,13 +1153,7 @@ export class Runtime extends EventEmitter {
994
1153
 
995
1154
  if (!allWorkers || hasWorkerIndex(id)) {
996
1155
  const service = await this.#getApplicationWorkerForProfiling(id, ensureStarted)
997
- const nodeModulesSourceMaps = profilingOptions.nodeModulesSourceMaps ??
998
- service[kConfig].nodeModulesSourceMaps ?? this.#config.nodeModulesSourceMaps
999
-
1000
- return sendViaITC(service, 'startProfiling', {
1001
- ...profilingOptions,
1002
- nodeModulesSourceMaps
1003
- })
1156
+ return sendViaITC(service, 'startProfiling', profilingOptions)
1004
1157
  }
1005
1158
 
1006
1159
  const started = []
@@ -1009,14 +1162,7 @@ export class Runtime extends EventEmitter {
1009
1162
 
1010
1163
  for (const { workerIndex, worker } of await this.#getApplicationWorkersForProfiling(id, ensureStarted)) {
1011
1164
  try {
1012
- const nodeModulesSourceMaps = profilingOptions.nodeModulesSourceMaps ??
1013
- worker[kConfig].nodeModulesSourceMaps ??
1014
- this.#config.nodeModulesSourceMaps
1015
-
1016
- await sendViaITC(worker, 'startProfiling', {
1017
- ...profilingOptions,
1018
- nodeModulesSourceMaps
1019
- })
1165
+ await sendViaITC(worker, 'startProfiling', profilingOptions)
1020
1166
  started.push(workerIndex)
1021
1167
  } catch (error) {
1022
1168
  // A worker which is already being profiled is considered covered, but
@@ -1271,7 +1417,8 @@ export class Runtime extends EventEmitter {
1271
1417
  }
1272
1418
 
1273
1419
  /**
1274
- * Updates the metrics configuration at runtime without restarting the runtime or workers.
1420
+ * Merges top-level metrics options without restarting the runtime or workers.
1421
+ * Nested objects are replaced when supplied. Identical successful updates are no-ops.
1275
1422
  *
1276
1423
  * This method allows you to:
1277
1424
  * - Enable or disable metrics collection
@@ -1290,52 +1437,138 @@ export class Runtime extends EventEmitter {
1290
1437
  * await runtime.updateMetricsConfig({ enabled: false })
1291
1438
  */
1292
1439
  async updateMetricsConfig (metricsConfig) {
1293
- if (this.#prometheusServer) {
1294
- await this.#prometheusServer.close()
1295
- this.#prometheusServer = null
1296
- }
1440
+ // Serialize changes because they share listeners and worker registries.
1441
+ const previousUpdate = this.#metricsConfigUpdatePromise
1442
+ const { promise, resolve } = Promise.withResolvers()
1443
+ this.#metricsConfigUpdatePromise = promise
1444
+ await previousUpdate
1297
1445
 
1298
- if (this.#healthProbesServer) {
1299
- await this.#healthProbesServer.close()
1300
- this.#healthProbesServer = null
1301
- }
1446
+ try {
1447
+ const currentConfig = this.#config.metrics
1448
+ const previousConfig = typeof currentConfig === 'boolean' ? { enabled: currentConfig } : (currentConfig ?? {})
1449
+ metricsConfig = { ...previousConfig, ...metricsConfig }
1302
1450
 
1303
- if (this.#opentelemetryMetricsForwarder) {
1304
- await this.#opentelemetryMetricsForwarder.close()
1305
- this.#opentelemetryMetricsForwarder = null
1306
- }
1451
+ // An omitted enabled option already means enabled for an object configuration.
1452
+ const previousOptions = { ...previousConfig, enabled: previousConfig.enabled !== false }
1453
+ const nextOptions = { ...metricsConfig, enabled: metricsConfig.enabled !== false }
1454
+ if (!this.#metricsConfigUpdateFailed && isDeepStrictEqual(previousOptions, nextOptions)) {
1455
+ this.#config.metrics = metricsConfig
1456
+ return { success: true, config: structuredClone(metricsConfig) }
1457
+ }
1307
1458
 
1308
- this.#config.metrics = metricsConfig
1309
- this.#metricsLabelName = metricsConfig?.applicationLabel || 'applicationId'
1459
+ // Failed updates can be retried even when the main-thread configuration already changed.
1460
+ this.#metricsConfigUpdateFailed = true
1310
1461
 
1311
- // Allow extension health routes to be re-applied on the restarted servers.
1312
- for (const entry of this.#extensionHealthRoutes) {
1313
- entry.applied = false
1314
- }
1462
+ if (this.#metricsServersStartPromise) {
1463
+ await this.#metricsServersStartPromise
1464
+ }
1315
1465
 
1316
- this.#prometheusServer = await startPrometheusServer(this, metricsConfig, this.#config.healthProbes)
1466
+ if (this.#prometheusServer) {
1467
+ await this.#prometheusServer.close()
1468
+ this.#prometheusServer = null
1469
+ }
1317
1470
 
1318
- this.#healthProbesServer = await startHealthProbesServer(this, metricsConfig, this.#config.healthProbes)
1319
- this.#assertExtensionHealthRoutesApplied()
1471
+ if (this.#healthProbesServer) {
1472
+ await this.#healthProbesServer.close()
1473
+ this.#healthProbesServer = null
1474
+ }
1320
1475
 
1321
- await this.#startOpenTelemetryMetricsForwarder(metricsConfig?.opentelemetry)
1476
+ this.#metricsServersInitialized = false
1322
1477
 
1323
- const promises = []
1324
- for (const worker of this.#workers.values()) {
1325
- if (worker[kWorkerStatus] === 'started') {
1326
- promises.push(sendViaITC(worker, 'updateMetricsConfig', metricsConfig))
1478
+ if (this.#opentelemetryMetricsForwarder) {
1479
+ await this.#opentelemetryMetricsForwarder.close()
1480
+ this.#opentelemetryMetricsForwarder = null
1327
1481
  }
1328
- }
1329
1482
 
1330
- const results = await Promise.allSettled(promises)
1331
- for (const result of results) {
1332
- if (result.status === 'rejected') {
1333
- this.logger.error({ err: result.reason }, 'Cannot update metrics config on worker')
1483
+ this.#config.metrics = metricsConfig
1484
+ this.#metricsLabelName = metricsConfig?.applicationLabel || 'applicationId'
1485
+
1486
+ if (metricsConfig.enabled === false) {
1487
+ if (this.#processMetricsRegistry) {
1488
+ clearRegistry(this.#processMetricsRegistry)
1489
+ this.#processMetricsRegistry = null
1490
+ }
1491
+ } else if (!this.#processMetricsRegistry) {
1492
+ this.#processMetricsRegistry = new metricsClient.Registry()
1493
+ collectProcessMetrics(this.#processMetricsRegistry)
1334
1494
  }
1495
+
1496
+ // Allow extension health routes to be re-applied on the restarted servers.
1497
+ for (const entry of this.#extensionHealthRoutes) {
1498
+ entry.applied = false
1499
+ }
1500
+
1501
+ await this.#ensureMetricsServersStarted()
1502
+
1503
+ if (metricsConfig.enabled !== false) {
1504
+ await this.#startOpenTelemetryMetricsForwarder(metricsConfig.opentelemetry)
1505
+ }
1506
+
1507
+ const promises = []
1508
+ for (const worker of this.#workers.values()) {
1509
+ if (worker[kWorkerStatus] === 'started') {
1510
+ promises.push(
1511
+ sendViaITC(worker, 'updateMetricsConfig', metricsConfig).catch(err => {
1512
+ throw new FailedToUpdateMetricsConfigError(worker[kFullId], err.message)
1513
+ })
1514
+ )
1515
+ }
1516
+ }
1517
+
1518
+ const results = await Promise.allSettled(promises)
1519
+ for (const result of results) {
1520
+ if (result.status === 'rejected') {
1521
+ this.logger.error({ err: result.reason }, 'Cannot update metrics config on worker')
1522
+ }
1523
+ }
1524
+
1525
+ const failure = results.find(result => result.status === 'rejected')
1526
+ if (failure) {
1527
+ throw failure.reason
1528
+ }
1529
+
1530
+ this.#metricsConfigUpdateFailed = false
1531
+ this.logger.info('Metrics configuration updated')
1532
+ return { success: true, config: structuredClone(metricsConfig) }
1533
+ } finally {
1534
+ // Failed and no-op updates must also release the next queued request.
1535
+ resolve()
1536
+ }
1537
+ }
1538
+
1539
+ async #ensureMetricsServersStarted () {
1540
+ if (this.#metricsServersInitialized) {
1541
+ return
1335
1542
  }
1336
1543
 
1337
- this.logger.info({ metricsConfig }, 'Metrics configuration updated')
1338
- return { success: true, config: metricsConfig }
1544
+ if (this.#metricsServersStartPromise) {
1545
+ return this.#metricsServersStartPromise
1546
+ }
1547
+
1548
+ this.#metricsServersStartPromise = (async () => {
1549
+ const config = this.#config
1550
+
1551
+ try {
1552
+ if (config.metrics || (typeof config.healthProbes === 'object' && config.healthProbes !== null)) {
1553
+ this.#prometheusServer = await startPrometheusServer(this, config.metrics ?? false, config.healthProbes)
1554
+ }
1555
+
1556
+ this.#healthProbesServer = await startHealthProbesServer(this, config.metrics, config.healthProbes)
1557
+ this.#assertExtensionHealthRoutesApplied()
1558
+ this.#metricsServersInitialized = true
1559
+ } catch (err) {
1560
+ await Promise.allSettled([this.#prometheusServer?.close(), this.#healthProbesServer?.close()])
1561
+ this.#prometheusServer = null
1562
+ this.#healthProbesServer = null
1563
+ throw err
1564
+ }
1565
+ })()
1566
+
1567
+ try {
1568
+ await this.#metricsServersStartPromise
1569
+ } finally {
1570
+ this.#metricsServersStartPromise = null
1571
+ }
1339
1572
  }
1340
1573
 
1341
1574
  async #startOpenTelemetryMetricsForwarder (config) {
@@ -1490,7 +1723,7 @@ export class Runtime extends EventEmitter {
1490
1723
  const { applicationId, config: applicationConfig, workers, health, currentWorkers, currentHealth } = update
1491
1724
 
1492
1725
  if (workers && health) {
1493
- const r = await this.#updateApplicationWorkersAndHealth(
1726
+ const resourceReport = await this.#updateApplicationWorkersAndHealth(
1494
1727
  applicationId,
1495
1728
  config,
1496
1729
  applicationConfig,
@@ -1501,11 +1734,11 @@ export class Runtime extends EventEmitter {
1501
1734
  )
1502
1735
  report.push({
1503
1736
  application: applicationId,
1504
- workers: r.workers,
1505
- health: r.health
1737
+ workers: resourceReport.workers,
1738
+ health: resourceReport.health
1506
1739
  })
1507
1740
  } else if (health) {
1508
- const r = await this.#updateApplicationHealth(
1741
+ const healthReport = await this.#updateApplicationHealth(
1509
1742
  applicationId,
1510
1743
  config,
1511
1744
  applicationConfig,
@@ -1515,10 +1748,10 @@ export class Runtime extends EventEmitter {
1515
1748
  )
1516
1749
  report.push({
1517
1750
  application: applicationId,
1518
- health: r.health
1751
+ health: healthReport
1519
1752
  })
1520
1753
  } else if (workers) {
1521
- const r = await this.#updateApplicationWorkers(
1754
+ const workersReport = await this.#updateApplicationWorkers(
1522
1755
  applicationId,
1523
1756
  config,
1524
1757
  applicationConfig,
@@ -1527,7 +1760,7 @@ export class Runtime extends EventEmitter {
1527
1760
  )
1528
1761
  report.push({
1529
1762
  application: applicationId,
1530
- workers: r.workers
1763
+ workers: workersReport
1531
1764
  })
1532
1765
  }
1533
1766
  }
@@ -1543,8 +1776,20 @@ export class Runtime extends EventEmitter {
1543
1776
  return this.#root
1544
1777
  }
1545
1778
 
1546
- getUrl () {
1547
- return this.#url
1779
+ getUrls (applicationId) {
1780
+ const urls = {}
1781
+ const applicationIds = applicationId ? [applicationId] : this.#applications.keys()
1782
+
1783
+ for (const id of applicationIds) {
1784
+ for (const workerId of this.#workers.getKeys(id)) {
1785
+ const url = this.#workers.get(workerId)?.[kWorkerUrl]
1786
+ if (url) {
1787
+ urls[workerId] = url
1788
+ }
1789
+ }
1790
+ }
1791
+
1792
+ return urls
1548
1793
  }
1549
1794
 
1550
1795
  getRuntimeStatus () {
@@ -1553,8 +1798,6 @@ export class Runtime extends EventEmitter {
1553
1798
 
1554
1799
  async getRuntimeMetadata () {
1555
1800
  const packageJson = await this.#getRuntimePackageJson()
1556
- const entrypointDetails = await this.getEntrypointDetails()
1557
-
1558
1801
  return {
1559
1802
  pid: process.pid,
1560
1803
  cwd: process.cwd(),
@@ -1563,10 +1806,18 @@ export class Runtime extends EventEmitter {
1563
1806
  execPath: process.execPath,
1564
1807
  nodeVersion: process.version,
1565
1808
  projectDir: this.#root,
1809
+ /*
1810
+ What `applications:add`/`remove --save` actually consume, so they can stop reading the whole
1811
+ runtime configuration over HTTP to get at three values. `autoload` is the declaration as
1812
+ authored, not the expansion: it is expanded in the eval worker, and --save has to edit what
1813
+ the file says rather than what it produced.
1814
+ */
1815
+ configPath: this.#config[kMetadata]?.path ?? null,
1816
+ autoload: this.#config.autoload ?? null,
1566
1817
  packageName: packageJson.name ?? null,
1567
1818
  packageVersion: packageJson.version ?? null,
1568
- url: entrypointDetails?.url ?? null,
1569
- platformaticVersion: version
1819
+ platformaticVersion: version,
1820
+ urls: this.getUrls()
1570
1821
  }
1571
1822
  }
1572
1823
 
@@ -1574,13 +1825,31 @@ export class Runtime extends EventEmitter {
1574
1825
  return this.#env
1575
1826
  }
1576
1827
 
1828
+ /*
1829
+ What a watcher has to follow to know this configuration changed. The whole set the evaluation
1830
+ read is reported; resolving per worker had no such set, so it was the deciding file alone --
1831
+ which is what dev watched before either way.
1832
+ */
1833
+ getConfigurationWatchTargets () {
1834
+ const metadata = this.#config[kMetadata]
1835
+ const targets = metadata?.loader?.watchTargets
1836
+
1837
+ if (targets) {
1838
+ return { files: [...targets.files], directories: [...targets.directories] }
1839
+ }
1840
+
1841
+ return { files: metadata?.path ? [metadata.path] : [], directories: [] }
1842
+ }
1843
+
1577
1844
  getRuntimeConfig (includeMeta = false) {
1845
+ // includeMeta is an internal contract and hands back live state, symbol key and all. It leaves
1846
+ // the public surface with the DTO change; until then, its in-tree callers depend on identity.
1578
1847
  if (includeMeta) {
1579
1848
  return this.#config
1580
1849
  }
1581
1850
 
1582
1851
  const { [kMetadata]: _, ...config } = this.#config
1583
- return config
1852
+ return frozenSnapshot(config)
1584
1853
  }
1585
1854
 
1586
1855
  getInterceptor () {
@@ -1599,12 +1868,39 @@ export class Runtime extends EventEmitter {
1599
1868
  return this.#managementApi?.server.address() ?? null
1600
1869
  }
1601
1870
 
1602
- async getEntrypointDetails () {
1603
- if (!this.#entrypointId) {
1604
- return null
1871
+ /*
1872
+ servingState is computed per worker: for a worker-classified capability it depends on what the
1873
+ application's factory returned in *that* worker, and nothing stops arbitrary code from returning
1874
+ a server from worker 0 and a background result from worker 1. Sampling one worker -- which is
1875
+ what getApplicationDetails does for every other field -- would make the reported value depend on
1876
+ which worker the selector picked, and, worse, would leave mesh dispatch routing a share of
1877
+ requests to a worker that destroys them. So every worker answers and a mixed answer is refused,
1878
+ naming each worker and the state it reported.
1879
+ */
1880
+ async #collectServingState (id) {
1881
+ const invocations = this.#workers.getKeys(id).map(workerId => [workerId, this.#workers.get(workerId)])
1882
+
1883
+ if (invocations.length === 0) {
1884
+ this.#servingStates.delete(id)
1885
+ return
1605
1886
  }
1606
1887
 
1607
- return this.getApplicationDetails(this.#entrypointId)
1888
+ const states = await sendMultipleViaITC(invocations, 'getServingState', undefined, [], this.#concurrency)
1889
+ const reported = Object.entries(states).filter(([, state]) => typeof state === 'string')
1890
+
1891
+ if (reported.length === 0) {
1892
+ this.#servingStates.delete(id)
1893
+ return
1894
+ }
1895
+
1896
+ const distinct = new Set(reported.map(([, state]) => state))
1897
+
1898
+ if (distinct.size > 1) {
1899
+ this.#servingStates.delete(id)
1900
+ throw new MixedServingStateError(id, reported.map(([worker, state]) => `${worker} reported ${state}`).join(', '))
1901
+ }
1902
+
1903
+ this.#servingStates.set(id, reported[0][1])
1608
1904
  }
1609
1905
 
1610
1906
  async getCustomHealthChecks () {
@@ -1663,7 +1959,7 @@ export class Runtime extends EventEmitter {
1663
1959
 
1664
1960
  async getMetrics (format = 'json') {
1665
1961
  if (this.#config.metrics === false || this.#config.metrics?.enabled === false) {
1666
- throw new Error('Metrics are disabled')
1962
+ throw new MetricsDisabledError()
1667
1963
  }
1668
1964
 
1669
1965
  let metrics = null
@@ -1758,7 +2054,7 @@ export class Runtime extends EventEmitter {
1758
2054
  // Drop any configured custom label that shares the name of the application
1759
2055
  // label (a config can set both `applicationLabel: 'serviceId'` and a static
1760
2056
  // `serviceId` label): keeping it would make these runtime-wide metrics look
1761
- // like they belong to an application, both here and in getFormattedMetrics().
2057
+ // like they belong to an application during metrics aggregation.
1762
2058
  const processLabels = { ...this.#config.metrics?.labels }
1763
2059
  delete processLabels[this.#metricsLabelName]
1764
2060
  this.#applyLabelsToMetrics(processMetricsJson, processLabels, processMetrics)
@@ -1923,180 +2219,41 @@ export class Runtime extends EventEmitter {
1923
2219
  return output
1924
2220
  }
1925
2221
 
1926
- async getFormattedMetrics () {
1927
- try {
1928
- const { metrics } = await this.getMetrics()
1929
-
1930
- if (metrics === null || metrics.length === 0) {
1931
- return null
1932
- }
1933
-
1934
- const metricsNames = [
1935
- 'process_cpu_percent_usage',
1936
- 'process_resident_memory_bytes',
1937
- 'nodejs_heap_size_total_bytes',
1938
- 'nodejs_heap_size_used_bytes',
1939
- 'nodejs_heap_space_size_total_bytes',
1940
- 'nodejs_eventloop_utilization',
1941
- 'http_request_all_summary_seconds'
1942
- ]
1943
-
1944
- const applicationsMetrics = {}
1945
-
1946
- // Process-level metrics are reported only once for the whole runtime, without
1947
- // an application label, since they are shared by all the applications running
1948
- // in worker threads (see issue #3332). Applications running as separate OS
1949
- // processes report their own labeled values, which take precedence below.
1950
- const runtimeProcessMetrics = {}
1951
- const applicationProcessMetrics = new Set()
1952
-
1953
- for (const metric of metrics) {
1954
- const { name, values } = metric
1955
-
1956
- if (!metricsNames.includes(name)) continue
1957
- if (!values || values.length === 0) continue
1958
-
1959
- const labels = values[0].labels
1960
- // Use the configured label name (serviceId for v2 compatibility, applicationId for v3+)
1961
- const applicationId = labels?.[this.#metricsLabelName]
1962
-
1963
- if (!applicationId) {
1964
- if (name === 'process_cpu_percent_usage') {
1965
- runtimeProcessMetrics.cpu = values[0].value
1966
- continue
1967
- }
1968
- if (name === 'process_resident_memory_bytes') {
1969
- runtimeProcessMetrics.rss = values[0].value
1970
- continue
1971
- }
1972
-
1973
- throw new Error(`Missing ${this.#metricsLabelName} label in metrics`)
1974
- }
1975
-
1976
- if (name === 'process_cpu_percent_usage' || name === 'process_resident_memory_bytes') {
1977
- applicationProcessMetrics.add(`${applicationId}:${name}`)
1978
- }
1979
-
1980
- let applicationMetrics = applicationsMetrics[applicationId]
1981
- if (!applicationMetrics) {
1982
- applicationMetrics = {
1983
- cpu: 0,
1984
- rss: 0,
1985
- totalHeapSize: 0,
1986
- usedHeapSize: 0,
1987
- newSpaceSize: 0,
1988
- oldSpaceSize: 0,
1989
- elu: 0,
1990
- latency: {
1991
- p50: 0,
1992
- p90: 0,
1993
- p95: 0,
1994
- p99: 0
1995
- }
1996
- }
1997
- applicationsMetrics[applicationId] = applicationMetrics
1998
- }
1999
-
2000
- parsePromMetric(applicationMetrics, metric)
2001
- }
2002
-
2003
- // Apply the runtime-wide process-level values to every application that did
2004
- // not report its own (i.e. every application running in a worker thread).
2005
- for (const [applicationId, applicationMetrics] of Object.entries(applicationsMetrics)) {
2006
- if (
2007
- runtimeProcessMetrics.cpu !== undefined &&
2008
- !applicationProcessMetrics.has(`${applicationId}:process_cpu_percent_usage`)
2009
- ) {
2010
- applicationMetrics.cpu = runtimeProcessMetrics.cpu
2011
- }
2012
-
2013
- if (
2014
- runtimeProcessMetrics.rss !== undefined &&
2015
- !applicationProcessMetrics.has(`${applicationId}:process_resident_memory_bytes`)
2016
- ) {
2017
- applicationMetrics.rss = runtimeProcessMetrics.rss
2018
- }
2019
- }
2020
-
2021
- function parsePromMetric (applicationMetrics, promMetric) {
2022
- const { name } = promMetric
2023
-
2024
- if (name === 'process_cpu_percent_usage') {
2025
- applicationMetrics.cpu = promMetric.values[0].value
2026
- return
2027
- }
2028
- if (name === 'process_resident_memory_bytes') {
2029
- applicationMetrics.rss = promMetric.values[0].value
2030
- return
2031
- }
2032
- if (name === 'nodejs_heap_size_total_bytes') {
2033
- applicationMetrics.totalHeapSize = promMetric.values[0].value
2034
- return
2035
- }
2036
- if (name === 'nodejs_heap_size_used_bytes') {
2037
- applicationMetrics.usedHeapSize = promMetric.values[0].value
2038
- return
2039
- }
2040
- if (name === 'nodejs_heap_space_size_total_bytes') {
2041
- const newSpaceSize = promMetric.values.find(value => value.labels.space === 'new')
2042
- const oldSpaceSize = promMetric.values.find(value => value.labels.space === 'old')
2043
-
2044
- applicationMetrics.newSpaceSize = newSpaceSize.value
2045
- applicationMetrics.oldSpaceSize = oldSpaceSize.value
2046
- return
2047
- }
2048
- if (name === 'nodejs_eventloop_utilization') {
2049
- applicationMetrics.elu = promMetric.values[0].value
2050
- return
2051
- }
2052
- if (name === 'http_request_all_summary_seconds') {
2053
- applicationMetrics.latency = {
2054
- p50: promMetric.values.find(value => value.labels.quantile === 0.5)?.value || 0,
2055
- p90: promMetric.values.find(value => value.labels.quantile === 0.9)?.value || 0,
2056
- p95: promMetric.values.find(value => value.labels.quantile === 0.95)?.value || 0,
2057
- p99: promMetric.values.find(value => value.labels.quantile === 0.99)?.value || 0
2058
- }
2059
- }
2060
- }
2061
-
2062
- return {
2063
- version: 1,
2064
- date: new Date().toISOString(),
2065
- applications: applicationsMetrics
2066
- }
2067
- } catch (err) {
2068
- // If any metric is missing, return nothing
2069
- this.logger.warn({ err }, 'Cannot fetch metrics')
2070
-
2071
- return null
2072
- }
2073
- }
2074
-
2075
2222
  getSharedContext () {
2076
2223
  return this.#sharedContext
2077
2224
  }
2078
2225
 
2079
2226
  async getApplicationResourcesInfo (id) {
2080
- const workersCount = this.#workers.getKeys(id).length
2227
+ const workers = this.#replicas.get(id)?.length ?? 0
2081
2228
  // Use round-robin to get any available worker instead of assuming index 0 exists
2082
2229
  const worker = await this.#getWorkerByIdOrNext(id, null, false, false)
2083
- const health = worker[kConfig].health
2084
2230
 
2085
- return { workers: workersCount, health }
2231
+ if (worker) {
2232
+ return { workers, health: worker[kConfig].health }
2233
+ }
2234
+
2235
+ // No worker can answer (the application is stopped or its only worker is being restarted), use the configuration
2236
+ const applicationConfig = this.#applications.get(id)
2237
+ const health = deepmerge(this.#config.health ?? {}, applicationConfig.health ?? {})
2238
+
2239
+ return { workers, health }
2086
2240
  }
2087
2241
 
2088
2242
  getApplicationsIds () {
2089
- return Array.from(this.#applications.keys())
2243
+ return Array.from(this.#applications.keys()).sort()
2090
2244
  }
2091
2245
 
2092
2246
  async getApplications (allowUnloaded = false) {
2093
- return {
2094
- entrypoint: this.#entrypointId,
2247
+ return frozenSnapshot({
2095
2248
  production: this.#isProduction,
2096
2249
  applications: await Promise.all(
2097
- this.getApplicationsIds().map(id => this.getApplicationDetails(id, allowUnloaded))
2250
+ this.getApplicationsIds().map(id => this.#buildApplicationDetails(id, allowUnloaded))
2098
2251
  )
2099
- }
2252
+ })
2253
+ }
2254
+
2255
+ async getApplicationDetails (id, allowUnloaded = false) {
2256
+ return frozenSnapshot(await this.#buildApplicationDetails(id, allowUnloaded))
2100
2257
  }
2101
2258
 
2102
2259
  async getApplicationMeta (id) {
@@ -2124,7 +2281,7 @@ export class Runtime extends EventEmitter {
2124
2281
  }
2125
2282
  }
2126
2283
 
2127
- async getApplicationDetails (id, allowUnloaded = false) {
2284
+ async #buildApplicationDetails (id, allowUnloaded = false) {
2128
2285
  let application
2129
2286
 
2130
2287
  try {
@@ -2137,7 +2294,7 @@ export class Runtime extends EventEmitter {
2137
2294
  throw e
2138
2295
  }
2139
2296
 
2140
- const { entrypoint, localUrl, config, path } = application[kConfig]
2297
+ const { localUrl, config, configPath, path } = application[kConfig]
2141
2298
 
2142
2299
  const sourceMaps = application[kConfig].sourceMaps ?? this.#config.sourceMaps
2143
2300
  const status = await sendViaITC(application, 'getStatus')
@@ -2146,24 +2303,43 @@ export class Runtime extends EventEmitter {
2146
2303
  const applicationDetails = {
2147
2304
  id,
2148
2305
  type,
2149
- config,
2150
2306
  path,
2151
2307
  status,
2152
2308
  dependencies,
2153
2309
  version,
2154
2310
  localUrl,
2155
- entrypoint,
2156
2311
  sourceMaps
2157
2312
  }
2158
2313
 
2159
- if (this.#isProduction) {
2160
- applicationDetails.workers = this.#workers.getKeys(id).length
2314
+ /*
2315
+ The `config` key once held the application's configuration file path. That file is now
2316
+ evaluated main-side and the worker receives the payload, so the path becomes `configPath` and the entry's own
2317
+ `config` is no longer a path to report. Emitting the key that matches the dialect keeps a
2318
+ consumer from reading one and silently getting the other.
2319
+ */
2320
+ if (typeof config === 'string') {
2321
+ applicationDetails.config = config
2322
+ } else if (configPath) {
2323
+ applicationDetails.configPath = configPath
2161
2324
  }
2162
2325
 
2163
- if (entrypoint) {
2164
- applicationDetails.url = status === 'started' ? this.#url : null
2326
+ // status is worker lifecycle; servingState is how the thing serves. They are different
2327
+ // questions, and the runtime gates URL emission on status === 'started', so they cannot share
2328
+ // a field.
2329
+ const servingState = this.#servingStates.get(id)
2330
+
2331
+ if (servingState) {
2332
+ applicationDetails.servingState = servingState
2165
2333
  }
2166
2334
 
2335
+ if (this.#isProduction) {
2336
+ applicationDetails.workers = this.#replicas.get(id)?.length ?? 0
2337
+ }
2338
+
2339
+ const urls = status === 'started' ? Object.values(this.getUrls(id)) : []
2340
+ applicationDetails.urls = urls
2341
+ applicationDetails.url = urls[0] ?? null
2342
+
2167
2343
  return applicationDetails
2168
2344
  }
2169
2345
 
@@ -2211,10 +2387,6 @@ export class Runtime extends EventEmitter {
2211
2387
  return this.#scheduler?.getJobs() ?? []
2212
2388
  }
2213
2389
 
2214
- getScheduler () {
2215
- return this.getSchedulerJobs()
2216
- }
2217
-
2218
2390
  pauseSchedulerJob (name) {
2219
2391
  return this.#schedulerOrThrow().pauseJob(name)
2220
2392
  }
@@ -2283,6 +2455,7 @@ export class Runtime extends EventEmitter {
2283
2455
  worker: index,
2284
2456
  status: worker[kWorkerStatus],
2285
2457
  thread: worker.threadId,
2458
+ url: worker[kWorkerUrl] ?? null,
2286
2459
  raw: includeRaw ? worker : undefined
2287
2460
  }
2288
2461
  }
@@ -2309,10 +2482,6 @@ export class Runtime extends EventEmitter {
2309
2482
  elu = worker.performance.eventLoopUtilization(elu, previousELU)
2310
2483
  }
2311
2484
 
2312
- if (!features.node.worker.getHeapStatistics) {
2313
- return { elu: elu.utilization, currentELU }
2314
- }
2315
-
2316
2485
  // Only refresh heap statistics every 60 health checks (once per minute).
2317
2486
  // This keeps the common path fully synchronous — no promise allocation.
2318
2487
  const counter = (worker[kHeapCheckCounter] ?? 0) + 1
@@ -2404,12 +2573,37 @@ export class Runtime extends EventEmitter {
2404
2573
  this.emitAndNotify(status, args)
2405
2574
  }
2406
2575
 
2407
- #showUrl () {
2408
- if (!this.#url) {
2409
- return
2576
+ #showUrls (workerIds) {
2577
+ const whitelist = workerIds ? new Set(workerIds) : null
2578
+
2579
+ for (const applicationId of this.#applications.keys()) {
2580
+ for (const workerId of this.#workers.getKeys(applicationId)) {
2581
+ if (whitelist && !whitelist.has(workerId)) {
2582
+ continue
2583
+ }
2584
+
2585
+ const worker = this.#workers.get(workerId)
2586
+ const url = worker?.[kWorkerUrl]
2587
+ if (!url) {
2588
+ continue
2589
+ }
2590
+
2591
+ this.logger.info(
2592
+ `Platformatic is now listening at ${url} for ${this.#workerExtendedLabel(applicationId, worker[kWorkerId])}`
2593
+ )
2594
+ }
2410
2595
  }
2596
+ }
2597
+
2598
+ async #refuseUnresolvedApplication (id) {
2599
+ const executable = getExecutable() ?? 'wattpm'
2411
2600
 
2412
- this.logger.info(`Platformatic is now listening at ${this.#url}`)
2601
+ this.logger.error(
2602
+ `The path for application "%s" does not exist. Please run "${executable} resolve" and try again.`,
2603
+ id
2604
+ )
2605
+
2606
+ await this.closeAndThrow(new RuntimeAbortedError())
2413
2607
  }
2414
2608
 
2415
2609
  async #setupApplication (applicationConfig) {
@@ -2427,13 +2621,7 @@ export class Runtime extends EventEmitter {
2427
2621
  applicationConfig.path = join(this.#root, config.resolvedApplicationsBasePath, id)
2428
2622
 
2429
2623
  if (!existsSync(applicationConfig.path)) {
2430
- const executable = getExecutable() ?? 'platformatic'
2431
- this.logger.error(
2432
- `The path for application "%s" does not exist. Please run "${executable} resolve" and try again.`,
2433
- id
2434
- )
2435
-
2436
- await this.closeAndThrow(new RuntimeAbortedError())
2624
+ await this.#refuseUnresolvedApplication(id)
2437
2625
  }
2438
2626
  } else {
2439
2627
  this.logger.error(
@@ -2443,17 +2631,47 @@ export class Runtime extends EventEmitter {
2443
2631
 
2444
2632
  await this.closeAndThrow(new RuntimeAbortedError())
2445
2633
  }
2634
+ } else if (applicationConfig.unresolved) {
2635
+ /*
2636
+ A remote application that declares where its clone belongs, and whose clone is not there --
2637
+ either the directory is missing or it holds no application at all. The pathless case above
2638
+ is the same state said differently, and both are what `resolve` exists to fix, so both get
2639
+ the message that names it rather than the capability detector's report that the directory
2640
+ holds nothing.
2641
+ */
2642
+ await this.#refuseUnresolvedApplication(id)
2446
2643
  }
2447
2644
 
2448
- const workers = applicationConfig.workers.static
2449
- const setupInvocations = []
2645
+ let workers = applicationConfig.workers.static
2646
+
2647
+ this.#replicas.set(id, [])
2450
2648
 
2451
- for (let i = 0; i < workers; i++) {
2452
- this.#workerPortOffsets.set(`${id}:${i}`, i)
2453
- setupInvocations.push([config, applicationConfig, workers, id, i])
2649
+ let firstIndex = 0
2650
+
2651
+ // On platforms where reusePort is not available, multiple workers cannot listen on the same fixed port.
2652
+ // The listener configuration is owned by the capability, so it can only be inspected after setting up the first
2653
+ // worker: if the application would try to share a fixed port between workers, clamp it to a single worker.
2654
+ if (!features.node.reusePort && (workers > 1 || applicationConfig.workers.dynamic)) {
2655
+ const { worker } = await this.#setupReplica(config, applicationConfig, workers, id, 0, 0)
2656
+ firstIndex = 1
2657
+
2658
+ if (await this.#usesSharedFixedPort(worker)) {
2659
+ this.logger.warn(
2660
+ `The application "${id}" is configured to listen on a fixed port with multiple workers, but reusePort is not available in your OS. ${workers > 1 ? `Setting workers to 1 instead of ${workers}` : 'Disabling dynamic workers scaling'}. To run multiple workers, set "server.portAssignment" to "perWorkerIncrement" in the application configuration.`
2661
+ )
2662
+
2663
+ applicationConfig.workers = { dynamic: false, static: 1 }
2664
+ workers = 1
2665
+ await sendViaITC(worker, 'updateWorkersCount', { applicationId: id, workers })
2666
+ }
2667
+ }
2668
+
2669
+ const setupInvocations = []
2670
+ for (let i = firstIndex; i < workers; i++) {
2671
+ setupInvocations.push([config, applicationConfig, workers, id, i, i])
2454
2672
  }
2455
2673
 
2456
- await executeInParallel(this.#setupWorker.bind(this), setupInvocations, this.#concurrency)
2674
+ await executeInParallel(this.#setupReplica.bind(this), setupInvocations, this.#concurrency)
2457
2675
 
2458
2676
  // Initialize the next worker index counter (next index starts after initial workers)
2459
2677
  this.#nextWorkerIndex.set(id, workers)
@@ -2462,7 +2680,51 @@ export class Runtime extends EventEmitter {
2462
2680
  this.emitAndNotify('application:init', id)
2463
2681
  }
2464
2682
 
2465
- async #setupWorker (config, applicationConfig, workersCount, applicationId, index, enabled = true, attempt = 0) {
2683
+ // Adds a replica to the application (see the #replicas field) and sets up its worker.
2684
+ // Returns null if the replica was removed in the meanwhile.
2685
+ async #setupReplica (config, applicationConfig, workersCount, applicationId, index, portOffset) {
2686
+ const replica = {
2687
+ portOffset,
2688
+ // The worker serving the replica, null while the replica is being refilled
2689
+ worker: null,
2690
+ // The worker being started to refill the replica or to replace its worker
2691
+ replacement: null,
2692
+ // The operation in progress: 'refill' or 'replace'
2693
+ pending: null,
2694
+ abortController: new AbortController()
2695
+ }
2696
+
2697
+ // Added right away, so that concurrent operations see the requested workers
2698
+ this.#replicas.get(applicationId).push(replica)
2699
+
2700
+ let worker
2701
+ try {
2702
+ worker = await this.#setupWorker(config, applicationConfig, workersCount, applicationId, index, portOffset)
2703
+ } catch (err) {
2704
+ await this.#detachReplica(applicationId, replica)
2705
+ throw err
2706
+ }
2707
+
2708
+ // The replica was removed in the meanwhile (i.e. scale-down or application stopped)
2709
+ if (replica.abortController.signal.aborted) {
2710
+ await this.#discardWorker(worker)
2711
+ return null
2712
+ }
2713
+
2714
+ replica.worker = worker
2715
+ return replica
2716
+ }
2717
+
2718
+ async #setupWorker (
2719
+ config,
2720
+ applicationConfig,
2721
+ workersCount,
2722
+ applicationId,
2723
+ index,
2724
+ portOffset,
2725
+ enabled = true,
2726
+ attempt = 0
2727
+ ) {
2466
2728
  const restartOnError = this.#getApplicationRestartOnError(config, applicationConfig)
2467
2729
  const workerId = `${applicationId}:${index}`
2468
2730
 
@@ -2477,17 +2739,11 @@ export class Runtime extends EventEmitter {
2477
2739
  inspectorOptions.port = inspectorOptions.port + this.#workers.size + 1
2478
2740
  }
2479
2741
 
2480
- let serverConfigOverride
2481
- if (this.#config.server?.portAssignment === 'perWorkerIncrement') {
2482
- const portOffset = this.#workerPortOffsets.get(workerId)
2483
- serverConfigOverride = { port: Number(this.#config.server.port) + portOffset }
2484
- }
2485
-
2486
- if (config.telemetry) {
2487
- applicationConfig.telemetry = {
2488
- ...config.telemetry,
2489
- ...applicationConfig.telemetry,
2490
- applicationName: `${config.telemetry.applicationName}-${applicationConfig.id}`
2742
+ if (config.tracing) {
2743
+ applicationConfig.tracing = {
2744
+ ...config.tracing,
2745
+ ...applicationConfig.tracing,
2746
+ applicationName: `${config.tracing.applicationName}-${applicationConfig.id}`
2491
2747
  }
2492
2748
  }
2493
2749
 
@@ -2496,9 +2752,9 @@ export class Runtime extends EventEmitter {
2496
2752
 
2497
2753
  const execArgv = applicationConfig.execArgv ?? []
2498
2754
 
2499
- if (!applicationConfig.skipTelemetryHooks && config.telemetry && config.telemetry.enabled !== false) {
2755
+ if (!applicationConfig.skipTracingHooks && config.tracing && config.tracing.enabled !== false) {
2500
2756
  const require = createRequire(import.meta.url)
2501
- const telemetryPath = require.resolve('@platformatic/telemetry')
2757
+ const telemetryPath = require.resolve('@platformatic/tracing')
2502
2758
  const openTelemetrySetupPath = join(telemetryPath, '..', 'lib', 'node-telemetry.js')
2503
2759
  const hookUrl = pathToFileURL(require.resolve('@opentelemetry/instrumentation/hook.mjs'))
2504
2760
 
@@ -2523,7 +2779,10 @@ export class Runtime extends EventEmitter {
2523
2779
  preload = preload.filter(p => p !== pprofCapturePath)
2524
2780
  }
2525
2781
 
2526
- const workerEnv = structuredClone(this.#env)
2782
+ // Each application's worker environment is resolved main-side, with its own env-file chain, the
2783
+ // two env blocks and the injected topology URLs. Every worker used to be seeded from one loadEnv
2784
+ // at the runtime root, which is what #env still holds.
2785
+ const workerEnv = structuredClone(applicationConfig.workerEnv ?? this.#env)
2527
2786
 
2528
2787
  if (applicationConfig.nodeOptions?.trim().length > 0) {
2529
2788
  const originalNodeOptions = workerEnv.NODE_OPTIONS ?? ''
@@ -2546,21 +2805,12 @@ export class Runtime extends EventEmitter {
2546
2805
  const maxYoungGenerationSizeMb = maxYoungGeneration ? Math.floor(maxYoungGeneration / (1024 * 1024)) : undefined
2547
2806
  const codeRangeSizeMb = codeRangeSize ? Math.floor(codeRangeSize / (1024 * 1024)) : undefined
2548
2807
 
2549
- const workerConfig = {
2550
- ...config,
2551
- preload
2552
- }
2553
-
2554
- if (config.server && serverConfigOverride) {
2555
- workerConfig.server = {
2556
- ...config.server,
2557
- ...serverConfigOverride
2558
- }
2559
- }
2808
+ const workerConfig = { ...config, preload }
2560
2809
 
2561
2810
  const worker = new Worker(kWorkerFile, {
2562
2811
  workerData: {
2563
2812
  config: workerConfig,
2813
+ meshId: this.#meshId,
2564
2814
  applicationConfig: {
2565
2815
  ...applicationConfig,
2566
2816
  isProduction: this.#isProduction,
@@ -2569,7 +2819,8 @@ export class Runtime extends EventEmitter {
2569
2819
  worker: {
2570
2820
  id: workerId,
2571
2821
  index,
2572
- count: workersCount
2822
+ count: workersCount,
2823
+ portOffset
2573
2824
  },
2574
2825
  resourceLimits: {
2575
2826
  maxOldGenerationSizeMb,
@@ -2577,10 +2828,7 @@ export class Runtime extends EventEmitter {
2577
2828
  codeRangeSizeMb
2578
2829
  },
2579
2830
  inspectorOptions,
2580
- dirname: this.#root,
2581
- // Keys of the worker environment which only come from an env file of the runtime:
2582
- // the env file of the application is allowed to override those.
2583
- envFileFallbackKeys: this.#env[kEnvFileFallbackKeys] ?? []
2831
+ dirname: this.#root
2584
2832
  },
2585
2833
  argv: applicationConfig.arguments,
2586
2834
  execArgv,
@@ -2604,15 +2852,24 @@ export class Runtime extends EventEmitter {
2604
2852
  const eventPayload = { application: applicationId, worker: index, workersCount }
2605
2853
 
2606
2854
  worker.once('exit', code => {
2855
+ worker[kWorkerExitCode] = code
2856
+
2607
2857
  if (worker[kWorkerStatus] === 'exited') {
2608
2858
  return
2609
2859
  }
2610
2860
 
2611
2861
  const started = worker[kWorkerStatus] === 'started'
2862
+
2863
+ // A crashed worker leaves its replica empty until the replica is refilled below.
2864
+ // Workers which crash while starting are handled by #startWorker.
2865
+ const replica = started ? this.#getWorkerReplica(applicationId, worker) : undefined
2866
+ if (replica) {
2867
+ replica.worker = null
2868
+ }
2869
+
2612
2870
  worker[kWorkerStatus] = 'exited'
2613
2871
  this.emitAndNotify('application:worker:exited', eventPayload)
2614
2872
 
2615
- const portOffset = this.#workerPortOffsets.get(worker[kFullId]) ?? index
2616
2873
  this.#cleanupWorker(worker)
2617
2874
 
2618
2875
  if (this.#status === 'stopping') {
@@ -2628,8 +2885,9 @@ export class Runtime extends EventEmitter {
2628
2885
  this.logger.warn(`The ${errorLabel} unexpectedly exited with code ${code}.`)
2629
2886
  }
2630
2887
 
2631
- // Restart the application if it was started
2632
- if (started && this.#status === 'started') {
2888
+ // Refill the replica of the worker if it was started. Workers which were not serving a replica anymore
2889
+ // (i.e. being retired after a restart) are not replaced, and a replica being replaced gets the replacement.
2890
+ if (started && this.#status === 'started' && replica && replica.pending !== 'replace') {
2633
2891
  if (restartOnError > 0) {
2634
2892
  if (restartOnError < IMMEDIATE_RESTART_MAX_THRESHOLD) {
2635
2893
  this.logger.warn(`The ${errorLabel} is being restarted ...`)
@@ -2637,14 +2895,16 @@ export class Runtime extends EventEmitter {
2637
2895
  this.logger.warn(`The ${errorLabel} will be restarted in ${restartOnError}ms ...`)
2638
2896
  }
2639
2897
 
2640
- this.#restartCrashedWorker(config, applicationConfig, workersCount, applicationId, index, false, 0, portOffset).catch(
2898
+ this.#refillReplica(config, applicationConfig, workersCount, applicationId, replica, false, 0).catch(
2641
2899
  err => {
2642
2900
  this.logger.error({ err: ensureLoggableError(err) }, `${errorLabel} could not be restarted.`)
2901
+ this.#detachReplica(applicationId, replica)
2643
2902
  }
2644
2903
  )
2645
2904
  } else {
2646
2905
  this.emitAndNotify('application:worker:unvailable', eventPayload)
2647
2906
  this.logger.warn(`The ${errorLabel} is no longer available.`)
2907
+ this.#detachReplica(applicationId, replica)
2648
2908
  }
2649
2909
  }
2650
2910
  })
@@ -2654,6 +2914,7 @@ export class Runtime extends EventEmitter {
2654
2914
  worker[kFullId] = workerId
2655
2915
  worker[kApplicationId] = applicationId
2656
2916
  worker[kWorkerId] = index
2917
+ worker[kWorkerPortOffset] = portOffset
2657
2918
  worker[kWorkerStatus] = 'boot'
2658
2919
 
2659
2920
  if (inspectorOptions) {
@@ -2694,18 +2955,47 @@ export class Runtime extends EventEmitter {
2694
2955
  // Forward events from the worker
2695
2956
  // Do not use emitAndNotify here since we don't want to forward unknown events
2696
2957
  worker[kITC].on('event', ({ event, payload }) => {
2958
+ if (event === 'serverOptions') {
2959
+ worker[kWorkerServerOptions] = payload[0]
2960
+ }
2961
+
2697
2962
  event = `application:worker:event:${event}`
2698
2963
 
2699
2964
  this.emit(event, ...payload, workerId, applicationId, index)
2700
2965
  this.logger.trace({ event, payload, id: workerId, application: applicationId, worker: index }, 'Runtime event')
2701
2966
  })
2702
2967
 
2968
+ worker[kITC].on('compile-cache:flushed', ({ flushed, source }) => {
2969
+ this.emit('application:worker:compile-cache:flushed', {
2970
+ application: applicationId,
2971
+ worker: index,
2972
+ workersCount,
2973
+ flushed,
2974
+ source
2975
+ })
2976
+ })
2977
+
2978
+ worker[kITC].on('compile-cache:unavailable', ({ source }) => {
2979
+ this.emit('application:worker:compile-cache:unavailable', {
2980
+ application: applicationId,
2981
+ worker: index,
2982
+ workersCount,
2983
+ source
2984
+ })
2985
+ })
2986
+
2703
2987
  // The worker notifies us when its capability has spawned a child process
2704
2988
  // (e.g. Next.js in dev mode). From that point on health metrics must come
2705
2989
  // from the child via ITC; for thread-only workers we keep reading the
2706
2990
  // handle directly in getWorkerHealth().
2707
- worker[kITC].on('subprocess:started', () => {
2991
+ worker[kITC].on('subprocess:started', ({ pid } = {}) => {
2708
2992
  worker[kIsSubprocessHost] = true
2993
+ worker[kSubprocessPid] = pid
2994
+ })
2995
+ worker[kITC].on('subprocess:exited', ({ pid }) => {
2996
+ if (worker[kSubprocessPid] === pid) {
2997
+ worker[kSubprocessPid] = undefined
2998
+ }
2709
2999
  })
2710
3000
 
2711
3001
  worker[kITC].on(openTelemetryITCMessage, resourceMetrics => {
@@ -2867,9 +3157,7 @@ export class Runtime extends EventEmitter {
2867
3157
  this.logger.info(`The application "${applicationId}" has been successfully reloaded.`)
2868
3158
  this.emitAndNotify('application:worker:reloaded', eventPayload)
2869
3159
 
2870
- if (applicationConfig.entrypoint) {
2871
- this.#showUrl()
2872
- }
3160
+ this.#showUrls(this.#workers.getKeys(applicationId))
2873
3161
  } catch (e) {
2874
3162
  this.logger.error(e)
2875
3163
  }
@@ -2881,25 +3169,6 @@ export class Runtime extends EventEmitter {
2881
3169
  this.#workers.set(workerId, worker)
2882
3170
  }
2883
3171
 
2884
- // Setup the interceptor
2885
- // kInterceptorReadyPromise resolves when the worker
2886
- // is ready to receive requests: after calling the replaceServer method
2887
- //
2888
- // It is stored, not awaited: #startWorker awaits it later, and only if the
2889
- // worker is actually started. A worker that is discarded or torn down
2890
- // before that — a replacement abandoned because the runtime stopped
2891
- // mid-restart, for instance — leaves the promise with nobody to observe
2892
- // it. Closing the mesh interceptor rejects any routing still in flight
2893
- // ('The dispatcher has been closed.'), so that is a routine shutdown
2894
- // outcome, not an exotic one, and it surfaced as an unhandledRejection
2895
- // that could take the process down before graceful shutdown finished.
2896
- //
2897
- // Attaching a no-op handler marks the rejection observed WITHOUT
2898
- // swallowing it: the await in #startWorker still sees and reports it.
2899
- const interceptorReady = this.#meshInterceptor.route(applicationId, worker)
2900
- interceptorReady.catch(() => {})
2901
- worker[kInterceptorReadyPromise] = interceptorReady
2902
-
2903
3172
  // Wait for initialization
2904
3173
  try {
2905
3174
  await waitEventFromITC(worker, 'init')
@@ -2918,11 +3187,16 @@ export class Runtime extends EventEmitter {
2918
3187
  }
2919
3188
 
2920
3189
  this.#workers.delete(workerId)
2921
- return this.#setupWorker(config, applicationConfig, workersCount, applicationId, index, enabled, attempt + 1)
2922
- }
2923
-
2924
- if (applicationConfig.entrypoint) {
2925
- this.#entrypointId = applicationId
3190
+ return this.#setupWorker(
3191
+ config,
3192
+ applicationConfig,
3193
+ workersCount,
3194
+ applicationId,
3195
+ index,
3196
+ portOffset,
3197
+ enabled,
3198
+ attempt + 1
3199
+ )
2926
3200
  }
2927
3201
 
2928
3202
  worker[kConfig] = { ...applicationConfig, health, workers: workersCount }
@@ -3339,29 +3613,8 @@ export class Runtime extends EventEmitter {
3339
3613
 
3340
3614
  const { url: workerUrl, scheduledTasks } = workerStartResult
3341
3615
  worker[kWorkerScheduledTasks] = scheduledTasks
3342
- if (workerUrl) {
3343
- this.#url = workerUrl
3344
-
3345
- // Pin the entrypoint port so that subsequent restarts (especially with
3346
- // stopBeforeStart when reuseTcpPorts is false) bind to the same port
3347
- // instead of getting a new random one.
3348
- if (applicationConfig.entrypoint) {
3349
- try {
3350
- const boundPort = Number(new URL(workerUrl).port)
3351
- if (boundPort) {
3352
- this.#entrypointPort = boundPort
3353
- }
3354
- } catch {
3355
- // URL parsing failed, leave unchanged
3356
- }
3357
- }
3358
- }
3359
3616
 
3360
- // Wait for the interceptor to be ready
3361
- const interceptorResult = await executeWithTimeout(worker[kInterceptorReadyPromise], config.startTimeout)
3362
- if (interceptorResult === kTimeout) {
3363
- throw new WorkerInterceptorJoinTimeoutError(label, config.startTimeout)
3364
- }
3617
+ this.#recordWorkerUrl(worker, id, workerUrl)
3365
3618
 
3366
3619
  worker[kWorkerStatus] = 'started'
3367
3620
  worker[kWorkerStartTime] = Date.now()
@@ -3385,12 +3638,28 @@ export class Runtime extends EventEmitter {
3385
3638
  ).unref()
3386
3639
  }
3387
3640
  } catch (err) {
3388
- const error = ensureError(err)
3641
+ let error = ensureError(err)
3642
+ // Forced shutdown may terminate the thread before its start handler can report the abort.
3643
+ if (
3644
+ worker[kWorkerStartAborted] &&
3645
+ error.code === 'PLT_RUNTIME_APPLICATION_WORKER_EXIT'
3646
+ ) {
3647
+ error = new RuntimeAbortedError({ cause: error })
3648
+ }
3389
3649
  worker[kITC].notify('application:worker:start:processed')
3390
- const portOffset = this.#workerPortOffsets.get(worker[kFullId]) ?? index
3391
3650
 
3392
- // TODO: handle port allocation error here
3393
- if (error.code === 'EADDRINUSE' || error.code === 'EACCES') throw error
3651
+ if (error.code === 'EADDRINUSE' && Number.isInteger(Number(error.port))) {
3652
+ const port = Number(error.port)
3653
+ const owner = this.#getPortOwner(port, id)
3654
+
3655
+ if (owner) {
3656
+ error = new AddressInUseError(port, owner, id)
3657
+ } else if (this.#getPortOwner(port, id, undefined, true)) {
3658
+ error = new WorkerAddressInUseError(port, id)
3659
+ }
3660
+ // Otherwise the port is used by an external process: keep the original error, which already describes it
3661
+ }
3662
+ if (error.code === 'EACCES') throw error
3394
3663
 
3395
3664
  this.#cleanupWorker(worker)
3396
3665
 
@@ -3414,7 +3683,16 @@ export class Runtime extends EventEmitter {
3414
3683
 
3415
3684
  const restartOnError = this.#getApplicationRestartOnError(config, applicationConfig)
3416
3685
 
3417
- if (disableRestartAttempts || !restartOnError) {
3686
+ if (
3687
+ disableRestartAttempts ||
3688
+ !restartOnError ||
3689
+ error.code === 'EACCES' ||
3690
+ error.code === 'EADDRINUSE' ||
3691
+ error.code === 'EADDRNOTAVAIL' ||
3692
+ error.code === 'PLT_RUNTIME_EADDR_IN_USE' ||
3693
+ error.code === 'PLT_RUNTIME_RUNTIME_ABORT' ||
3694
+ error.code === 'PLT_RUNTIME_WORKER_EADDR_IN_USE'
3695
+ ) {
3418
3696
  throw error
3419
3697
  }
3420
3698
 
@@ -3434,7 +3712,64 @@ export class Runtime extends EventEmitter {
3434
3712
  )
3435
3713
  }
3436
3714
 
3437
- await this.#restartCrashedWorker(config, applicationConfig, workersCount, id, index, silent, bootstrapAttempt, portOffset)
3715
+ // The replica was removed while the worker was starting, there is nothing to refill
3716
+ const replica = this.#getWorkerReplica(id, worker)
3717
+ if (!replica) {
3718
+ throw error
3719
+ }
3720
+ replica.worker = null
3721
+ await this.#refillReplica(config, applicationConfig, workersCount, id, replica, silent, bootstrapAttempt)
3722
+ }
3723
+ }
3724
+
3725
+ async #startFirstWorkerWithCompileCache (config, applicationConfig, workers, id, silent) {
3726
+ const cacheFlushEvent = 'application:worker:compile-cache:flushed'
3727
+ const cacheUnavailableEvent = 'application:worker:compile-cache:unavailable'
3728
+ const { promise, resolve } = Promise.withResolvers()
3729
+ const cacheEvents = []
3730
+ const runtime = this
3731
+ let expectedSource
3732
+
3733
+ function removeCacheFlushListener () {
3734
+ runtime.off(cacheFlushEvent, cacheFlushListener)
3735
+ runtime.off(cacheUnavailableEvent, cacheUnavailableListener)
3736
+ }
3737
+
3738
+ function resolveIfReady () {
3739
+ if (expectedSource && cacheEvents.some(function (event) {
3740
+ return event.source === expectedSource
3741
+ })) {
3742
+ removeCacheFlushListener()
3743
+ resolve()
3744
+ }
3745
+ }
3746
+
3747
+ function cacheFlushListener (event) {
3748
+ if (event.application === id && event.worker === 0) {
3749
+ cacheEvents.push(event)
3750
+ resolveIfReady()
3751
+ }
3752
+ }
3753
+
3754
+ function cacheUnavailableListener (event) {
3755
+ if (event.application === id && event.worker === 0) {
3756
+ cacheEvents.push(event)
3757
+ resolveIfReady()
3758
+ }
3759
+ }
3760
+
3761
+ runtime.on(cacheFlushEvent, cacheFlushListener)
3762
+ runtime.on(cacheUnavailableEvent, cacheUnavailableListener)
3763
+
3764
+ try {
3765
+ await this.#startWorker(config, applicationConfig, workers, id, 0, silent)
3766
+ const firstWorker = this.#workers.get(`${id}:0`)
3767
+ expectedSource = firstWorker?.[kIsSubprocessHost] === true ? 'child-process' : 'worker'
3768
+ resolveIfReady()
3769
+ await promise
3770
+ } catch (err) {
3771
+ removeCacheFlushListener()
3772
+ throw err
3438
3773
  }
3439
3774
  }
3440
3775
 
@@ -3454,6 +3789,9 @@ export class Runtime extends EventEmitter {
3454
3789
 
3455
3790
  const eventPayload = { application: id, worker: index, workersCount }
3456
3791
 
3792
+ if (worker[kWorkerStatus] === 'starting' && this.error) {
3793
+ worker[kWorkerStartAborted] = true
3794
+ }
3457
3795
  worker[kWorkerStatus] = 'stopping'
3458
3796
  worker[kITC].removeAllListeners('changed')
3459
3797
  this.emitAndNotify('application:worker:stopping', eventPayload)
@@ -3469,13 +3807,18 @@ export class Runtime extends EventEmitter {
3469
3807
 
3470
3808
  // Always send the stop message, it will shut down workers that only had ITC and interceptors setup
3471
3809
  try {
3472
- const res = await executeWithTimeout(sendViaITC(worker, 'stop', { force: !!this.error, dependents }), exitTimeout)
3810
+ const res = await executeWithTimeout(
3811
+ sendViaITC(worker, 'stop', { force: !!this.error, dependents, shutdownTimeout: exitTimeout }),
3812
+ exitTimeout
3813
+ )
3473
3814
 
3474
3815
  if (res === kTimeout) {
3816
+ this.#terminateSubprocess(worker)
3475
3817
  this.emitAndNotify('application:worker:stop:timeout', eventPayload)
3476
3818
  this.logger.error(`Timeout while stopping ${label}. Killing a worker thread.`)
3477
3819
  }
3478
3820
  } catch (error) {
3821
+ this.#terminateSubprocess(worker)
3479
3822
  this.emitAndNotify('application:worker:stop:error', eventPayload)
3480
3823
  this.logger.error({ err: ensureLoggableError(error) }, `Failed to stop ${label}. Killing a worker thread.`)
3481
3824
  } finally {
@@ -3503,13 +3846,32 @@ export class Runtime extends EventEmitter {
3503
3846
  await this.#avoidOutOfOrderThreadLogs()
3504
3847
 
3505
3848
  worker[kWorkerStatus] = 'stopped'
3849
+ worker[kWorkerUrl] = undefined
3506
3850
  this.emitAndNotify('application:worker:stopped', eventPayload)
3507
3851
  this.#broadcastWorkers()
3508
3852
  }
3509
3853
 
3854
+ #terminateSubprocess (worker) {
3855
+ const pid = worker[kSubprocessPid]
3856
+ if (!Number.isInteger(pid) || pid <= 0) {
3857
+ return
3858
+ }
3859
+ try {
3860
+ // The main thread owns the fallback when a worker cannot supervise its child.
3861
+ this.logger.debug({ pid }, 'Terminating application child process from the runtime.')
3862
+ process.kill(pid, 'SIGKILL')
3863
+ } catch (error) {
3864
+ if (error.code === 'ESRCH') {
3865
+ // ESRCH means no such process: the child exited before the fallback signal arrived.
3866
+ this.logger.debug({ pid }, 'Application child process already exited before termination.')
3867
+ } else {
3868
+ this.logger.error({ err: ensureLoggableError(error) }, 'Failed to terminate application child process.')
3869
+ }
3870
+ }
3871
+ }
3872
+
3510
3873
  #cleanupWorker (worker) {
3511
3874
  clearTimeout(worker[kHealthCheckTimer])
3512
- this.#workerPortOffsets.delete(worker[kFullId])
3513
3875
 
3514
3876
  const currentWorker = this.#workers.get(worker[kFullId])
3515
3877
 
@@ -3521,8 +3883,8 @@ export class Runtime extends EventEmitter {
3521
3883
  }
3522
3884
 
3523
3885
  async #discardWorker (worker) {
3524
- await this.#meshInterceptor.unroute(worker[kApplicationId], worker, true)
3525
- worker.removeAllListeners('exit')
3886
+ // Marked synchronously, so that the exit handler ignores the worker and nobody else tries to stop it
3887
+ worker[kWorkerStatus] = 'exited'
3526
3888
  await worker.terminate()
3527
3889
 
3528
3890
  return this.#cleanupWorker(worker)
@@ -3538,6 +3900,57 @@ export class Runtime extends EventEmitter {
3538
3900
  return index
3539
3901
  }
3540
3902
 
3903
+ #getWorkerReplica (applicationId, worker) {
3904
+ return this.#replicas.get(applicationId)?.find(replica => replica.worker === worker)
3905
+ }
3906
+
3907
+ // Removes a replica and cancels the work in progress for it. Stopping its worker is up to the caller.
3908
+ async #detachReplica (applicationId, replica) {
3909
+ const replicas = this.#replicas.get(applicationId)
3910
+ const position = replicas?.indexOf(replica) ?? -1
3911
+
3912
+ if (position !== -1) {
3913
+ replicas.splice(position, 1)
3914
+ }
3915
+
3916
+ replica.abortController.abort()
3917
+
3918
+ if (replica.replacement) {
3919
+ await this.#discardWorker(replica.replacement)
3920
+ }
3921
+ }
3922
+
3923
+ // Returns the lowest port offset which is not used by any replica of the application.
3924
+ // Empty replicas keep their port offset, as they will be refilled.
3925
+ #getNextReplicaPortOffset (applicationId) {
3926
+ const used = new Set(this.#replicas.get(applicationId)?.map(replica => replica.portOffset))
3927
+
3928
+ let offset = 0
3929
+ while (used.has(offset)) {
3930
+ offset++
3931
+ }
3932
+
3933
+ return offset
3934
+ }
3935
+
3936
+ // Returns true if the application of the worker is configured to listen on a fixed port shared by all its workers
3937
+ async #usesSharedFixedPort (worker) {
3938
+ let server
3939
+
3940
+ try {
3941
+ server = (await sendViaITC(worker, 'getApplicationConfig'))?.server
3942
+ } catch {
3943
+ return false
3944
+ }
3945
+
3946
+ if (!server || server.portAssignment === 'perWorkerIncrement') {
3947
+ return false
3948
+ }
3949
+
3950
+ const port = Number(server.port)
3951
+ return Number.isInteger(port) && port > 0
3952
+ }
3953
+
3541
3954
  // Returns the effective restartOnError value for an application: the application-level
3542
3955
  // value, when defined, takes precedence over the runtime-level one.
3543
3956
  // The value is normalized to a number: false becomes 0 (never restart),
@@ -3554,89 +3967,101 @@ export class Runtime extends EventEmitter {
3554
3967
  return restartOnError
3555
3968
  }
3556
3969
 
3557
- async #restartCrashedWorker (config, applicationConfig, workersCount, id, oldIndex, silent, bootstrapAttempt, portOffset) {
3970
+ // Starts a new worker for a replica whose worker crashed (or failed to start), after the restartOnError delay
3971
+ async #refillReplica (config, applicationConfig, workersCount, id, replica, silent, bootstrapAttempt) {
3558
3972
  const restartOnError = this.#getApplicationRestartOnError(config, applicationConfig)
3973
+ const { signal } = replica.abortController
3974
+ replica.pending = 'refill'
3559
3975
 
3560
- // Use oldIndex for tracking to prevent duplicate restarts of the same crashed worker
3561
- const restartKey = `${id}:${oldIndex}`
3562
-
3563
- let restartPromise = this.#restartingWorkers.get(restartKey)
3564
- if (restartPromise) {
3565
- await restartPromise
3566
- return
3567
- }
3568
-
3569
- restartPromise = new Promise((resolve, reject) => {
3570
- async function restart () {
3571
- this.#restartingWorkers.delete(restartKey)
3572
-
3573
- // If some processes were scheduled to restart
3574
- // but the runtime is stopped, ignore it
3575
- if (!this.#status.startsWith('start')) {
3576
- return
3976
+ try {
3977
+ try {
3978
+ if (restartOnError < IMMEDIATE_RESTART_MAX_THRESHOLD) {
3979
+ await new Promise(resolve => process.nextTick(resolve))
3980
+ } else {
3981
+ await sleep(restartOnError, undefined, { signal })
3577
3982
  }
3983
+ } catch {
3984
+ // The replica was removed in the meanwhile
3985
+ return
3986
+ }
3578
3987
 
3579
- // Get a new unique index for the restarted worker
3580
- const newIndex = this.#getNextWorkerIndex(id)
3581
- const newWorkerId = `${id}:${newIndex}`
3582
- this.#workerPortOffsets.set(newWorkerId, portOffset)
3988
+ // If the replica was removed or the runtime was stopped in the meanwhile, ignore it
3989
+ if (signal.aborted || !this.#status.startsWith('start')) {
3990
+ return
3991
+ }
3583
3992
 
3584
- try {
3585
- await this.#setupWorker(config, applicationConfig, workersCount, id, newIndex)
3586
- await this.#startWorker(config, applicationConfig, workersCount, id, newIndex, silent, bootstrapAttempt)
3587
- this.#incrementApplicationRestartCount(id)
3993
+ // Get a new unique index for the worker, which inherits the port offset of the replica
3994
+ const newIndex = this.#getNextWorkerIndex(id)
3588
3995
 
3589
- this.logger.info(
3590
- `The ${this.#workerExtendedLabel(id, newIndex, workersCount)} has been successfully restarted ...`
3591
- )
3592
- resolve()
3593
- } catch (err) {
3594
- this.#workerPortOffsets.delete(newWorkerId)
3996
+ try {
3997
+ const worker = await this.#setupWorker(
3998
+ config,
3999
+ applicationConfig,
4000
+ workersCount,
4001
+ id,
4002
+ newIndex,
4003
+ replica.portOffset
4004
+ )
3595
4005
 
3596
- // The runtime was stopped while the restart was happening, ignore any error.
3597
- if (!this.#status.startsWith('start')) {
3598
- resolve()
3599
- }
4006
+ if (signal.aborted) {
4007
+ await this.#discardWorker(worker)
4008
+ return
4009
+ }
3600
4010
 
3601
- reject(err)
4011
+ replica.worker = worker
4012
+ await this.#startWorker(config, applicationConfig, workersCount, id, newIndex, silent, bootstrapAttempt, worker)
4013
+ } catch (err) {
4014
+ // The replica was removed or the runtime was stopped while the worker was starting, ignore any error.
4015
+ if (signal.aborted || !this.#status.startsWith('start')) {
4016
+ return
3602
4017
  }
3603
- }
3604
4018
 
3605
- if (restartOnError < IMMEDIATE_RESTART_MAX_THRESHOLD) {
3606
- process.nextTick(restart.bind(this))
3607
- } else {
3608
- setTimeout(restart.bind(this), restartOnError)
4019
+ throw err
3609
4020
  }
3610
- })
3611
4021
 
3612
- this.#restartingWorkers.set(restartKey, restartPromise)
3613
- await restartPromise
4022
+ this.#incrementApplicationRestartCount(id)
4023
+ this.logger.info(`The ${this.#workerExtendedLabel(id, newIndex, workersCount)} has been successfully restarted ...`)
4024
+ } finally {
4025
+ if (replica.pending === 'refill') {
4026
+ replica.pending = null
4027
+ }
4028
+ }
3614
4029
  }
3615
4030
 
3616
- async #replaceWorker (config, applicationConfig, workersCount, applicationId, oldIndex, worker, silent) {
4031
+ // Replaces the worker of a replica with a new one. Only one operation at a time is performed on a replica.
4032
+ async #replaceWorker (
4033
+ config,
4034
+ applicationConfig,
4035
+ workersCount,
4036
+ applicationId,
4037
+ oldIndex,
4038
+ worker,
4039
+ silent,
4040
+ deferOldWorkerRetirement = false
4041
+ ) {
4042
+ const replica = this.#getWorkerReplica(applicationId, worker)
4043
+
4044
+ // The worker is not serving a replica anymore, or the replica is already being refilled or replaced
4045
+ if (!replica || replica.pending) {
4046
+ return
4047
+ }
4048
+
4049
+ const { signal } = replica.abortController
3617
4050
  const oldLabel = this.#workerExtendedLabel(applicationId, oldIndex, workersCount)
3618
4051
  let newWorker
4052
+ let error
3619
4053
 
3620
- // Get a new unique index for the replacement worker
4054
+ // Get a new unique index for the replacement worker, which inherits the port offset of the replica
3621
4055
  const newIndex = this.#getNextWorkerIndex(applicationId)
3622
4056
  const newWorkerId = `${applicationId}:${newIndex}`
3623
4057
  const newLabel = this.#workerExtendedLabel(applicationId, newIndex, workersCount)
3624
- const portOffset = this.#workerPortOffsets.get(`${applicationId}:${oldIndex}`)
3625
- this.#workerPortOffsets.set(newWorkerId, portOffset)
3626
4058
 
3627
4059
  const stopBeforeStart =
3628
- applicationConfig.entrypoint &&
4060
+ Boolean(worker[kWorkerUrl]) &&
4061
+ worker[kWorkerServerOptions]?.port !== 0 &&
3629
4062
  (config.reuseTcpPorts === false || applicationConfig.reuseTcpPorts === false || !features.node.reusePort)
3630
4063
 
3631
- // When we must stop before start (no reusePort available), pin the entrypoint
3632
- // port in the config so the replacement worker binds to the same port.
3633
- // We only do this for stopBeforeStart because when reusePort is available
3634
- // the new worker starts alongside the old one and must use port 0 to avoid
3635
- // SO_REUSEPORT routing requests to the stale old worker.
3636
- let configForNewWorker = config
3637
- if (stopBeforeStart && this.#entrypointPort && config.server) {
3638
- configForNewWorker = { ...config, server: { ...config.server, port: this.#entrypointPort } }
3639
- }
4064
+ replica.pending = 'replace'
3640
4065
 
3641
4066
  try {
3642
4067
  if (!silent) {
@@ -3645,13 +4070,22 @@ export class Runtime extends EventEmitter {
3645
4070
 
3646
4071
  if (stopBeforeStart) {
3647
4072
  await this.#removeWorker(workersCount, applicationId, oldIndex, worker, silent, oldLabel)
4073
+ replica.worker = null
3648
4074
  }
3649
4075
 
3650
- // Create a new worker with a new index, preserving any pinned port when stopBeforeStart is required.
3651
- newWorker = await this.#setupWorker(configForNewWorker, applicationConfig, workersCount, applicationId, newIndex, false)
4076
+ newWorker = await this.#setupWorker(
4077
+ config,
4078
+ applicationConfig,
4079
+ workersCount,
4080
+ applicationId,
4081
+ newIndex,
4082
+ replica.portOffset,
4083
+ false
4084
+ )
4085
+ replica.replacement = newWorker
3652
4086
 
3653
- // Make sure the runtime hasn't been stopped in the meanwhile
3654
- if (this.#status !== 'started') {
4087
+ // Make sure the runtime hasn't been stopped and the replica hasn't been removed in the meanwhile
4088
+ if (this.#status !== 'started' || signal.aborted) {
3655
4089
  return this.#discardWorker(newWorker)
3656
4090
  }
3657
4091
 
@@ -3662,7 +4096,7 @@ export class Runtime extends EventEmitter {
3662
4096
 
3663
4097
  // Add the worker to the mesh
3664
4098
  await this.#startWorker(
3665
- configForNewWorker,
4099
+ config,
3666
4100
  applicationConfig,
3667
4101
  workersCount,
3668
4102
  applicationId,
@@ -3673,27 +4107,68 @@ export class Runtime extends EventEmitter {
3673
4107
  true
3674
4108
  )
3675
4109
 
3676
- // Make sure the runtime hasn't been stopped in the meanwhile
3677
- if (this.#status !== 'started') {
4110
+ // Make sure the runtime hasn't been stopped and the replica hasn't been removed in the meanwhile
4111
+ if (this.#status !== 'started' || signal.aborted) {
3678
4112
  return this.#discardWorker(newWorker)
3679
4113
  }
3680
4114
  } catch (e) {
3681
- this.#workerPortOffsets.delete(newWorkerId)
4115
+ error = e
4116
+ } finally {
4117
+ replica.pending = null
4118
+ replica.replacement = null
4119
+ }
3682
4120
 
4121
+ if (error) {
3683
4122
  if (this.#workers.get(newWorkerId) === newWorker) {
3684
4123
  this.#workers.delete(newWorkerId)
3685
4124
  }
3686
4125
 
3687
4126
  newWorker?.terminate?.()
3688
- throw e
4127
+
4128
+ // The replica was removed in the meanwhile, the new worker was meant to be discarded anyway
4129
+ if (signal.aborted) {
4130
+ return
4131
+ }
4132
+
4133
+ // The replaced worker crashed (or was stopped before starting the new one): the replica must be refilled
4134
+ if (!replica.worker && this.#getApplicationRestartOnError(config, applicationConfig) > 0) {
4135
+ this.#refillReplica(config, applicationConfig, workersCount, applicationId, replica, true, 0).catch(err => {
4136
+ this.logger.error({ err: ensureLoggableError(err) }, `${oldLabel} could not be restarted.`)
4137
+ this.#detachReplica(applicationId, replica)
4138
+ })
4139
+ }
4140
+
4141
+ throw error
3689
4142
  }
3690
4143
 
4144
+ // The replaced worker might have crashed in the meanwhile, in which case it is not in the replica anymore
4145
+ replica.worker = newWorker
4146
+
3691
4147
  if (!stopBeforeStart) {
4148
+ if (deferOldWorkerRetirement) {
4149
+ return {
4150
+ discard: async () => {
4151
+ // Roll back: the replaced worker serves the replica again
4152
+ if (replica.worker === newWorker) {
4153
+ replica.worker = typeof worker[kWorkerExitCode] === 'number' ? null : worker
4154
+ }
4155
+
4156
+ await this.#discardWorker(newWorker)
4157
+ },
4158
+ retire: () => this.#removeWorker(workersCount, applicationId, oldIndex, worker, silent, oldLabel)
4159
+ }
4160
+ }
4161
+
3692
4162
  await this.#removeWorker(workersCount, applicationId, oldIndex, worker, silent, oldLabel)
3693
4163
  }
3694
4164
  }
3695
4165
 
3696
4166
  async #removeWorker (workersCount, applicationId, index, worker, silent, label) {
4167
+ // The worker crashed in the meanwhile, there is nothing left to remove
4168
+ if (typeof worker[kWorkerExitCode] === 'number') {
4169
+ return
4170
+ }
4171
+
3697
4172
  if (!silent) {
3698
4173
  this.logger.debug(`Preparing to stop the old version of ${label} ...`)
3699
4174
  }
@@ -3759,7 +4234,13 @@ export class Runtime extends EventEmitter {
3759
4234
 
3760
4235
  // Note that in this class "== null" is purposely used instead of "===" to check for both null and undefined
3761
4236
  if (workerId == null) {
3762
- worker = this.#workers.next(applicationId)
4237
+ // During restarts and scaling the application also has workers which are still booting or being stopped:
4238
+ // only pick a worker which can serve the call. Booting workers cannot handle ITC calls yet,
4239
+ // stopping ones might exit before replying.
4240
+ worker = this.#workers.next(applicationId, worker => {
4241
+ const status = worker[kWorkerStatus]
4242
+ return status === 'started' || (!ensureStarted && (status === 'init' || status === 'starting'))
4243
+ })
3763
4244
  } else {
3764
4245
  worker = this.#workers.get(`${applicationId}:${workerId}`)
3765
4246
  }
@@ -3771,6 +4252,11 @@ export class Runtime extends EventEmitter {
3771
4252
  return null
3772
4253
  }
3773
4254
 
4255
+ // The application has workers, but they are all booting or being stopped
4256
+ if (workerId == null && this.#workers.getKeys(applicationId).length > 0) {
4257
+ throw new ApplicationNotStartedError(applicationId)
4258
+ }
4259
+
3774
4260
  if (applicationsIds.includes(applicationId)) {
3775
4261
  const availableWorkers = this.#workers
3776
4262
  .getKeys(applicationId)
@@ -3864,11 +4350,20 @@ export class Runtime extends EventEmitter {
3864
4350
  }
3865
4351
 
3866
4352
  async #getRuntimePackageJson () {
3867
- const runtimeDir = this.#root
3868
- const packageJsonPath = join(runtimeDir, 'package.json')
3869
- const packageJsonFile = await readFile(packageJsonPath, 'utf8')
3870
- const packageJson = JSON.parse(packageJsonFile)
3871
- return packageJson
4353
+ const packageJsonPath = join(this.#root, 'package.json')
4354
+
4355
+ /*
4356
+ A project need not have one, and a runtime that cannot report its metadata without one is a
4357
+ runtime whose `applications:add --save` stops working -- that command reads `projectDir`,
4358
+ `configPath` and `autoload` from this, and has done since `GET /config` was removed. The
4359
+ metadata already declares `packageName` and `packageVersion` as nullable, so absence is a
4360
+ state its shape allows.
4361
+ */
4362
+ try {
4363
+ return JSON.parse(await readFile(packageJsonPath, 'utf8'))
4364
+ } catch {
4365
+ return {}
4366
+ }
3872
4367
  }
3873
4368
 
3874
4369
  #handleWorkerStandardStreams (worker, applicationId, workerId) {
@@ -4217,16 +4712,21 @@ export class Runtime extends EventEmitter {
4217
4712
  await this.#updateApplicationConfigHealth(applicationId, health)
4218
4713
  }
4219
4714
 
4220
- // Get actual worker keys to iterate over existing workers (snapshot to avoid mutation during iteration)
4221
- const workerKeys = [...this.#workers.getKeys(applicationId)]
4222
- for (const workerKey of workerKeys) {
4223
- const workerIndex = parseInt(workerKey.split(':')[1], 10)
4715
+ // A copy: while the loop awaits, scale-downs remove replicas from the list, which would shift the remaining ones
4716
+ // and skip some. Replicas added in the meanwhile (scale-up) already use the new health config.
4717
+ for (const replica of [...(this.#replicas.get(applicationId) ?? [])]) {
4718
+ // The worker crashed, the replica is refilled with a fresh worker which uses the new health config
4719
+ const worker = replica.worker
4720
+ if (!worker) {
4721
+ continue
4722
+ }
4723
+
4724
+ const workerIndex = worker[kWorkerId]
4224
4725
  this.logger.info(
4225
4726
  { health: { current: currentHealth, new: health } },
4226
4727
  `Restarting application "${applicationId}" worker ${workerIndex} to update config health heap...`
4227
4728
  )
4228
4729
 
4229
- const worker = this.#workers.get(workerKey)
4230
4730
  if (health.maxHeapTotal) {
4231
4731
  worker[kConfig].health.maxHeapTotal = health.maxHeapTotal
4232
4732
  }
@@ -4269,24 +4769,51 @@ export class Runtime extends EventEmitter {
4269
4769
 
4270
4770
  if (currentWorkers < workers) {
4271
4771
  report.started = []
4272
- let pendingWorkerId
4772
+
4773
+ let pendingReplica
4273
4774
 
4274
4775
  try {
4275
4776
  for (let i = currentWorkers; i < workers; i++) {
4276
4777
  const newIndex = this.#getNextWorkerIndex(applicationId)
4277
- pendingWorkerId = `${applicationId}:${newIndex}`
4278
- this.#workerPortOffsets.set(pendingWorkerId, i)
4778
+ pendingReplica = await this.#setupReplica(
4779
+ config,
4780
+ applicationConfig,
4781
+ workers,
4782
+ applicationId,
4783
+ newIndex,
4784
+ this.#getNextReplicaPortOffset(applicationId)
4785
+ )
4786
+
4787
+ // The replica was removed in the meanwhile (i.e. scale-down or application stopped)
4788
+ if (!pendingReplica) {
4789
+ continue
4790
+ }
4279
4791
 
4280
- await this.#setupWorker(config, applicationConfig, workers, applicationId, newIndex)
4281
- await this.#startWorker(config, applicationConfig, workers, applicationId, newIndex, false, 0)
4792
+ await this.#startWorker(
4793
+ config,
4794
+ applicationConfig,
4795
+ workers,
4796
+ applicationId,
4797
+ newIndex,
4798
+ false,
4799
+ 0,
4800
+ pendingReplica.worker
4801
+ )
4282
4802
 
4283
- pendingWorkerId = undefined
4803
+ pendingReplica = undefined
4284
4804
  report.started.push(newIndex)
4285
4805
  startedWorkersCount++
4286
4806
  }
4807
+
4808
+ // A worker added after boot re-runs the application's code, so it can answer differently
4809
+ // from the workers already running. Scale-up is the second place the answers must agree.
4810
+ await this.#collectServingState(applicationId)
4287
4811
  report.success = true
4288
4812
  } catch (err) {
4289
- pendingWorkerId && this.#workerPortOffsets.delete(pendingWorkerId)
4813
+ // The worker of the replica could not be started
4814
+ if (pendingReplica && !pendingReplica.worker) {
4815
+ await this.#detachReplica(applicationId, pendingReplica)
4816
+ }
4290
4817
 
4291
4818
  if (startedWorkersCount < 1) {
4292
4819
  this.logger.error({ err }, 'Cannot start application workers, no worker started')
@@ -4303,31 +4830,30 @@ export class Runtime extends EventEmitter {
4303
4830
  report.stopped = []
4304
4831
  try {
4305
4832
  const workersToStop = currentWorkers - workers
4306
- const allInOnePort = this.#config.server?.portAssignment !== 'perWorkerIncrement'
4307
- const workerIdsToStop = this.#workers
4308
- .getKeys(applicationId)
4309
- .map(key => parseInt(key.split(':')[1], 10))
4310
-
4311
- if (allInOnePort) {
4312
- // Stop most recent workers first.
4313
- workerIdsToStop.sort((a, b) => b - a)
4314
- } else {
4315
- // When port increment is enabled, disable the workers with the highest ports.
4316
- workerIdsToStop.sort((a, b) => {
4317
- const offsetA = this.#workerPortOffsets.get(`${applicationId}:${a}`) ?? a
4318
- const offsetB = this.#workerPortOffsets.get(`${applicationId}:${b}`) ?? b
4319
- return offsetB - offsetA
4320
- })
4321
- }
4833
+ // Remove the empty replicas first (their worker crashed and they are waiting to be refilled), then the ones
4834
+ // with the highest port offsets, so that applications using server.portAssignment=perWorkerIncrement keep
4835
+ // listening on a contiguous range of ports starting from the configured one.
4836
+ const removed = (this.#replicas.get(applicationId) ?? [])
4837
+ .toSorted((a, b) => Number(Boolean(a.worker)) - Number(Boolean(b.worker)) || b.portOffset - a.portOffset)
4838
+ .slice(0, workersToStop)
4839
+
4840
+ // Detached right away, so that concurrent operations see the requested workers.
4841
+ // This also cancels the work in progress for them, i.e. refills and replacements.
4842
+ const detached = Promise.all(removed.map(replica => this.#detachReplica(applicationId, replica)))
4843
+
4844
+ for (const replica of removed) {
4845
+ const worker = replica.worker
4846
+ if (worker && typeof worker[kWorkerExitCode] !== 'number') {
4847
+ await sendViaITC(worker, 'removeFromMesh')
4848
+ await this.#stopWorker(currentWorkers, applicationId, worker[kWorkerId], false, worker, [])
4849
+ report.stopped.push(worker[kWorkerId])
4850
+ }
4322
4851
 
4323
- for (const workerIndex of workerIdsToStop.splice(0, workersToStop)) {
4324
- const worker = this.#workers.get(`${applicationId}:${workerIndex}`)
4325
- await sendViaITC(worker, 'removeFromMesh')
4326
- await this.#stopWorker(currentWorkers, applicationId, workerIndex, false, worker, [])
4327
- report.stopped.push(workerIndex)
4328
4852
  stoppedWorkersCount++
4329
4853
  }
4330
4854
 
4855
+ await detached
4856
+
4331
4857
  report.success = true
4332
4858
  } catch (err) {
4333
4859
  if (stoppedWorkersCount < 1) {
@@ -4373,6 +4899,10 @@ export class Runtime extends EventEmitter {
4373
4899
  }
4374
4900
  }
4375
4901
 
4902
+ if (applicationConfig.sourcePath) {
4903
+ allows.add(`--allow-fs-read=${join(applicationConfig.sourcePath, '*')}`)
4904
+ }
4905
+
4376
4906
  if (write?.length) {
4377
4907
  for (const p of write) {
4378
4908
  allows.add(`--allow-fs-write=${isAbsolute(p) ? p : join(applicationConfig.path, p)}`)
@@ -4418,6 +4948,11 @@ export class Runtime extends EventEmitter {
4418
4948
  #processHealthSignals ({ workerId, signals }) {
4419
4949
  const worker = this.#workers.get(workerId)
4420
4950
 
4951
+ // Signals can arrive from a worker which is not registered (anymore)
4952
+ if (!worker) {
4953
+ return
4954
+ }
4955
+
4421
4956
  worker[kWorkerHealthSignals] ??= new HealthSignalsQueue()
4422
4957
  worker[kWorkerHealthSignals].add(signals)
4423
4958
  }
@@ -4493,18 +5028,16 @@ export class Runtime extends EventEmitter {
4493
5028
  }
4494
5029
 
4495
5030
  #isApplicationStarted (id) {
4496
- const applicationConfig = this.#applications.get(id)
4497
- if (!applicationConfig) {
5031
+ if (!this.#applications.has(id)) {
4498
5032
  return false
4499
5033
  }
4500
5034
 
4501
- const workers = applicationConfig.workers.static
4502
- for (let i = 0; i < workers; i++) {
4503
- const worker = this.#workers.get(`${id}:${i}`)
4504
- const status = worker?.[kWorkerStatus]
5035
+ // Workers replaced by restarts get new indexes, so check all the workers of the application.
5036
+ // Anything past boot/init means already started.
5037
+ for (const workerId of this.#workers.getKeys(id)) {
5038
+ const status = this.#workers.get(workerId)[kWorkerStatus]
4505
5039
 
4506
- // Match startApplication(): anything past boot/init means already started.
4507
- if (status && status !== 'boot' && status !== 'init') {
5040
+ if (status !== 'boot' && status !== 'init') {
4508
5041
  return true
4509
5042
  }
4510
5043
  }
@@ -4847,15 +5380,55 @@ export class Runtime extends EventEmitter {
4847
5380
  this.#loggerContext.updatePrefixes(ids)
4848
5381
  }
4849
5382
 
4850
- #onMeshInterceptorError (error) {
4851
- const worker = error.port
5383
+ #getPortOwner (port, applicationId, hostname, includeSameApplication = false) {
5384
+ if (!Number.isInteger(port) || port <= 0) {
5385
+ return null
5386
+ }
4852
5387
 
4853
- this.logger.error(
4854
- { err: ensureLoggableError(error.cause) },
4855
- `The ${this.#workerExtendedLabel(worker[kApplicationId], worker[kWorkerId])} threw an error during mesh network setup. Replacing it ...`
4856
- )
5388
+ for (const worker of this.#workers.values()) {
5389
+ if (!worker[kWorkerUrl] || (!includeSameApplication && worker[kApplicationId] === applicationId)) {
5390
+ continue
5391
+ }
5392
+
5393
+ try {
5394
+ const url = new URL(worker[kWorkerUrl])
5395
+ const workerPort = Number(url.port || (url.protocol === 'https:' ? 443 : 80))
5396
+ const workerHost = url.hostname
5397
+ if (workerPort === port && this.#listenersOverlap(hostname ?? workerHost, workerHost)) {
5398
+ return worker[kApplicationId]
5399
+ }
5400
+ } catch {}
5401
+ }
5402
+
5403
+ return null
5404
+ }
5405
+
5406
+ #listenersOverlap (host, otherHost) {
5407
+ host = host.toLowerCase()
5408
+ const wildcards = new Set(['0.0.0.0', '::', '[::]'])
5409
+ otherHost = otherHost.toLowerCase()
5410
+ return host === otherHost || wildcards.has(host) || wildcards.has(otherHost)
5411
+ }
5412
+
5413
+ #recordWorkerUrl (worker, applicationId, workerUrl) {
5414
+ if (!workerUrl) {
5415
+ worker[kWorkerUrl] = undefined
5416
+ return
5417
+ }
5418
+
5419
+ let hostname
5420
+ let port
5421
+ try {
5422
+ const url = new URL(workerUrl)
5423
+ hostname = url.hostname
5424
+ port = Number(url.port || (url.protocol === 'https:' ? 443 : 80))
5425
+ } catch {}
5426
+
5427
+ const owner = this.#getPortOwner(port, applicationId, hostname)
5428
+ if (owner) {
5429
+ throw new AddressInUseError(port, owner, applicationId)
5430
+ }
4857
5431
 
4858
- this.emit('application:worker:init:failed', { application: worker[kApplicationId], worker: worker[kWorkerId] })
4859
- worker.terminate()
5432
+ worker[kWorkerUrl] = workerUrl
4860
5433
  }
4861
5434
  }