@platformatic/runtime 3.72.0 → 4.0.0-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config.d.ts +602 -673
- package/index.d.ts +31 -28
- package/index.js +290 -30
- package/lib/config.js +270 -367
- package/lib/errors.js +32 -13
- package/lib/generator.js +249 -127
- package/lib/logger.js +3 -3
- package/lib/management-api.js +44 -72
- package/lib/management-handlers.js +3 -5
- package/lib/memory.js +29 -0
- package/lib/policies.js +27 -0
- package/lib/predictive-scaling.js +910 -0
- package/lib/predictive-worker-scaler.js +378 -0
- package/lib/runtime.js +1179 -606
- package/lib/scheduler.js +6 -41
- package/lib/schema.js +77 -15
- package/lib/upgrade.js +16 -1
- package/lib/versions/v4.0.0.js +31 -0
- package/lib/worker/controller.js +89 -109
- package/lib/worker/interceptors.js +34 -34
- package/lib/worker/itc.js +248 -66
- package/lib/worker/main.js +50 -98
- package/lib/worker/messaging.js +42 -1
- package/lib/worker/round-robin-map.js +10 -3
- package/lib/worker/symbols.js +6 -2
- package/package.json +33 -24
- package/schema.json +252 -433
- package/.gitignore +0 -3
- package/.npmignore +0 -2
- package/lib/scaling-algorithm.js +0 -256
- package/lib/worker-scaler.js +0 -249
package/lib/runtime.js
CHANGED
|
@@ -5,7 +5,6 @@ import {
|
|
|
5
5
|
executeInParallel,
|
|
6
6
|
executeWithTimeout,
|
|
7
7
|
features,
|
|
8
|
-
kEnvFileFallbackKeys,
|
|
9
8
|
kMetadata,
|
|
10
9
|
kTimeout,
|
|
11
10
|
parseMemorySize,
|
|
@@ -14,11 +13,13 @@ import {
|
|
|
14
13
|
import { getExecutable } from '@platformatic/globals'
|
|
15
14
|
import { ITC } from '@platformatic/itc'
|
|
16
15
|
import {
|
|
16
|
+
clearRegistry,
|
|
17
17
|
collectProcessMetrics,
|
|
18
18
|
client as metricsClient,
|
|
19
19
|
openTelemetryITCMessage
|
|
20
20
|
} from '@platformatic/metrics'
|
|
21
21
|
import fastify from 'fastify'
|
|
22
|
+
import { randomUUID } from 'node:crypto'
|
|
22
23
|
import { EventEmitter, once } from 'node:events'
|
|
23
24
|
import { existsSync } from 'node:fs'
|
|
24
25
|
import { readFile } from 'node:fs/promises'
|
|
@@ -30,46 +31,50 @@ import { Readable } from 'node:stream'
|
|
|
30
31
|
import { finished } from 'node:stream/promises'
|
|
31
32
|
import { setImmediate as immediate, setTimeout as sleep } from 'node:timers/promises'
|
|
32
33
|
import { pathToFileURL } from 'node:url'
|
|
34
|
+
import { isDeepStrictEqual } from 'node:util'
|
|
33
35
|
import { Worker } from 'node:worker_threads'
|
|
34
36
|
import SonicBoom from 'sonic-boom'
|
|
35
37
|
import { Agent, request, interceptors as undiciInterceptors } from 'undici'
|
|
36
|
-
import {
|
|
38
|
+
import { createCoordinator, createInterceptor } from 'undici-thread-interceptor'
|
|
37
39
|
import { pprofCapturePreloadPath } from './config.js'
|
|
38
40
|
import {
|
|
41
|
+
AddressInUseError,
|
|
39
42
|
ApplicationAlreadyStartedError,
|
|
40
43
|
ApplicationNotFoundError,
|
|
41
44
|
ApplicationNotStartedError,
|
|
42
45
|
ApplicationStartTimeoutError,
|
|
43
|
-
CannotRemoveEntrypointError,
|
|
44
46
|
DuplicateExtensionHealthCheckError,
|
|
45
47
|
DuplicateITCHandlerNameError,
|
|
46
48
|
ExtensionHealthRoutesUnavailableError,
|
|
47
49
|
FailedToLoadExtensionError,
|
|
48
50
|
FailedToStartExtensionError,
|
|
49
51
|
FailedToStopExtensionError,
|
|
52
|
+
FailedToUpdateMetricsConfigError,
|
|
50
53
|
InvalidArgumentError,
|
|
51
54
|
InvalidExtensionError,
|
|
52
55
|
LastProfileTimeoutError,
|
|
53
56
|
MessagingError,
|
|
54
57
|
MetricFamilyCollisionError,
|
|
58
|
+
MetricsDisabledError,
|
|
55
59
|
MissingPprofCapture,
|
|
60
|
+
MixedServingStateError,
|
|
56
61
|
ReservedITCHandlerNameError,
|
|
57
62
|
RuntimeAbortedError,
|
|
58
63
|
RuntimeExtensionBuildAlreadyCalledError,
|
|
59
|
-
|
|
64
|
+
WorkerAddressInUseError,
|
|
60
65
|
WorkerNotFoundError
|
|
61
66
|
} from './errors.js'
|
|
62
67
|
import { abstractLogger, createLogger } from './logger.js'
|
|
63
68
|
import { startManagementApi } from './management-api.js'
|
|
64
69
|
import { createManagementHandlers } from './management-handlers.js'
|
|
65
70
|
import { OpenTelemetryMetricsForwarder } from './opentelemetry-metrics.js'
|
|
66
|
-
import { createChannelCreationHook } from './policies.js'
|
|
71
|
+
import { createChannelCreationHook, createTargetPermissionHook } from './policies.js'
|
|
72
|
+
import { PredictiveWorkersScaler } from './predictive-worker-scaler.js'
|
|
67
73
|
import { startHealthProbesServer, startPrometheusServer } from './prom-server.js'
|
|
68
74
|
import { startScheduler } from './scheduler.js'
|
|
69
75
|
import { createSharedStore } from './shared-http-cache.js'
|
|
70
76
|
import { topologicalLevels, topologicalSort } from './utils.js'
|
|
71
77
|
import { version } from './version.js'
|
|
72
|
-
import { DynamicWorkersScaler } from './worker-scaler.js'
|
|
73
78
|
import { HealthSignalsQueue } from './worker/health-signals.js'
|
|
74
79
|
import { sendMultipleViaITC, sendViaITC, waitEventFromITC } from './worker/itc.js'
|
|
75
80
|
import { RoundRobinMap } from './worker/round-robin-map.js'
|
|
@@ -79,16 +84,21 @@ import {
|
|
|
79
84
|
kFullId,
|
|
80
85
|
kHealthCheckTimer,
|
|
81
86
|
kId,
|
|
82
|
-
kInterceptorReadyPromise,
|
|
83
87
|
kIsSubprocessHost,
|
|
84
88
|
kITC,
|
|
85
89
|
kLastHealthCheckELU,
|
|
86
90
|
kStderrMarker,
|
|
91
|
+
kSubprocessPid,
|
|
92
|
+
kWorkerExitCode,
|
|
87
93
|
kWorkerHealthSignals,
|
|
88
94
|
kWorkerId,
|
|
95
|
+
kWorkerPortOffset,
|
|
89
96
|
kWorkersBroadcast,
|
|
97
|
+
kWorkerServerOptions,
|
|
98
|
+
kWorkerStartAborted,
|
|
90
99
|
kWorkerStartTime,
|
|
91
|
-
kWorkerStatus
|
|
100
|
+
kWorkerStatus,
|
|
101
|
+
kWorkerUrl
|
|
92
102
|
} from './worker/symbols.js'
|
|
93
103
|
|
|
94
104
|
const kWorkerFile = join(import.meta.dirname, 'worker/main.js')
|
|
@@ -118,6 +128,10 @@ const kLastProfileFallbackCodes = new Set([
|
|
|
118
128
|
const kApplicationRestartsMetricName = 'platformatic_application_restarts_total'
|
|
119
129
|
const kApplicationRestartsMetricHelp = 'Total number of restarts triggered by the runtime for an application.'
|
|
120
130
|
|
|
131
|
+
function normalizeCompileCache (value) {
|
|
132
|
+
return typeof value === 'boolean' ? { enabled: value } : value ?? {}
|
|
133
|
+
}
|
|
134
|
+
|
|
121
135
|
const MAX_LISTENERS_COUNT = 100
|
|
122
136
|
|
|
123
137
|
function hasWorkerIndex (applicationId) {
|
|
@@ -174,6 +188,56 @@ const IMMEDIATE_RESTART_MAX_THRESHOLD = 10
|
|
|
174
188
|
const MAX_WORKERS = 100
|
|
175
189
|
const DEFAULT_RESTART_ON_ERROR_DELAY = 5000
|
|
176
190
|
|
|
191
|
+
/*
|
|
192
|
+
Both public payloads are built from a snapshot and frozen through. What a consumer could observe
|
|
193
|
+
could once observe was scalars and a file path, so handing out interior state was harmless in
|
|
194
|
+
practice; the payload now nests resolvedConfig -- an entire capability payload -- inside every
|
|
195
|
+
entry, and the getters read
|
|
196
|
+
straight off live state. A consumer mutating what it received would be editing the configuration
|
|
197
|
+
that later restarts and scale-up workers read, silently, and would make worker generations
|
|
198
|
+
disagree about what they are running. setApplicationConfigPatch exists precisely so that changing
|
|
199
|
+
a running application's configuration is explicit, patch-shaped and visible.
|
|
200
|
+
|
|
201
|
+
Only plain objects and arrays are copied. Anything carrying its own prototype -- a class
|
|
202
|
+
instance, a stream, a buffer -- is handed back as it is, because a copy of it would not be the
|
|
203
|
+
thing; the hazard this closes is a consumer editing configuration, not one editing a socket.
|
|
204
|
+
*/
|
|
205
|
+
function frozenSnapshot (value, seen = new Map()) {
|
|
206
|
+
if (value === null || typeof value !== 'object') {
|
|
207
|
+
return value
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
if (seen.has(value)) {
|
|
211
|
+
return seen.get(value)
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
if (Array.isArray(value)) {
|
|
215
|
+
const copy = []
|
|
216
|
+
seen.set(value, copy)
|
|
217
|
+
|
|
218
|
+
for (const entry of value) {
|
|
219
|
+
copy.push(frozenSnapshot(entry, seen))
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
return Object.freeze(copy)
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
const prototype = Object.getPrototypeOf(value)
|
|
226
|
+
|
|
227
|
+
if (prototype !== Object.prototype && prototype !== null) {
|
|
228
|
+
return value
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
const copy = {}
|
|
232
|
+
seen.set(value, copy)
|
|
233
|
+
|
|
234
|
+
for (const [key, entry] of Object.entries(value)) {
|
|
235
|
+
copy[key] = frozenSnapshot(entry, seen)
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
return Object.freeze(copy)
|
|
239
|
+
}
|
|
240
|
+
|
|
177
241
|
export class Runtime extends EventEmitter {
|
|
178
242
|
logger
|
|
179
243
|
error
|
|
@@ -194,19 +258,22 @@ export class Runtime extends EventEmitter {
|
|
|
194
258
|
#sharedContext
|
|
195
259
|
#isProduction
|
|
196
260
|
#concurrency
|
|
197
|
-
#entrypointId
|
|
198
|
-
#url
|
|
199
|
-
#entrypointPort
|
|
200
261
|
|
|
201
262
|
#healthMetricsTimer
|
|
202
263
|
#healthMetricsCollectionActive
|
|
203
264
|
|
|
204
265
|
#meshInterceptor
|
|
266
|
+
#meshCoordinator
|
|
267
|
+
#meshId
|
|
205
268
|
#dispatcher
|
|
206
269
|
|
|
207
270
|
#managementApi
|
|
208
271
|
#prometheusServer
|
|
209
272
|
#healthProbesServer
|
|
273
|
+
#metricsServersInitialized
|
|
274
|
+
#metricsServersStartPromise
|
|
275
|
+
#metricsConfigUpdatePromise
|
|
276
|
+
#metricsConfigUpdateFailed
|
|
210
277
|
#opentelemetryMetricsForwarder
|
|
211
278
|
#inspectorServer
|
|
212
279
|
#metricsLabelName
|
|
@@ -224,11 +291,11 @@ export class Runtime extends EventEmitter {
|
|
|
224
291
|
#extensionLivenessChecks
|
|
225
292
|
#extensionHealthRoutes
|
|
226
293
|
#lastOverloadProfiles
|
|
294
|
+
#servingStates
|
|
227
295
|
#restartingApplications
|
|
228
|
-
#
|
|
296
|
+
#replicas
|
|
229
297
|
#dynamicWorkersScaler
|
|
230
298
|
#nextWorkerIndex
|
|
231
|
-
#workerPortOffsets
|
|
232
299
|
|
|
233
300
|
#sharedHttpCache
|
|
234
301
|
#scheduler
|
|
@@ -255,22 +322,13 @@ export class Runtime extends EventEmitter {
|
|
|
255
322
|
this.#applications = new Map()
|
|
256
323
|
this.#applicationRestartCounts = new Map()
|
|
257
324
|
this.#workers = new RoundRobinMap()
|
|
258
|
-
this.#url = undefined
|
|
259
|
-
this.#entrypointPort = undefined
|
|
260
325
|
this.#channelCreationHook = createChannelCreationHook(this.#config)
|
|
261
|
-
this.#
|
|
262
|
-
domain: '.plt.local',
|
|
263
|
-
timeout: this.#config.applicationTimeout,
|
|
264
|
-
meshTimeout: this.#context.meshTimeout ?? true,
|
|
265
|
-
onChannelCreation: this.#channelCreationHook,
|
|
266
|
-
onError: this.#onMeshInterceptorError.bind(this)
|
|
267
|
-
})
|
|
326
|
+
this.#meshId = `runtime-${randomUUID()}`
|
|
268
327
|
this.logger = abstractLogger // This is replaced by the real logger in init() and eventually removed in close()
|
|
269
328
|
this.#status = undefined
|
|
270
329
|
this.#restartingApplications = new Set()
|
|
271
|
-
this.#
|
|
330
|
+
this.#replicas = new Map()
|
|
272
331
|
this.#nextWorkerIndex = new Map()
|
|
273
|
-
this.#workerPortOffsets = new Map() // fullWorkerId => portOffset
|
|
274
332
|
this.#sharedHttpCache = null
|
|
275
333
|
this.#applicationsConfigsPatches = new Map()
|
|
276
334
|
|
|
@@ -299,20 +357,29 @@ export class Runtime extends EventEmitter {
|
|
|
299
357
|
// Registered per-worker in #setupWorker, reserved so extensions cannot clobber it
|
|
300
358
|
this.#reservedITCHandlerNames.add('profiling:started')
|
|
301
359
|
this.#extensions = []
|
|
360
|
+
this.#metricsServersInitialized = false
|
|
361
|
+
this.#metricsServersStartPromise = null
|
|
362
|
+
this.#metricsConfigUpdatePromise = Promise.resolve()
|
|
363
|
+
this.#metricsConfigUpdateFailed = false
|
|
364
|
+
this.#processMetricsRegistry = null
|
|
302
365
|
this.#extensionsWantHealthMetrics = false
|
|
303
366
|
this.#extensionReadinessChecks = new Map()
|
|
304
367
|
this.#extensionLivenessChecks = new Map()
|
|
305
368
|
this.#extensionHealthRoutes = []
|
|
306
369
|
this.#lastOverloadProfiles = new Map()
|
|
370
|
+
this.#servingStates = new Map()
|
|
307
371
|
this.#sharedContext = {}
|
|
308
372
|
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
373
|
+
/*
|
|
374
|
+
`PLT_DEV` and `PLT_ENVIRONMENT` are no longer injected here: an application branches on
|
|
375
|
+
its own variables, or the decision moves into the configuration, where the context carries
|
|
376
|
+
`production` and `mode` with types. `NODE_ENV` is the one the runtime still defaults, at the
|
|
377
|
+
bottom of the env ladder, so anything the project sets outranks it.
|
|
378
|
+
|
|
379
|
+
They were worse than redundant by the end -- the worker environment is the one the loader
|
|
380
|
+
resolved per application, and these were written onto the runtime's own copy, so under
|
|
381
|
+
`wattpm start` an application still read `PLT_ENVIRONMENT=development`.
|
|
382
|
+
*/
|
|
316
383
|
}
|
|
317
384
|
|
|
318
385
|
async init () {
|
|
@@ -330,10 +397,9 @@ export class Runtime extends EventEmitter {
|
|
|
330
397
|
this.#metricsLabelName = 'applicationId'
|
|
331
398
|
}
|
|
332
399
|
|
|
333
|
-
// Initialize process-level metrics registry
|
|
400
|
+
// Initialize process-level metrics registry only when metrics are enabled.
|
|
334
401
|
// These metrics are the same across all workers and only need to be collected once
|
|
335
|
-
|
|
336
|
-
if (config.metrics || config.managementApi) {
|
|
402
|
+
if (config.metrics && config.metrics.enabled !== false) {
|
|
337
403
|
this.#processMetricsRegistry = new metricsClient.Registry()
|
|
338
404
|
collectProcessMetrics(this.#processMetricsRegistry)
|
|
339
405
|
}
|
|
@@ -349,18 +415,14 @@ export class Runtime extends EventEmitter {
|
|
|
349
415
|
this.#managementApi = await startManagementApi(this, config.managementApi)
|
|
350
416
|
}
|
|
351
417
|
|
|
352
|
-
|
|
418
|
+
if (config.metrics && config.metrics.enabled !== false) {
|
|
419
|
+
await this.#startOpenTelemetryMetricsForwarder(config.metrics.opentelemetry)
|
|
420
|
+
}
|
|
353
421
|
|
|
354
422
|
this.#createWorkersBroadcastChannel()
|
|
355
423
|
|
|
356
424
|
if (this.#config.workers.dynamic) {
|
|
357
|
-
|
|
358
|
-
this.logger.warn(
|
|
359
|
-
`Worker scaler disabled because the "workers" configuration is set to ${this.#config.workers.static}.`
|
|
360
|
-
)
|
|
361
|
-
} else {
|
|
362
|
-
this.#dynamicWorkersScaler = new DynamicWorkersScaler(this, this.#config.workers)
|
|
363
|
-
}
|
|
425
|
+
this.#dynamicWorkersScaler = new PredictiveWorkersScaler(this, this.#config.workers)
|
|
364
426
|
}
|
|
365
427
|
|
|
366
428
|
// Load extensions before creating any worker so that custom ITC handlers
|
|
@@ -369,18 +431,28 @@ export class Runtime extends EventEmitter {
|
|
|
369
431
|
// readiness/liveness checks and probe routes before Fastify starts listening.
|
|
370
432
|
await this.#loadExtensions()
|
|
371
433
|
|
|
372
|
-
|
|
373
|
-
|
|
434
|
+
// A runtime without applications exits during start and must not briefly
|
|
435
|
+
// claim the default health-probe port. Applications added later, including
|
|
436
|
+
// by extension start hooks, initialize these servers in addApplications().
|
|
437
|
+
if (config.applications.length > 0) {
|
|
438
|
+
await this.#ensureMetricsServersStarted()
|
|
374
439
|
}
|
|
375
440
|
|
|
376
|
-
this.#
|
|
377
|
-
this.#
|
|
441
|
+
this.#meshCoordinator = createCoordinator({ meshId: this.#meshId })
|
|
442
|
+
this.#meshInterceptor = createInterceptor({
|
|
443
|
+
meshId: this.#meshId,
|
|
444
|
+
domain: '.plt.local',
|
|
445
|
+
connectTimeout: this.#config.applicationTimeout,
|
|
446
|
+
bootstrapTimeout: this.#config.applicationTimeout,
|
|
447
|
+
allowTarget: createTargetPermissionHook(this.#config)
|
|
448
|
+
})
|
|
449
|
+
await this.#meshInterceptor.ready
|
|
378
450
|
|
|
379
451
|
await this.addApplications(this.#config.applications)
|
|
380
452
|
await this.#setDispatcher(config.undici)
|
|
381
453
|
|
|
382
454
|
if (!this.#context.build) {
|
|
383
|
-
this.#scheduler = startScheduler(
|
|
455
|
+
this.#scheduler = startScheduler(logger)
|
|
384
456
|
}
|
|
385
457
|
|
|
386
458
|
this.#updateStatus('init')
|
|
@@ -407,7 +479,7 @@ export class Runtime extends EventEmitter {
|
|
|
407
479
|
if (this.getApplicationsIds().length === 0) {
|
|
408
480
|
this.#updateStatus('started')
|
|
409
481
|
await this.close(silent)
|
|
410
|
-
return
|
|
482
|
+
return {}
|
|
411
483
|
}
|
|
412
484
|
|
|
413
485
|
if (this.#config.inspectorOptions) {
|
|
@@ -450,19 +522,13 @@ export class Runtime extends EventEmitter {
|
|
|
450
522
|
|
|
451
523
|
this.#updateStatus('started')
|
|
452
524
|
|
|
453
|
-
// The CLI enables the module compile cache for this process. Node.js would only write it when
|
|
454
|
-
// the process terminates, so flush it now that the boot is complete to not lose it when the
|
|
455
|
-
// process is killed abruptly. This is a no-op when the compile cache is not enabled.
|
|
456
|
-
scheduleCompileCacheFlush()
|
|
457
|
-
|
|
458
525
|
// Start the global health metrics timer for all workers if needed
|
|
459
526
|
this.#startHealthMetricsCollectionIfNeeded()
|
|
460
527
|
|
|
461
528
|
await this.#dynamicWorkersScaler?.start()
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
return this.#url
|
|
529
|
+
this.#showUrls()
|
|
530
|
+
scheduleCompileCacheFlush(this.logger)
|
|
531
|
+
return this.getUrls()
|
|
466
532
|
}
|
|
467
533
|
|
|
468
534
|
async stop (silent = false) {
|
|
@@ -491,18 +557,14 @@ export class Runtime extends EventEmitter {
|
|
|
491
557
|
|
|
492
558
|
await this.#dynamicWorkersScaler?.stop()
|
|
493
559
|
|
|
494
|
-
// Stop the entrypoint first so that no new requests are accepted
|
|
495
|
-
if (this.#entrypointId) {
|
|
496
|
-
await this.stopApplication(this.#entrypointId, silent)
|
|
497
|
-
}
|
|
498
|
-
|
|
499
560
|
// Await extension stop hooks before stopping remaining applications so that
|
|
500
561
|
// control-plane extensions can settle work and hand off state first.
|
|
501
562
|
await this.#stopExtensions()
|
|
502
563
|
|
|
503
564
|
await this.stopApplications(this.getApplicationsIds(), silent)
|
|
504
565
|
|
|
505
|
-
await this.#meshInterceptor
|
|
566
|
+
await this.#meshInterceptor?.close()
|
|
567
|
+
this.#meshCoordinator?.destroy()
|
|
506
568
|
this.#workersBroadcastChannel?.close()
|
|
507
569
|
|
|
508
570
|
this.#updateStatus('stopped')
|
|
@@ -520,8 +582,6 @@ export class Runtime extends EventEmitter {
|
|
|
520
582
|
await this.restartApplications(toRestart)
|
|
521
583
|
|
|
522
584
|
this.emitAndNotify('restarted')
|
|
523
|
-
|
|
524
|
-
return this.#url
|
|
525
585
|
}
|
|
526
586
|
|
|
527
587
|
async close (silent = false) {
|
|
@@ -537,6 +597,7 @@ export class Runtime extends EventEmitter {
|
|
|
537
597
|
clearTimeout(this.#healthMetricsTimer)
|
|
538
598
|
this.#healthMetricsCollectionActive = false
|
|
539
599
|
this.#lastOverloadProfiles.clear()
|
|
600
|
+
this.#servingStates.clear()
|
|
540
601
|
|
|
541
602
|
await this.stop(silent)
|
|
542
603
|
this.#updateStatus('closing')
|
|
@@ -561,7 +622,7 @@ export class Runtime extends EventEmitter {
|
|
|
561
622
|
|
|
562
623
|
// Clean up process metrics registry
|
|
563
624
|
if (this.#processMetricsRegistry) {
|
|
564
|
-
this.#processMetricsRegistry
|
|
625
|
+
clearRegistry(this.#processMetricsRegistry)
|
|
565
626
|
this.#processMetricsRegistry = null
|
|
566
627
|
}
|
|
567
628
|
|
|
@@ -673,25 +734,8 @@ export class Runtime extends EventEmitter {
|
|
|
673
734
|
|
|
674
735
|
async addApplications (applications, start = false) {
|
|
675
736
|
const setupInvocations = []
|
|
676
|
-
// Per-worker ports do not need SO_REUSEPORT because each worker binds a different port.
|
|
677
|
-
const usesPerWorkerPorts = this.#config.server?.portAssignment === 'perWorkerIncrement'
|
|
678
|
-
|
|
679
737
|
const toStart = []
|
|
680
738
|
for (const application of applications) {
|
|
681
|
-
const workers = application.workers
|
|
682
|
-
|
|
683
|
-
if (
|
|
684
|
-
(workers.static > 1 || workers.minimum > 1) &&
|
|
685
|
-
application.entrypoint &&
|
|
686
|
-
!features.node.reusePort &&
|
|
687
|
-
!usesPerWorkerPorts
|
|
688
|
-
) {
|
|
689
|
-
this.logger.warn(
|
|
690
|
-
`"${application.id}" is set as the entrypoint, but reusePort is not available in your OS; setting workers to 1 instead of ${workers.static}`
|
|
691
|
-
)
|
|
692
|
-
application.workers = { dynamic: false, static: 1 }
|
|
693
|
-
}
|
|
694
|
-
|
|
695
739
|
this.#applications.set(application.id, application)
|
|
696
740
|
this.#applicationRestartCounts.set(application.id, this.#applicationRestartCounts.get(application.id) ?? 0)
|
|
697
741
|
setupInvocations.push([application])
|
|
@@ -700,8 +744,12 @@ export class Runtime extends EventEmitter {
|
|
|
700
744
|
|
|
701
745
|
await executeInParallel(this.#setupApplication.bind(this), setupInvocations, this.#concurrency)
|
|
702
746
|
|
|
747
|
+
if (applications.length > 0) {
|
|
748
|
+
await this.#ensureMetricsServersStarted()
|
|
749
|
+
}
|
|
750
|
+
|
|
703
751
|
for (const application of applications) {
|
|
704
|
-
this.logger.debug(`Added application "${application.id}"
|
|
752
|
+
this.logger.debug(`Added application "${application.id}".`)
|
|
705
753
|
this.emitAndNotify('application:added', application)
|
|
706
754
|
}
|
|
707
755
|
|
|
@@ -711,7 +759,7 @@ export class Runtime extends EventEmitter {
|
|
|
711
759
|
|
|
712
760
|
const created = []
|
|
713
761
|
for (const { id } of applications) {
|
|
714
|
-
created.push(await this
|
|
762
|
+
created.push(await this.#buildApplicationDetails(id))
|
|
715
763
|
}
|
|
716
764
|
|
|
717
765
|
this.#updateLoggingPrefixes()
|
|
@@ -719,10 +767,6 @@ export class Runtime extends EventEmitter {
|
|
|
719
767
|
}
|
|
720
768
|
|
|
721
769
|
async removeApplications (applications, silent = false) {
|
|
722
|
-
if (applications.includes(this.#entrypointId)) {
|
|
723
|
-
throw new CannotRemoveEntrypointError()
|
|
724
|
-
}
|
|
725
|
-
|
|
726
770
|
const removed = []
|
|
727
771
|
for (const application of applications) {
|
|
728
772
|
if (!this.#applications.has(application)) {
|
|
@@ -731,18 +775,23 @@ export class Runtime extends EventEmitter {
|
|
|
731
775
|
|
|
732
776
|
// Use allowUnloaded so that applications without a live worker
|
|
733
777
|
// (stopped or crashed with restartOnError: 0) can still be removed.
|
|
734
|
-
const details = await this
|
|
778
|
+
const details = await this.#buildApplicationDetails(application, true)
|
|
735
779
|
details.status = 'removed'
|
|
780
|
+
|
|
781
|
+
// The snapshot is taken while the application is still up, but what it reports is an
|
|
782
|
+
// application about to be gone: it is not making a claim about how it would serve.
|
|
783
|
+
delete details.servingState
|
|
784
|
+
|
|
736
785
|
removed.push(details)
|
|
737
786
|
}
|
|
738
787
|
|
|
739
788
|
await this.stopApplications(applications, silent, true)
|
|
740
789
|
|
|
741
790
|
for (const application of applications) {
|
|
742
|
-
this.#dynamicWorkersScaler?.remove(application)
|
|
743
791
|
await this.#scheduler?.removeApplicationJobs(application)
|
|
744
792
|
this.#applications.delete(application)
|
|
745
793
|
this.#applicationRestartCounts.delete(application)
|
|
794
|
+
this.#replicas.delete(application)
|
|
746
795
|
}
|
|
747
796
|
|
|
748
797
|
for (const application of applications) {
|
|
@@ -773,8 +822,24 @@ export class Runtime extends EventEmitter {
|
|
|
773
822
|
const levels = topologicalLevels(applications, dependencies)
|
|
774
823
|
|
|
775
824
|
for (const level of levels) {
|
|
776
|
-
const
|
|
777
|
-
|
|
825
|
+
const applicationsWithPort = await Promise.all(
|
|
826
|
+
level.map(async applicationId => {
|
|
827
|
+
const worker = await this.#getWorkerByIdOrNext(applicationId)
|
|
828
|
+
const applicationConfig = await sendViaITC(worker, 'getApplicationConfig')
|
|
829
|
+
const port = Number(applicationConfig?.server?.port)
|
|
830
|
+
return { applicationId, hasPort: Number.isInteger(port) && port > 0 }
|
|
831
|
+
})
|
|
832
|
+
)
|
|
833
|
+
|
|
834
|
+
for (const hasPort of [true, false]) {
|
|
835
|
+
const startInvocations = applicationsWithPort
|
|
836
|
+
.filter(application => application.hasPort === hasPort)
|
|
837
|
+
.map(({ applicationId }) => [applicationId, silent])
|
|
838
|
+
|
|
839
|
+
if (startInvocations.length > 0) {
|
|
840
|
+
await executeInParallel(this.startApplication.bind(this), startInvocations, this.#concurrency)
|
|
841
|
+
}
|
|
842
|
+
}
|
|
778
843
|
}
|
|
779
844
|
}
|
|
780
845
|
|
|
@@ -786,16 +851,21 @@ export class Runtime extends EventEmitter {
|
|
|
786
851
|
|
|
787
852
|
if (!skipDependencies) {
|
|
788
853
|
try {
|
|
789
|
-
const
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
854
|
+
const details = await executeWithTimeout(
|
|
855
|
+
this.getApplications(true),
|
|
856
|
+
this.#config.gracefulShutdown.application
|
|
857
|
+
)
|
|
858
|
+
if (details !== kTimeout) {
|
|
859
|
+
for (const application of details.applications) {
|
|
860
|
+
for (const dependency of application.dependencies ?? []) {
|
|
861
|
+
let applicationDependents = dependents[dependency]
|
|
862
|
+
if (!applicationDependents) {
|
|
863
|
+
applicationDependents = new Set()
|
|
864
|
+
dependents[dependency] = applicationDependents
|
|
865
|
+
}
|
|
797
866
|
|
|
798
|
-
|
|
867
|
+
applicationDependents.add(application.id)
|
|
868
|
+
}
|
|
799
869
|
}
|
|
800
870
|
}
|
|
801
871
|
} catch (e) {
|
|
@@ -804,11 +874,6 @@ export class Runtime extends EventEmitter {
|
|
|
804
874
|
}
|
|
805
875
|
|
|
806
876
|
for (const application of applicationsToStop) {
|
|
807
|
-
// The entrypoint has been stopped above
|
|
808
|
-
if (application === this.#entrypointId) {
|
|
809
|
-
continue
|
|
810
|
-
}
|
|
811
|
-
|
|
812
877
|
stopInvocations.push([application, silent, Array.from(dependents[application] ?? [])])
|
|
813
878
|
}
|
|
814
879
|
|
|
@@ -816,13 +881,39 @@ export class Runtime extends EventEmitter {
|
|
|
816
881
|
}
|
|
817
882
|
|
|
818
883
|
async restartApplications (applicationsToRestart) {
|
|
819
|
-
const restartInvocations = []
|
|
884
|
+
const restartInvocations = applicationsToRestart.map(application => [application, true])
|
|
885
|
+
const restarts = await executeInParallel(
|
|
886
|
+
this.restartApplication.bind(this),
|
|
887
|
+
restartInvocations,
|
|
888
|
+
this.#concurrency,
|
|
889
|
+
false
|
|
890
|
+
)
|
|
891
|
+
const failed = restarts.filter(result => result instanceof Error)
|
|
892
|
+
const succeeded = restarts.filter(result => result && !(result instanceof Error))
|
|
893
|
+
|
|
894
|
+
if (failed.length > 0) {
|
|
895
|
+
await Promise.allSettled(succeeded.map(restart => restart.discard()))
|
|
896
|
+
throw failed[0]
|
|
897
|
+
}
|
|
820
898
|
|
|
821
|
-
|
|
822
|
-
|
|
899
|
+
const results = await executeInParallel(
|
|
900
|
+
retirement => retirement.retire(),
|
|
901
|
+
succeeded.flatMap(({ retirements }) => retirements.map(retirement => [retirement])),
|
|
902
|
+
this.#concurrency,
|
|
903
|
+
false
|
|
904
|
+
)
|
|
905
|
+
|
|
906
|
+
const error = results.find(result => result instanceof Error)
|
|
907
|
+
if (error) {
|
|
908
|
+
for (const restart of succeeded) {
|
|
909
|
+
restart.release()
|
|
910
|
+
}
|
|
911
|
+
throw error
|
|
823
912
|
}
|
|
824
913
|
|
|
825
|
-
|
|
914
|
+
for (const restart of succeeded) {
|
|
915
|
+
await restart.complete()
|
|
916
|
+
}
|
|
826
917
|
}
|
|
827
918
|
|
|
828
919
|
async startApplication (id, silent = false) {
|
|
@@ -833,26 +924,33 @@ export class Runtime extends EventEmitter {
|
|
|
833
924
|
throw new ApplicationNotFoundError(id, this.getApplicationsIds().join(', '))
|
|
834
925
|
}
|
|
835
926
|
|
|
927
|
+
if (this.#isApplicationStarted(id)) {
|
|
928
|
+
throw new ApplicationAlreadyStartedError()
|
|
929
|
+
}
|
|
930
|
+
|
|
836
931
|
const workers = applicationConfig.workers.static
|
|
837
|
-
|
|
838
|
-
const worker = this.#workers.get(`${id}:${i}`)
|
|
839
|
-
const status = worker?.[kWorkerStatus]
|
|
932
|
+
this.emitAndNotify('application:starting', id)
|
|
840
933
|
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
934
|
+
const compileCache = {
|
|
935
|
+
...normalizeCompileCache(config.compileCache),
|
|
936
|
+
...normalizeCompileCache(applicationConfig.compileCache)
|
|
844
937
|
}
|
|
938
|
+
const awaitFirstWorker =
|
|
939
|
+
typeof compileCache === 'object' && compileCache.enabled !== false && compileCache.awaitFirstWorker === true
|
|
845
940
|
|
|
846
|
-
|
|
941
|
+
if (awaitFirstWorker && workers > 0) {
|
|
942
|
+
await this.#startFirstWorkerWithCompileCache(config, applicationConfig, workers, id, silent)
|
|
943
|
+
}
|
|
847
944
|
|
|
848
|
-
|
|
945
|
+
// Keep the first worker separate so compile-cache startup can be made a barrier without changing this loop later.
|
|
946
|
+
for (let i = awaitFirstWorker ? 1 : 0; i < workers; i++) {
|
|
849
947
|
await this.#startWorker(config, applicationConfig, workers, id, i, silent)
|
|
850
948
|
}
|
|
851
949
|
|
|
852
950
|
await this.#registerApplicationSchedulerJobs(id)
|
|
951
|
+
await this.#collectServingState(id)
|
|
853
952
|
|
|
854
953
|
this.emitAndNotify('application:started', id)
|
|
855
|
-
await this.#dynamicWorkersScaler?.applyPendingUpdate(id)
|
|
856
954
|
}
|
|
857
955
|
|
|
858
956
|
async stopApplication (id, silent = false, dependents = []) {
|
|
@@ -860,7 +958,17 @@ export class Runtime extends EventEmitter {
|
|
|
860
958
|
throw new ApplicationNotFoundError(id, this.getApplicationsIds().join(', '))
|
|
861
959
|
}
|
|
862
960
|
|
|
961
|
+
this.#dynamicWorkersScaler?.remove(id)
|
|
962
|
+
|
|
963
|
+
// Removing the replicas cancels the work in progress for them, i.e. refills of crashed workers and replacements
|
|
964
|
+
const replicas = this.#replicas.get(id) ?? []
|
|
965
|
+
this.#replicas.set(id, [])
|
|
966
|
+
const detached = Promise.all(replicas.map(replica => this.#detachReplica(id, replica)))
|
|
967
|
+
|
|
968
|
+
// Workers being discarded by the detach above are already marked as exited
|
|
863
969
|
const workersIds = this.#workers.getKeys(id)
|
|
970
|
+
.filter(workerId => this.#workers.get(workerId)[kWorkerStatus] !== 'exited')
|
|
971
|
+
|
|
864
972
|
const workersCount = workersIds.length
|
|
865
973
|
|
|
866
974
|
this.emitAndNotify('application:stopping', id)
|
|
@@ -876,10 +984,17 @@ export class Runtime extends EventEmitter {
|
|
|
876
984
|
await executeInParallel(this.#stopWorker.bind(this), stopInvocations, this.#concurrency)
|
|
877
985
|
}
|
|
878
986
|
|
|
987
|
+
await detached
|
|
988
|
+
|
|
989
|
+
// Absent rather than 'inactive': a stopped application is not making a claim about how it
|
|
990
|
+
// would serve, and conflating "not started" with "started and serving nothing" is the exact
|
|
991
|
+
// distinction this field exists to draw.
|
|
992
|
+
this.#servingStates.delete(id)
|
|
993
|
+
|
|
879
994
|
this.emitAndNotify('application:stopped', id)
|
|
880
995
|
}
|
|
881
996
|
|
|
882
|
-
async restartApplication (id) {
|
|
997
|
+
async restartApplication (id, deferOldWorkerRetirement = false) {
|
|
883
998
|
const applicationConfig = this.#applications.get(id)
|
|
884
999
|
|
|
885
1000
|
if (!applicationConfig) {
|
|
@@ -898,31 +1013,75 @@ export class Runtime extends EventEmitter {
|
|
|
898
1013
|
}
|
|
899
1014
|
|
|
900
1015
|
this.#restartingApplications.add(id)
|
|
1016
|
+
const retirements = []
|
|
1017
|
+
let deferred = false
|
|
901
1018
|
|
|
902
1019
|
try {
|
|
903
1020
|
const config = this.#config
|
|
904
|
-
|
|
905
|
-
|
|
1021
|
+
// A copy: while the loop awaits, scale-downs remove replicas from the list, which would shift the remaining ones
|
|
1022
|
+
// and skip some. Replicas added in the meanwhile (scale-up) already run fresh workers.
|
|
1023
|
+
const replicas = [...(this.#replicas.get(id) ?? [])]
|
|
1024
|
+
const replicasCount = replicas.length
|
|
906
1025
|
|
|
907
1026
|
this.emitAndNotify('application:restarting', id)
|
|
908
1027
|
|
|
909
|
-
for (let i = 0; i <
|
|
910
|
-
const
|
|
911
|
-
const worker = this.#workers.get(workerId)
|
|
912
|
-
const workerIndex = parseInt(workerId.split(':')[1], 10)
|
|
1028
|
+
for (let i = 0; i < replicasCount; i++) {
|
|
1029
|
+
const replica = replicas[i]
|
|
913
1030
|
|
|
914
1031
|
if (i > 0 && config.workersRestartDelay > 0) {
|
|
915
1032
|
await sleep(config.workersRestartDelay)
|
|
916
1033
|
}
|
|
917
1034
|
|
|
918
|
-
|
|
1035
|
+
// The replica might have been removed while the previous ones were being replaced,
|
|
1036
|
+
// or its worker crashed and the replica is being refilled with a fresh worker.
|
|
1037
|
+
const worker = replica.worker
|
|
1038
|
+
if (replica.abortController.signal.aborted || !worker) {
|
|
1039
|
+
continue
|
|
1040
|
+
}
|
|
1041
|
+
|
|
1042
|
+
const retirement = await this.#replaceWorker(
|
|
1043
|
+
config,
|
|
1044
|
+
applicationConfig,
|
|
1045
|
+
replicasCount,
|
|
1046
|
+
id,
|
|
1047
|
+
worker[kWorkerId],
|
|
1048
|
+
worker,
|
|
1049
|
+
true,
|
|
1050
|
+
deferOldWorkerRetirement
|
|
1051
|
+
)
|
|
1052
|
+
if (retirement) {
|
|
1053
|
+
retirements.push(retirement)
|
|
1054
|
+
}
|
|
1055
|
+
}
|
|
1056
|
+
|
|
1057
|
+
if (deferOldWorkerRetirement) {
|
|
1058
|
+
deferred = true
|
|
1059
|
+
return {
|
|
1060
|
+
retirements,
|
|
1061
|
+
release: () => this.#restartingApplications.delete(id),
|
|
1062
|
+
discard: async () => {
|
|
1063
|
+
await Promise.allSettled(retirements.map(retirement => retirement.discard()))
|
|
1064
|
+
this.#restartingApplications.delete(id)
|
|
1065
|
+
},
|
|
1066
|
+
complete: async () => {
|
|
1067
|
+
try {
|
|
1068
|
+
await this.#registerApplicationSchedulerJobs(id)
|
|
1069
|
+
this.#incrementApplicationRestartCount(id)
|
|
1070
|
+
this.emitAndNotify('application:restarted', id)
|
|
1071
|
+
} finally {
|
|
1072
|
+
this.#restartingApplications.delete(id)
|
|
1073
|
+
}
|
|
1074
|
+
}
|
|
1075
|
+
}
|
|
919
1076
|
}
|
|
920
1077
|
|
|
921
1078
|
await this.#registerApplicationSchedulerJobs(id)
|
|
922
1079
|
this.#incrementApplicationRestartCount(id)
|
|
923
1080
|
this.emitAndNotify('application:restarted', id)
|
|
924
1081
|
} finally {
|
|
925
|
-
|
|
1082
|
+
if (!deferred) {
|
|
1083
|
+
this.#restartingApplications.delete(id)
|
|
1084
|
+
}
|
|
926
1085
|
}
|
|
927
1086
|
}
|
|
928
1087
|
|
|
@@ -994,13 +1153,7 @@ export class Runtime extends EventEmitter {
|
|
|
994
1153
|
|
|
995
1154
|
if (!allWorkers || hasWorkerIndex(id)) {
|
|
996
1155
|
const service = await this.#getApplicationWorkerForProfiling(id, ensureStarted)
|
|
997
|
-
|
|
998
|
-
service[kConfig].nodeModulesSourceMaps ?? this.#config.nodeModulesSourceMaps
|
|
999
|
-
|
|
1000
|
-
return sendViaITC(service, 'startProfiling', {
|
|
1001
|
-
...profilingOptions,
|
|
1002
|
-
nodeModulesSourceMaps
|
|
1003
|
-
})
|
|
1156
|
+
return sendViaITC(service, 'startProfiling', profilingOptions)
|
|
1004
1157
|
}
|
|
1005
1158
|
|
|
1006
1159
|
const started = []
|
|
@@ -1009,14 +1162,7 @@ export class Runtime extends EventEmitter {
|
|
|
1009
1162
|
|
|
1010
1163
|
for (const { workerIndex, worker } of await this.#getApplicationWorkersForProfiling(id, ensureStarted)) {
|
|
1011
1164
|
try {
|
|
1012
|
-
|
|
1013
|
-
worker[kConfig].nodeModulesSourceMaps ??
|
|
1014
|
-
this.#config.nodeModulesSourceMaps
|
|
1015
|
-
|
|
1016
|
-
await sendViaITC(worker, 'startProfiling', {
|
|
1017
|
-
...profilingOptions,
|
|
1018
|
-
nodeModulesSourceMaps
|
|
1019
|
-
})
|
|
1165
|
+
await sendViaITC(worker, 'startProfiling', profilingOptions)
|
|
1020
1166
|
started.push(workerIndex)
|
|
1021
1167
|
} catch (error) {
|
|
1022
1168
|
// A worker which is already being profiled is considered covered, but
|
|
@@ -1271,7 +1417,8 @@ export class Runtime extends EventEmitter {
|
|
|
1271
1417
|
}
|
|
1272
1418
|
|
|
1273
1419
|
/**
|
|
1274
|
-
*
|
|
1420
|
+
* Merges top-level metrics options without restarting the runtime or workers.
|
|
1421
|
+
* Nested objects are replaced when supplied. Identical successful updates are no-ops.
|
|
1275
1422
|
*
|
|
1276
1423
|
* This method allows you to:
|
|
1277
1424
|
* - Enable or disable metrics collection
|
|
@@ -1290,52 +1437,138 @@ export class Runtime extends EventEmitter {
|
|
|
1290
1437
|
* await runtime.updateMetricsConfig({ enabled: false })
|
|
1291
1438
|
*/
|
|
1292
1439
|
async updateMetricsConfig (metricsConfig) {
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1440
|
+
// Serialize changes because they share listeners and worker registries.
|
|
1441
|
+
const previousUpdate = this.#metricsConfigUpdatePromise
|
|
1442
|
+
const { promise, resolve } = Promise.withResolvers()
|
|
1443
|
+
this.#metricsConfigUpdatePromise = promise
|
|
1444
|
+
await previousUpdate
|
|
1297
1445
|
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1446
|
+
try {
|
|
1447
|
+
const currentConfig = this.#config.metrics
|
|
1448
|
+
const previousConfig = typeof currentConfig === 'boolean' ? { enabled: currentConfig } : (currentConfig ?? {})
|
|
1449
|
+
metricsConfig = { ...previousConfig, ...metricsConfig }
|
|
1302
1450
|
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1451
|
+
// An omitted enabled option already means enabled for an object configuration.
|
|
1452
|
+
const previousOptions = { ...previousConfig, enabled: previousConfig.enabled !== false }
|
|
1453
|
+
const nextOptions = { ...metricsConfig, enabled: metricsConfig.enabled !== false }
|
|
1454
|
+
if (!this.#metricsConfigUpdateFailed && isDeepStrictEqual(previousOptions, nextOptions)) {
|
|
1455
|
+
this.#config.metrics = metricsConfig
|
|
1456
|
+
return { success: true, config: structuredClone(metricsConfig) }
|
|
1457
|
+
}
|
|
1307
1458
|
|
|
1308
|
-
|
|
1309
|
-
|
|
1459
|
+
// Failed updates can be retried even when the main-thread configuration already changed.
|
|
1460
|
+
this.#metricsConfigUpdateFailed = true
|
|
1310
1461
|
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
}
|
|
1462
|
+
if (this.#metricsServersStartPromise) {
|
|
1463
|
+
await this.#metricsServersStartPromise
|
|
1464
|
+
}
|
|
1315
1465
|
|
|
1316
|
-
|
|
1466
|
+
if (this.#prometheusServer) {
|
|
1467
|
+
await this.#prometheusServer.close()
|
|
1468
|
+
this.#prometheusServer = null
|
|
1469
|
+
}
|
|
1317
1470
|
|
|
1318
|
-
|
|
1319
|
-
|
|
1471
|
+
if (this.#healthProbesServer) {
|
|
1472
|
+
await this.#healthProbesServer.close()
|
|
1473
|
+
this.#healthProbesServer = null
|
|
1474
|
+
}
|
|
1320
1475
|
|
|
1321
|
-
|
|
1476
|
+
this.#metricsServersInitialized = false
|
|
1322
1477
|
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
promises.push(sendViaITC(worker, 'updateMetricsConfig', metricsConfig))
|
|
1478
|
+
if (this.#opentelemetryMetricsForwarder) {
|
|
1479
|
+
await this.#opentelemetryMetricsForwarder.close()
|
|
1480
|
+
this.#opentelemetryMetricsForwarder = null
|
|
1327
1481
|
}
|
|
1328
|
-
}
|
|
1329
1482
|
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1483
|
+
this.#config.metrics = metricsConfig
|
|
1484
|
+
this.#metricsLabelName = metricsConfig?.applicationLabel || 'applicationId'
|
|
1485
|
+
|
|
1486
|
+
if (metricsConfig.enabled === false) {
|
|
1487
|
+
if (this.#processMetricsRegistry) {
|
|
1488
|
+
clearRegistry(this.#processMetricsRegistry)
|
|
1489
|
+
this.#processMetricsRegistry = null
|
|
1490
|
+
}
|
|
1491
|
+
} else if (!this.#processMetricsRegistry) {
|
|
1492
|
+
this.#processMetricsRegistry = new metricsClient.Registry()
|
|
1493
|
+
collectProcessMetrics(this.#processMetricsRegistry)
|
|
1334
1494
|
}
|
|
1495
|
+
|
|
1496
|
+
// Allow extension health routes to be re-applied on the restarted servers.
|
|
1497
|
+
for (const entry of this.#extensionHealthRoutes) {
|
|
1498
|
+
entry.applied = false
|
|
1499
|
+
}
|
|
1500
|
+
|
|
1501
|
+
await this.#ensureMetricsServersStarted()
|
|
1502
|
+
|
|
1503
|
+
if (metricsConfig.enabled !== false) {
|
|
1504
|
+
await this.#startOpenTelemetryMetricsForwarder(metricsConfig.opentelemetry)
|
|
1505
|
+
}
|
|
1506
|
+
|
|
1507
|
+
const promises = []
|
|
1508
|
+
for (const worker of this.#workers.values()) {
|
|
1509
|
+
if (worker[kWorkerStatus] === 'started') {
|
|
1510
|
+
promises.push(
|
|
1511
|
+
sendViaITC(worker, 'updateMetricsConfig', metricsConfig).catch(err => {
|
|
1512
|
+
throw new FailedToUpdateMetricsConfigError(worker[kFullId], err.message)
|
|
1513
|
+
})
|
|
1514
|
+
)
|
|
1515
|
+
}
|
|
1516
|
+
}
|
|
1517
|
+
|
|
1518
|
+
const results = await Promise.allSettled(promises)
|
|
1519
|
+
for (const result of results) {
|
|
1520
|
+
if (result.status === 'rejected') {
|
|
1521
|
+
this.logger.error({ err: result.reason }, 'Cannot update metrics config on worker')
|
|
1522
|
+
}
|
|
1523
|
+
}
|
|
1524
|
+
|
|
1525
|
+
const failure = results.find(result => result.status === 'rejected')
|
|
1526
|
+
if (failure) {
|
|
1527
|
+
throw failure.reason
|
|
1528
|
+
}
|
|
1529
|
+
|
|
1530
|
+
this.#metricsConfigUpdateFailed = false
|
|
1531
|
+
this.logger.info('Metrics configuration updated')
|
|
1532
|
+
return { success: true, config: structuredClone(metricsConfig) }
|
|
1533
|
+
} finally {
|
|
1534
|
+
// Failed and no-op updates must also release the next queued request.
|
|
1535
|
+
resolve()
|
|
1536
|
+
}
|
|
1537
|
+
}
|
|
1538
|
+
|
|
1539
|
+
async #ensureMetricsServersStarted () {
|
|
1540
|
+
if (this.#metricsServersInitialized) {
|
|
1541
|
+
return
|
|
1335
1542
|
}
|
|
1336
1543
|
|
|
1337
|
-
this
|
|
1338
|
-
|
|
1544
|
+
if (this.#metricsServersStartPromise) {
|
|
1545
|
+
return this.#metricsServersStartPromise
|
|
1546
|
+
}
|
|
1547
|
+
|
|
1548
|
+
this.#metricsServersStartPromise = (async () => {
|
|
1549
|
+
const config = this.#config
|
|
1550
|
+
|
|
1551
|
+
try {
|
|
1552
|
+
if (config.metrics || (typeof config.healthProbes === 'object' && config.healthProbes !== null)) {
|
|
1553
|
+
this.#prometheusServer = await startPrometheusServer(this, config.metrics ?? false, config.healthProbes)
|
|
1554
|
+
}
|
|
1555
|
+
|
|
1556
|
+
this.#healthProbesServer = await startHealthProbesServer(this, config.metrics, config.healthProbes)
|
|
1557
|
+
this.#assertExtensionHealthRoutesApplied()
|
|
1558
|
+
this.#metricsServersInitialized = true
|
|
1559
|
+
} catch (err) {
|
|
1560
|
+
await Promise.allSettled([this.#prometheusServer?.close(), this.#healthProbesServer?.close()])
|
|
1561
|
+
this.#prometheusServer = null
|
|
1562
|
+
this.#healthProbesServer = null
|
|
1563
|
+
throw err
|
|
1564
|
+
}
|
|
1565
|
+
})()
|
|
1566
|
+
|
|
1567
|
+
try {
|
|
1568
|
+
await this.#metricsServersStartPromise
|
|
1569
|
+
} finally {
|
|
1570
|
+
this.#metricsServersStartPromise = null
|
|
1571
|
+
}
|
|
1339
1572
|
}
|
|
1340
1573
|
|
|
1341
1574
|
async #startOpenTelemetryMetricsForwarder (config) {
|
|
@@ -1490,7 +1723,7 @@ export class Runtime extends EventEmitter {
|
|
|
1490
1723
|
const { applicationId, config: applicationConfig, workers, health, currentWorkers, currentHealth } = update
|
|
1491
1724
|
|
|
1492
1725
|
if (workers && health) {
|
|
1493
|
-
const
|
|
1726
|
+
const resourceReport = await this.#updateApplicationWorkersAndHealth(
|
|
1494
1727
|
applicationId,
|
|
1495
1728
|
config,
|
|
1496
1729
|
applicationConfig,
|
|
@@ -1501,11 +1734,11 @@ export class Runtime extends EventEmitter {
|
|
|
1501
1734
|
)
|
|
1502
1735
|
report.push({
|
|
1503
1736
|
application: applicationId,
|
|
1504
|
-
workers:
|
|
1505
|
-
health:
|
|
1737
|
+
workers: resourceReport.workers,
|
|
1738
|
+
health: resourceReport.health
|
|
1506
1739
|
})
|
|
1507
1740
|
} else if (health) {
|
|
1508
|
-
const
|
|
1741
|
+
const healthReport = await this.#updateApplicationHealth(
|
|
1509
1742
|
applicationId,
|
|
1510
1743
|
config,
|
|
1511
1744
|
applicationConfig,
|
|
@@ -1515,10 +1748,10 @@ export class Runtime extends EventEmitter {
|
|
|
1515
1748
|
)
|
|
1516
1749
|
report.push({
|
|
1517
1750
|
application: applicationId,
|
|
1518
|
-
health:
|
|
1751
|
+
health: healthReport
|
|
1519
1752
|
})
|
|
1520
1753
|
} else if (workers) {
|
|
1521
|
-
const
|
|
1754
|
+
const workersReport = await this.#updateApplicationWorkers(
|
|
1522
1755
|
applicationId,
|
|
1523
1756
|
config,
|
|
1524
1757
|
applicationConfig,
|
|
@@ -1527,7 +1760,7 @@ export class Runtime extends EventEmitter {
|
|
|
1527
1760
|
)
|
|
1528
1761
|
report.push({
|
|
1529
1762
|
application: applicationId,
|
|
1530
|
-
workers:
|
|
1763
|
+
workers: workersReport
|
|
1531
1764
|
})
|
|
1532
1765
|
}
|
|
1533
1766
|
}
|
|
@@ -1543,8 +1776,20 @@ export class Runtime extends EventEmitter {
|
|
|
1543
1776
|
return this.#root
|
|
1544
1777
|
}
|
|
1545
1778
|
|
|
1546
|
-
|
|
1547
|
-
|
|
1779
|
+
getUrls (applicationId) {
|
|
1780
|
+
const urls = {}
|
|
1781
|
+
const applicationIds = applicationId ? [applicationId] : this.#applications.keys()
|
|
1782
|
+
|
|
1783
|
+
for (const id of applicationIds) {
|
|
1784
|
+
for (const workerId of this.#workers.getKeys(id)) {
|
|
1785
|
+
const url = this.#workers.get(workerId)?.[kWorkerUrl]
|
|
1786
|
+
if (url) {
|
|
1787
|
+
urls[workerId] = url
|
|
1788
|
+
}
|
|
1789
|
+
}
|
|
1790
|
+
}
|
|
1791
|
+
|
|
1792
|
+
return urls
|
|
1548
1793
|
}
|
|
1549
1794
|
|
|
1550
1795
|
getRuntimeStatus () {
|
|
@@ -1553,8 +1798,6 @@ export class Runtime extends EventEmitter {
|
|
|
1553
1798
|
|
|
1554
1799
|
async getRuntimeMetadata () {
|
|
1555
1800
|
const packageJson = await this.#getRuntimePackageJson()
|
|
1556
|
-
const entrypointDetails = await this.getEntrypointDetails()
|
|
1557
|
-
|
|
1558
1801
|
return {
|
|
1559
1802
|
pid: process.pid,
|
|
1560
1803
|
cwd: process.cwd(),
|
|
@@ -1563,10 +1806,18 @@ export class Runtime extends EventEmitter {
|
|
|
1563
1806
|
execPath: process.execPath,
|
|
1564
1807
|
nodeVersion: process.version,
|
|
1565
1808
|
projectDir: this.#root,
|
|
1809
|
+
/*
|
|
1810
|
+
What `applications:add`/`remove --save` actually consume, so they can stop reading the whole
|
|
1811
|
+
runtime configuration over HTTP to get at three values. `autoload` is the declaration as
|
|
1812
|
+
authored, not the expansion: it is expanded in the eval worker, and --save has to edit what
|
|
1813
|
+
the file says rather than what it produced.
|
|
1814
|
+
*/
|
|
1815
|
+
configPath: this.#config[kMetadata]?.path ?? null,
|
|
1816
|
+
autoload: this.#config.autoload ?? null,
|
|
1566
1817
|
packageName: packageJson.name ?? null,
|
|
1567
1818
|
packageVersion: packageJson.version ?? null,
|
|
1568
|
-
|
|
1569
|
-
|
|
1819
|
+
platformaticVersion: version,
|
|
1820
|
+
urls: this.getUrls()
|
|
1570
1821
|
}
|
|
1571
1822
|
}
|
|
1572
1823
|
|
|
@@ -1574,13 +1825,31 @@ export class Runtime extends EventEmitter {
|
|
|
1574
1825
|
return this.#env
|
|
1575
1826
|
}
|
|
1576
1827
|
|
|
1828
|
+
/*
|
|
1829
|
+
What a watcher has to follow to know this configuration changed. The whole set the evaluation
|
|
1830
|
+
read is reported; resolving per worker had no such set, so it was the deciding file alone --
|
|
1831
|
+
which is what dev watched before either way.
|
|
1832
|
+
*/
|
|
1833
|
+
getConfigurationWatchTargets () {
|
|
1834
|
+
const metadata = this.#config[kMetadata]
|
|
1835
|
+
const targets = metadata?.loader?.watchTargets
|
|
1836
|
+
|
|
1837
|
+
if (targets) {
|
|
1838
|
+
return { files: [...targets.files], directories: [...targets.directories] }
|
|
1839
|
+
}
|
|
1840
|
+
|
|
1841
|
+
return { files: metadata?.path ? [metadata.path] : [], directories: [] }
|
|
1842
|
+
}
|
|
1843
|
+
|
|
1577
1844
|
getRuntimeConfig (includeMeta = false) {
|
|
1845
|
+
// includeMeta is an internal contract and hands back live state, symbol key and all. It leaves
|
|
1846
|
+
// the public surface with the DTO change; until then, its in-tree callers depend on identity.
|
|
1578
1847
|
if (includeMeta) {
|
|
1579
1848
|
return this.#config
|
|
1580
1849
|
}
|
|
1581
1850
|
|
|
1582
1851
|
const { [kMetadata]: _, ...config } = this.#config
|
|
1583
|
-
return config
|
|
1852
|
+
return frozenSnapshot(config)
|
|
1584
1853
|
}
|
|
1585
1854
|
|
|
1586
1855
|
getInterceptor () {
|
|
@@ -1599,12 +1868,39 @@ export class Runtime extends EventEmitter {
|
|
|
1599
1868
|
return this.#managementApi?.server.address() ?? null
|
|
1600
1869
|
}
|
|
1601
1870
|
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1871
|
+
/*
|
|
1872
|
+
servingState is computed per worker: for a worker-classified capability it depends on what the
|
|
1873
|
+
application's factory returned in *that* worker, and nothing stops arbitrary code from returning
|
|
1874
|
+
a server from worker 0 and a background result from worker 1. Sampling one worker -- which is
|
|
1875
|
+
what getApplicationDetails does for every other field -- would make the reported value depend on
|
|
1876
|
+
which worker the selector picked, and, worse, would leave mesh dispatch routing a share of
|
|
1877
|
+
requests to a worker that destroys them. So every worker answers and a mixed answer is refused,
|
|
1878
|
+
naming each worker and the state it reported.
|
|
1879
|
+
*/
|
|
1880
|
+
async #collectServingState (id) {
|
|
1881
|
+
const invocations = this.#workers.getKeys(id).map(workerId => [workerId, this.#workers.get(workerId)])
|
|
1882
|
+
|
|
1883
|
+
if (invocations.length === 0) {
|
|
1884
|
+
this.#servingStates.delete(id)
|
|
1885
|
+
return
|
|
1605
1886
|
}
|
|
1606
1887
|
|
|
1607
|
-
|
|
1888
|
+
const states = await sendMultipleViaITC(invocations, 'getServingState', undefined, [], this.#concurrency)
|
|
1889
|
+
const reported = Object.entries(states).filter(([, state]) => typeof state === 'string')
|
|
1890
|
+
|
|
1891
|
+
if (reported.length === 0) {
|
|
1892
|
+
this.#servingStates.delete(id)
|
|
1893
|
+
return
|
|
1894
|
+
}
|
|
1895
|
+
|
|
1896
|
+
const distinct = new Set(reported.map(([, state]) => state))
|
|
1897
|
+
|
|
1898
|
+
if (distinct.size > 1) {
|
|
1899
|
+
this.#servingStates.delete(id)
|
|
1900
|
+
throw new MixedServingStateError(id, reported.map(([worker, state]) => `${worker} reported ${state}`).join(', '))
|
|
1901
|
+
}
|
|
1902
|
+
|
|
1903
|
+
this.#servingStates.set(id, reported[0][1])
|
|
1608
1904
|
}
|
|
1609
1905
|
|
|
1610
1906
|
async getCustomHealthChecks () {
|
|
@@ -1663,7 +1959,7 @@ export class Runtime extends EventEmitter {
|
|
|
1663
1959
|
|
|
1664
1960
|
async getMetrics (format = 'json') {
|
|
1665
1961
|
if (this.#config.metrics === false || this.#config.metrics?.enabled === false) {
|
|
1666
|
-
throw new
|
|
1962
|
+
throw new MetricsDisabledError()
|
|
1667
1963
|
}
|
|
1668
1964
|
|
|
1669
1965
|
let metrics = null
|
|
@@ -1758,7 +2054,7 @@ export class Runtime extends EventEmitter {
|
|
|
1758
2054
|
// Drop any configured custom label that shares the name of the application
|
|
1759
2055
|
// label (a config can set both `applicationLabel: 'serviceId'` and a static
|
|
1760
2056
|
// `serviceId` label): keeping it would make these runtime-wide metrics look
|
|
1761
|
-
// like they belong to an application
|
|
2057
|
+
// like they belong to an application during metrics aggregation.
|
|
1762
2058
|
const processLabels = { ...this.#config.metrics?.labels }
|
|
1763
2059
|
delete processLabels[this.#metricsLabelName]
|
|
1764
2060
|
this.#applyLabelsToMetrics(processMetricsJson, processLabels, processMetrics)
|
|
@@ -1923,180 +2219,41 @@ export class Runtime extends EventEmitter {
|
|
|
1923
2219
|
return output
|
|
1924
2220
|
}
|
|
1925
2221
|
|
|
1926
|
-
async getFormattedMetrics () {
|
|
1927
|
-
try {
|
|
1928
|
-
const { metrics } = await this.getMetrics()
|
|
1929
|
-
|
|
1930
|
-
if (metrics === null || metrics.length === 0) {
|
|
1931
|
-
return null
|
|
1932
|
-
}
|
|
1933
|
-
|
|
1934
|
-
const metricsNames = [
|
|
1935
|
-
'process_cpu_percent_usage',
|
|
1936
|
-
'process_resident_memory_bytes',
|
|
1937
|
-
'nodejs_heap_size_total_bytes',
|
|
1938
|
-
'nodejs_heap_size_used_bytes',
|
|
1939
|
-
'nodejs_heap_space_size_total_bytes',
|
|
1940
|
-
'nodejs_eventloop_utilization',
|
|
1941
|
-
'http_request_all_summary_seconds'
|
|
1942
|
-
]
|
|
1943
|
-
|
|
1944
|
-
const applicationsMetrics = {}
|
|
1945
|
-
|
|
1946
|
-
// Process-level metrics are reported only once for the whole runtime, without
|
|
1947
|
-
// an application label, since they are shared by all the applications running
|
|
1948
|
-
// in worker threads (see issue #3332). Applications running as separate OS
|
|
1949
|
-
// processes report their own labeled values, which take precedence below.
|
|
1950
|
-
const runtimeProcessMetrics = {}
|
|
1951
|
-
const applicationProcessMetrics = new Set()
|
|
1952
|
-
|
|
1953
|
-
for (const metric of metrics) {
|
|
1954
|
-
const { name, values } = metric
|
|
1955
|
-
|
|
1956
|
-
if (!metricsNames.includes(name)) continue
|
|
1957
|
-
if (!values || values.length === 0) continue
|
|
1958
|
-
|
|
1959
|
-
const labels = values[0].labels
|
|
1960
|
-
// Use the configured label name (serviceId for v2 compatibility, applicationId for v3+)
|
|
1961
|
-
const applicationId = labels?.[this.#metricsLabelName]
|
|
1962
|
-
|
|
1963
|
-
if (!applicationId) {
|
|
1964
|
-
if (name === 'process_cpu_percent_usage') {
|
|
1965
|
-
runtimeProcessMetrics.cpu = values[0].value
|
|
1966
|
-
continue
|
|
1967
|
-
}
|
|
1968
|
-
if (name === 'process_resident_memory_bytes') {
|
|
1969
|
-
runtimeProcessMetrics.rss = values[0].value
|
|
1970
|
-
continue
|
|
1971
|
-
}
|
|
1972
|
-
|
|
1973
|
-
throw new Error(`Missing ${this.#metricsLabelName} label in metrics`)
|
|
1974
|
-
}
|
|
1975
|
-
|
|
1976
|
-
if (name === 'process_cpu_percent_usage' || name === 'process_resident_memory_bytes') {
|
|
1977
|
-
applicationProcessMetrics.add(`${applicationId}:${name}`)
|
|
1978
|
-
}
|
|
1979
|
-
|
|
1980
|
-
let applicationMetrics = applicationsMetrics[applicationId]
|
|
1981
|
-
if (!applicationMetrics) {
|
|
1982
|
-
applicationMetrics = {
|
|
1983
|
-
cpu: 0,
|
|
1984
|
-
rss: 0,
|
|
1985
|
-
totalHeapSize: 0,
|
|
1986
|
-
usedHeapSize: 0,
|
|
1987
|
-
newSpaceSize: 0,
|
|
1988
|
-
oldSpaceSize: 0,
|
|
1989
|
-
elu: 0,
|
|
1990
|
-
latency: {
|
|
1991
|
-
p50: 0,
|
|
1992
|
-
p90: 0,
|
|
1993
|
-
p95: 0,
|
|
1994
|
-
p99: 0
|
|
1995
|
-
}
|
|
1996
|
-
}
|
|
1997
|
-
applicationsMetrics[applicationId] = applicationMetrics
|
|
1998
|
-
}
|
|
1999
|
-
|
|
2000
|
-
parsePromMetric(applicationMetrics, metric)
|
|
2001
|
-
}
|
|
2002
|
-
|
|
2003
|
-
// Apply the runtime-wide process-level values to every application that did
|
|
2004
|
-
// not report its own (i.e. every application running in a worker thread).
|
|
2005
|
-
for (const [applicationId, applicationMetrics] of Object.entries(applicationsMetrics)) {
|
|
2006
|
-
if (
|
|
2007
|
-
runtimeProcessMetrics.cpu !== undefined &&
|
|
2008
|
-
!applicationProcessMetrics.has(`${applicationId}:process_cpu_percent_usage`)
|
|
2009
|
-
) {
|
|
2010
|
-
applicationMetrics.cpu = runtimeProcessMetrics.cpu
|
|
2011
|
-
}
|
|
2012
|
-
|
|
2013
|
-
if (
|
|
2014
|
-
runtimeProcessMetrics.rss !== undefined &&
|
|
2015
|
-
!applicationProcessMetrics.has(`${applicationId}:process_resident_memory_bytes`)
|
|
2016
|
-
) {
|
|
2017
|
-
applicationMetrics.rss = runtimeProcessMetrics.rss
|
|
2018
|
-
}
|
|
2019
|
-
}
|
|
2020
|
-
|
|
2021
|
-
function parsePromMetric (applicationMetrics, promMetric) {
|
|
2022
|
-
const { name } = promMetric
|
|
2023
|
-
|
|
2024
|
-
if (name === 'process_cpu_percent_usage') {
|
|
2025
|
-
applicationMetrics.cpu = promMetric.values[0].value
|
|
2026
|
-
return
|
|
2027
|
-
}
|
|
2028
|
-
if (name === 'process_resident_memory_bytes') {
|
|
2029
|
-
applicationMetrics.rss = promMetric.values[0].value
|
|
2030
|
-
return
|
|
2031
|
-
}
|
|
2032
|
-
if (name === 'nodejs_heap_size_total_bytes') {
|
|
2033
|
-
applicationMetrics.totalHeapSize = promMetric.values[0].value
|
|
2034
|
-
return
|
|
2035
|
-
}
|
|
2036
|
-
if (name === 'nodejs_heap_size_used_bytes') {
|
|
2037
|
-
applicationMetrics.usedHeapSize = promMetric.values[0].value
|
|
2038
|
-
return
|
|
2039
|
-
}
|
|
2040
|
-
if (name === 'nodejs_heap_space_size_total_bytes') {
|
|
2041
|
-
const newSpaceSize = promMetric.values.find(value => value.labels.space === 'new')
|
|
2042
|
-
const oldSpaceSize = promMetric.values.find(value => value.labels.space === 'old')
|
|
2043
|
-
|
|
2044
|
-
applicationMetrics.newSpaceSize = newSpaceSize.value
|
|
2045
|
-
applicationMetrics.oldSpaceSize = oldSpaceSize.value
|
|
2046
|
-
return
|
|
2047
|
-
}
|
|
2048
|
-
if (name === 'nodejs_eventloop_utilization') {
|
|
2049
|
-
applicationMetrics.elu = promMetric.values[0].value
|
|
2050
|
-
return
|
|
2051
|
-
}
|
|
2052
|
-
if (name === 'http_request_all_summary_seconds') {
|
|
2053
|
-
applicationMetrics.latency = {
|
|
2054
|
-
p50: promMetric.values.find(value => value.labels.quantile === 0.5)?.value || 0,
|
|
2055
|
-
p90: promMetric.values.find(value => value.labels.quantile === 0.9)?.value || 0,
|
|
2056
|
-
p95: promMetric.values.find(value => value.labels.quantile === 0.95)?.value || 0,
|
|
2057
|
-
p99: promMetric.values.find(value => value.labels.quantile === 0.99)?.value || 0
|
|
2058
|
-
}
|
|
2059
|
-
}
|
|
2060
|
-
}
|
|
2061
|
-
|
|
2062
|
-
return {
|
|
2063
|
-
version: 1,
|
|
2064
|
-
date: new Date().toISOString(),
|
|
2065
|
-
applications: applicationsMetrics
|
|
2066
|
-
}
|
|
2067
|
-
} catch (err) {
|
|
2068
|
-
// If any metric is missing, return nothing
|
|
2069
|
-
this.logger.warn({ err }, 'Cannot fetch metrics')
|
|
2070
|
-
|
|
2071
|
-
return null
|
|
2072
|
-
}
|
|
2073
|
-
}
|
|
2074
|
-
|
|
2075
2222
|
getSharedContext () {
|
|
2076
2223
|
return this.#sharedContext
|
|
2077
2224
|
}
|
|
2078
2225
|
|
|
2079
2226
|
async getApplicationResourcesInfo (id) {
|
|
2080
|
-
const
|
|
2227
|
+
const workers = this.#replicas.get(id)?.length ?? 0
|
|
2081
2228
|
// Use round-robin to get any available worker instead of assuming index 0 exists
|
|
2082
2229
|
const worker = await this.#getWorkerByIdOrNext(id, null, false, false)
|
|
2083
|
-
const health = worker[kConfig].health
|
|
2084
2230
|
|
|
2085
|
-
|
|
2231
|
+
if (worker) {
|
|
2232
|
+
return { workers, health: worker[kConfig].health }
|
|
2233
|
+
}
|
|
2234
|
+
|
|
2235
|
+
// No worker can answer (the application is stopped or its only worker is being restarted), use the configuration
|
|
2236
|
+
const applicationConfig = this.#applications.get(id)
|
|
2237
|
+
const health = deepmerge(this.#config.health ?? {}, applicationConfig.health ?? {})
|
|
2238
|
+
|
|
2239
|
+
return { workers, health }
|
|
2086
2240
|
}
|
|
2087
2241
|
|
|
2088
2242
|
getApplicationsIds () {
|
|
2089
|
-
return Array.from(this.#applications.keys())
|
|
2243
|
+
return Array.from(this.#applications.keys()).sort()
|
|
2090
2244
|
}
|
|
2091
2245
|
|
|
2092
2246
|
async getApplications (allowUnloaded = false) {
|
|
2093
|
-
return {
|
|
2094
|
-
entrypoint: this.#entrypointId,
|
|
2247
|
+
return frozenSnapshot({
|
|
2095
2248
|
production: this.#isProduction,
|
|
2096
2249
|
applications: await Promise.all(
|
|
2097
|
-
this.getApplicationsIds().map(id => this
|
|
2250
|
+
this.getApplicationsIds().map(id => this.#buildApplicationDetails(id, allowUnloaded))
|
|
2098
2251
|
)
|
|
2099
|
-
}
|
|
2252
|
+
})
|
|
2253
|
+
}
|
|
2254
|
+
|
|
2255
|
+
async getApplicationDetails (id, allowUnloaded = false) {
|
|
2256
|
+
return frozenSnapshot(await this.#buildApplicationDetails(id, allowUnloaded))
|
|
2100
2257
|
}
|
|
2101
2258
|
|
|
2102
2259
|
async getApplicationMeta (id) {
|
|
@@ -2124,7 +2281,7 @@ export class Runtime extends EventEmitter {
|
|
|
2124
2281
|
}
|
|
2125
2282
|
}
|
|
2126
2283
|
|
|
2127
|
-
async
|
|
2284
|
+
async #buildApplicationDetails (id, allowUnloaded = false) {
|
|
2128
2285
|
let application
|
|
2129
2286
|
|
|
2130
2287
|
try {
|
|
@@ -2137,7 +2294,7 @@ export class Runtime extends EventEmitter {
|
|
|
2137
2294
|
throw e
|
|
2138
2295
|
}
|
|
2139
2296
|
|
|
2140
|
-
const {
|
|
2297
|
+
const { localUrl, config, configPath, path } = application[kConfig]
|
|
2141
2298
|
|
|
2142
2299
|
const sourceMaps = application[kConfig].sourceMaps ?? this.#config.sourceMaps
|
|
2143
2300
|
const status = await sendViaITC(application, 'getStatus')
|
|
@@ -2146,24 +2303,43 @@ export class Runtime extends EventEmitter {
|
|
|
2146
2303
|
const applicationDetails = {
|
|
2147
2304
|
id,
|
|
2148
2305
|
type,
|
|
2149
|
-
config,
|
|
2150
2306
|
path,
|
|
2151
2307
|
status,
|
|
2152
2308
|
dependencies,
|
|
2153
2309
|
version,
|
|
2154
2310
|
localUrl,
|
|
2155
|
-
entrypoint,
|
|
2156
2311
|
sourceMaps
|
|
2157
2312
|
}
|
|
2158
2313
|
|
|
2159
|
-
|
|
2160
|
-
|
|
2314
|
+
/*
|
|
2315
|
+
The `config` key once held the application's configuration file path. That file is now
|
|
2316
|
+
evaluated main-side and the worker receives the payload, so the path becomes `configPath` and the entry's own
|
|
2317
|
+
`config` is no longer a path to report. Emitting the key that matches the dialect keeps a
|
|
2318
|
+
consumer from reading one and silently getting the other.
|
|
2319
|
+
*/
|
|
2320
|
+
if (typeof config === 'string') {
|
|
2321
|
+
applicationDetails.config = config
|
|
2322
|
+
} else if (configPath) {
|
|
2323
|
+
applicationDetails.configPath = configPath
|
|
2161
2324
|
}
|
|
2162
2325
|
|
|
2163
|
-
|
|
2164
|
-
|
|
2326
|
+
// status is worker lifecycle; servingState is how the thing serves. They are different
|
|
2327
|
+
// questions, and the runtime gates URL emission on status === 'started', so they cannot share
|
|
2328
|
+
// a field.
|
|
2329
|
+
const servingState = this.#servingStates.get(id)
|
|
2330
|
+
|
|
2331
|
+
if (servingState) {
|
|
2332
|
+
applicationDetails.servingState = servingState
|
|
2165
2333
|
}
|
|
2166
2334
|
|
|
2335
|
+
if (this.#isProduction) {
|
|
2336
|
+
applicationDetails.workers = this.#replicas.get(id)?.length ?? 0
|
|
2337
|
+
}
|
|
2338
|
+
|
|
2339
|
+
const urls = status === 'started' ? Object.values(this.getUrls(id)) : []
|
|
2340
|
+
applicationDetails.urls = urls
|
|
2341
|
+
applicationDetails.url = urls[0] ?? null
|
|
2342
|
+
|
|
2167
2343
|
return applicationDetails
|
|
2168
2344
|
}
|
|
2169
2345
|
|
|
@@ -2211,10 +2387,6 @@ export class Runtime extends EventEmitter {
|
|
|
2211
2387
|
return this.#scheduler?.getJobs() ?? []
|
|
2212
2388
|
}
|
|
2213
2389
|
|
|
2214
|
-
getScheduler () {
|
|
2215
|
-
return this.getSchedulerJobs()
|
|
2216
|
-
}
|
|
2217
|
-
|
|
2218
2390
|
pauseSchedulerJob (name) {
|
|
2219
2391
|
return this.#schedulerOrThrow().pauseJob(name)
|
|
2220
2392
|
}
|
|
@@ -2283,6 +2455,7 @@ export class Runtime extends EventEmitter {
|
|
|
2283
2455
|
worker: index,
|
|
2284
2456
|
status: worker[kWorkerStatus],
|
|
2285
2457
|
thread: worker.threadId,
|
|
2458
|
+
url: worker[kWorkerUrl] ?? null,
|
|
2286
2459
|
raw: includeRaw ? worker : undefined
|
|
2287
2460
|
}
|
|
2288
2461
|
}
|
|
@@ -2309,10 +2482,6 @@ export class Runtime extends EventEmitter {
|
|
|
2309
2482
|
elu = worker.performance.eventLoopUtilization(elu, previousELU)
|
|
2310
2483
|
}
|
|
2311
2484
|
|
|
2312
|
-
if (!features.node.worker.getHeapStatistics) {
|
|
2313
|
-
return { elu: elu.utilization, currentELU }
|
|
2314
|
-
}
|
|
2315
|
-
|
|
2316
2485
|
// Only refresh heap statistics every 60 health checks (once per minute).
|
|
2317
2486
|
// This keeps the common path fully synchronous — no promise allocation.
|
|
2318
2487
|
const counter = (worker[kHeapCheckCounter] ?? 0) + 1
|
|
@@ -2404,12 +2573,37 @@ export class Runtime extends EventEmitter {
|
|
|
2404
2573
|
this.emitAndNotify(status, args)
|
|
2405
2574
|
}
|
|
2406
2575
|
|
|
2407
|
-
#
|
|
2408
|
-
|
|
2409
|
-
|
|
2576
|
+
#showUrls (workerIds) {
|
|
2577
|
+
const whitelist = workerIds ? new Set(workerIds) : null
|
|
2578
|
+
|
|
2579
|
+
for (const applicationId of this.#applications.keys()) {
|
|
2580
|
+
for (const workerId of this.#workers.getKeys(applicationId)) {
|
|
2581
|
+
if (whitelist && !whitelist.has(workerId)) {
|
|
2582
|
+
continue
|
|
2583
|
+
}
|
|
2584
|
+
|
|
2585
|
+
const worker = this.#workers.get(workerId)
|
|
2586
|
+
const url = worker?.[kWorkerUrl]
|
|
2587
|
+
if (!url) {
|
|
2588
|
+
continue
|
|
2589
|
+
}
|
|
2590
|
+
|
|
2591
|
+
this.logger.info(
|
|
2592
|
+
`Platformatic is now listening at ${url} for ${this.#workerExtendedLabel(applicationId, worker[kWorkerId])}`
|
|
2593
|
+
)
|
|
2594
|
+
}
|
|
2410
2595
|
}
|
|
2596
|
+
}
|
|
2597
|
+
|
|
2598
|
+
async #refuseUnresolvedApplication (id) {
|
|
2599
|
+
const executable = getExecutable() ?? 'wattpm'
|
|
2411
2600
|
|
|
2412
|
-
this.logger.
|
|
2601
|
+
this.logger.error(
|
|
2602
|
+
`The path for application "%s" does not exist. Please run "${executable} resolve" and try again.`,
|
|
2603
|
+
id
|
|
2604
|
+
)
|
|
2605
|
+
|
|
2606
|
+
await this.closeAndThrow(new RuntimeAbortedError())
|
|
2413
2607
|
}
|
|
2414
2608
|
|
|
2415
2609
|
async #setupApplication (applicationConfig) {
|
|
@@ -2427,13 +2621,7 @@ export class Runtime extends EventEmitter {
|
|
|
2427
2621
|
applicationConfig.path = join(this.#root, config.resolvedApplicationsBasePath, id)
|
|
2428
2622
|
|
|
2429
2623
|
if (!existsSync(applicationConfig.path)) {
|
|
2430
|
-
|
|
2431
|
-
this.logger.error(
|
|
2432
|
-
`The path for application "%s" does not exist. Please run "${executable} resolve" and try again.`,
|
|
2433
|
-
id
|
|
2434
|
-
)
|
|
2435
|
-
|
|
2436
|
-
await this.closeAndThrow(new RuntimeAbortedError())
|
|
2624
|
+
await this.#refuseUnresolvedApplication(id)
|
|
2437
2625
|
}
|
|
2438
2626
|
} else {
|
|
2439
2627
|
this.logger.error(
|
|
@@ -2443,17 +2631,47 @@ export class Runtime extends EventEmitter {
|
|
|
2443
2631
|
|
|
2444
2632
|
await this.closeAndThrow(new RuntimeAbortedError())
|
|
2445
2633
|
}
|
|
2634
|
+
} else if (applicationConfig.unresolved) {
|
|
2635
|
+
/*
|
|
2636
|
+
A remote application that declares where its clone belongs, and whose clone is not there --
|
|
2637
|
+
either the directory is missing or it holds no application at all. The pathless case above
|
|
2638
|
+
is the same state said differently, and both are what `resolve` exists to fix, so both get
|
|
2639
|
+
the message that names it rather than the capability detector's report that the directory
|
|
2640
|
+
holds nothing.
|
|
2641
|
+
*/
|
|
2642
|
+
await this.#refuseUnresolvedApplication(id)
|
|
2446
2643
|
}
|
|
2447
2644
|
|
|
2448
|
-
|
|
2449
|
-
|
|
2645
|
+
let workers = applicationConfig.workers.static
|
|
2646
|
+
|
|
2647
|
+
this.#replicas.set(id, [])
|
|
2450
2648
|
|
|
2451
|
-
|
|
2452
|
-
|
|
2453
|
-
|
|
2649
|
+
let firstIndex = 0
|
|
2650
|
+
|
|
2651
|
+
// On platforms where reusePort is not available, multiple workers cannot listen on the same fixed port.
|
|
2652
|
+
// The listener configuration is owned by the capability, so it can only be inspected after setting up the first
|
|
2653
|
+
// worker: if the application would try to share a fixed port between workers, clamp it to a single worker.
|
|
2654
|
+
if (!features.node.reusePort && (workers > 1 || applicationConfig.workers.dynamic)) {
|
|
2655
|
+
const { worker } = await this.#setupReplica(config, applicationConfig, workers, id, 0, 0)
|
|
2656
|
+
firstIndex = 1
|
|
2657
|
+
|
|
2658
|
+
if (await this.#usesSharedFixedPort(worker)) {
|
|
2659
|
+
this.logger.warn(
|
|
2660
|
+
`The application "${id}" is configured to listen on a fixed port with multiple workers, but reusePort is not available in your OS. ${workers > 1 ? `Setting workers to 1 instead of ${workers}` : 'Disabling dynamic workers scaling'}. To run multiple workers, set "server.portAssignment" to "perWorkerIncrement" in the application configuration.`
|
|
2661
|
+
)
|
|
2662
|
+
|
|
2663
|
+
applicationConfig.workers = { dynamic: false, static: 1 }
|
|
2664
|
+
workers = 1
|
|
2665
|
+
await sendViaITC(worker, 'updateWorkersCount', { applicationId: id, workers })
|
|
2666
|
+
}
|
|
2667
|
+
}
|
|
2668
|
+
|
|
2669
|
+
const setupInvocations = []
|
|
2670
|
+
for (let i = firstIndex; i < workers; i++) {
|
|
2671
|
+
setupInvocations.push([config, applicationConfig, workers, id, i, i])
|
|
2454
2672
|
}
|
|
2455
2673
|
|
|
2456
|
-
await executeInParallel(this.#
|
|
2674
|
+
await executeInParallel(this.#setupReplica.bind(this), setupInvocations, this.#concurrency)
|
|
2457
2675
|
|
|
2458
2676
|
// Initialize the next worker index counter (next index starts after initial workers)
|
|
2459
2677
|
this.#nextWorkerIndex.set(id, workers)
|
|
@@ -2462,7 +2680,51 @@ export class Runtime extends EventEmitter {
|
|
|
2462
2680
|
this.emitAndNotify('application:init', id)
|
|
2463
2681
|
}
|
|
2464
2682
|
|
|
2465
|
-
|
|
2683
|
+
// Adds a replica to the application (see the #replicas field) and sets up its worker.
|
|
2684
|
+
// Returns null if the replica was removed in the meanwhile.
|
|
2685
|
+
async #setupReplica (config, applicationConfig, workersCount, applicationId, index, portOffset) {
|
|
2686
|
+
const replica = {
|
|
2687
|
+
portOffset,
|
|
2688
|
+
// The worker serving the replica, null while the replica is being refilled
|
|
2689
|
+
worker: null,
|
|
2690
|
+
// The worker being started to refill the replica or to replace its worker
|
|
2691
|
+
replacement: null,
|
|
2692
|
+
// The operation in progress: 'refill' or 'replace'
|
|
2693
|
+
pending: null,
|
|
2694
|
+
abortController: new AbortController()
|
|
2695
|
+
}
|
|
2696
|
+
|
|
2697
|
+
// Added right away, so that concurrent operations see the requested workers
|
|
2698
|
+
this.#replicas.get(applicationId).push(replica)
|
|
2699
|
+
|
|
2700
|
+
let worker
|
|
2701
|
+
try {
|
|
2702
|
+
worker = await this.#setupWorker(config, applicationConfig, workersCount, applicationId, index, portOffset)
|
|
2703
|
+
} catch (err) {
|
|
2704
|
+
await this.#detachReplica(applicationId, replica)
|
|
2705
|
+
throw err
|
|
2706
|
+
}
|
|
2707
|
+
|
|
2708
|
+
// The replica was removed in the meanwhile (i.e. scale-down or application stopped)
|
|
2709
|
+
if (replica.abortController.signal.aborted) {
|
|
2710
|
+
await this.#discardWorker(worker)
|
|
2711
|
+
return null
|
|
2712
|
+
}
|
|
2713
|
+
|
|
2714
|
+
replica.worker = worker
|
|
2715
|
+
return replica
|
|
2716
|
+
}
|
|
2717
|
+
|
|
2718
|
+
async #setupWorker (
|
|
2719
|
+
config,
|
|
2720
|
+
applicationConfig,
|
|
2721
|
+
workersCount,
|
|
2722
|
+
applicationId,
|
|
2723
|
+
index,
|
|
2724
|
+
portOffset,
|
|
2725
|
+
enabled = true,
|
|
2726
|
+
attempt = 0
|
|
2727
|
+
) {
|
|
2466
2728
|
const restartOnError = this.#getApplicationRestartOnError(config, applicationConfig)
|
|
2467
2729
|
const workerId = `${applicationId}:${index}`
|
|
2468
2730
|
|
|
@@ -2477,17 +2739,11 @@ export class Runtime extends EventEmitter {
|
|
|
2477
2739
|
inspectorOptions.port = inspectorOptions.port + this.#workers.size + 1
|
|
2478
2740
|
}
|
|
2479
2741
|
|
|
2480
|
-
|
|
2481
|
-
|
|
2482
|
-
|
|
2483
|
-
|
|
2484
|
-
|
|
2485
|
-
|
|
2486
|
-
if (config.telemetry) {
|
|
2487
|
-
applicationConfig.telemetry = {
|
|
2488
|
-
...config.telemetry,
|
|
2489
|
-
...applicationConfig.telemetry,
|
|
2490
|
-
applicationName: `${config.telemetry.applicationName}-${applicationConfig.id}`
|
|
2742
|
+
if (config.tracing) {
|
|
2743
|
+
applicationConfig.tracing = {
|
|
2744
|
+
...config.tracing,
|
|
2745
|
+
...applicationConfig.tracing,
|
|
2746
|
+
applicationName: `${config.tracing.applicationName}-${applicationConfig.id}`
|
|
2491
2747
|
}
|
|
2492
2748
|
}
|
|
2493
2749
|
|
|
@@ -2496,9 +2752,9 @@ export class Runtime extends EventEmitter {
|
|
|
2496
2752
|
|
|
2497
2753
|
const execArgv = applicationConfig.execArgv ?? []
|
|
2498
2754
|
|
|
2499
|
-
if (!applicationConfig.
|
|
2755
|
+
if (!applicationConfig.skipTracingHooks && config.tracing && config.tracing.enabled !== false) {
|
|
2500
2756
|
const require = createRequire(import.meta.url)
|
|
2501
|
-
const telemetryPath = require.resolve('@platformatic/
|
|
2757
|
+
const telemetryPath = require.resolve('@platformatic/tracing')
|
|
2502
2758
|
const openTelemetrySetupPath = join(telemetryPath, '..', 'lib', 'node-telemetry.js')
|
|
2503
2759
|
const hookUrl = pathToFileURL(require.resolve('@opentelemetry/instrumentation/hook.mjs'))
|
|
2504
2760
|
|
|
@@ -2523,7 +2779,10 @@ export class Runtime extends EventEmitter {
|
|
|
2523
2779
|
preload = preload.filter(p => p !== pprofCapturePath)
|
|
2524
2780
|
}
|
|
2525
2781
|
|
|
2526
|
-
|
|
2782
|
+
// Each application's worker environment is resolved main-side, with its own env-file chain, the
|
|
2783
|
+
// two env blocks and the injected topology URLs. Every worker used to be seeded from one loadEnv
|
|
2784
|
+
// at the runtime root, which is what #env still holds.
|
|
2785
|
+
const workerEnv = structuredClone(applicationConfig.workerEnv ?? this.#env)
|
|
2527
2786
|
|
|
2528
2787
|
if (applicationConfig.nodeOptions?.trim().length > 0) {
|
|
2529
2788
|
const originalNodeOptions = workerEnv.NODE_OPTIONS ?? ''
|
|
@@ -2546,21 +2805,12 @@ export class Runtime extends EventEmitter {
|
|
|
2546
2805
|
const maxYoungGenerationSizeMb = maxYoungGeneration ? Math.floor(maxYoungGeneration / (1024 * 1024)) : undefined
|
|
2547
2806
|
const codeRangeSizeMb = codeRangeSize ? Math.floor(codeRangeSize / (1024 * 1024)) : undefined
|
|
2548
2807
|
|
|
2549
|
-
const workerConfig = {
|
|
2550
|
-
...config,
|
|
2551
|
-
preload
|
|
2552
|
-
}
|
|
2553
|
-
|
|
2554
|
-
if (config.server && serverConfigOverride) {
|
|
2555
|
-
workerConfig.server = {
|
|
2556
|
-
...config.server,
|
|
2557
|
-
...serverConfigOverride
|
|
2558
|
-
}
|
|
2559
|
-
}
|
|
2808
|
+
const workerConfig = { ...config, preload }
|
|
2560
2809
|
|
|
2561
2810
|
const worker = new Worker(kWorkerFile, {
|
|
2562
2811
|
workerData: {
|
|
2563
2812
|
config: workerConfig,
|
|
2813
|
+
meshId: this.#meshId,
|
|
2564
2814
|
applicationConfig: {
|
|
2565
2815
|
...applicationConfig,
|
|
2566
2816
|
isProduction: this.#isProduction,
|
|
@@ -2569,7 +2819,8 @@ export class Runtime extends EventEmitter {
|
|
|
2569
2819
|
worker: {
|
|
2570
2820
|
id: workerId,
|
|
2571
2821
|
index,
|
|
2572
|
-
count: workersCount
|
|
2822
|
+
count: workersCount,
|
|
2823
|
+
portOffset
|
|
2573
2824
|
},
|
|
2574
2825
|
resourceLimits: {
|
|
2575
2826
|
maxOldGenerationSizeMb,
|
|
@@ -2577,10 +2828,7 @@ export class Runtime extends EventEmitter {
|
|
|
2577
2828
|
codeRangeSizeMb
|
|
2578
2829
|
},
|
|
2579
2830
|
inspectorOptions,
|
|
2580
|
-
dirname: this.#root
|
|
2581
|
-
// Keys of the worker environment which only come from an env file of the runtime:
|
|
2582
|
-
// the env file of the application is allowed to override those.
|
|
2583
|
-
envFileFallbackKeys: this.#env[kEnvFileFallbackKeys] ?? []
|
|
2831
|
+
dirname: this.#root
|
|
2584
2832
|
},
|
|
2585
2833
|
argv: applicationConfig.arguments,
|
|
2586
2834
|
execArgv,
|
|
@@ -2604,15 +2852,24 @@ export class Runtime extends EventEmitter {
|
|
|
2604
2852
|
const eventPayload = { application: applicationId, worker: index, workersCount }
|
|
2605
2853
|
|
|
2606
2854
|
worker.once('exit', code => {
|
|
2855
|
+
worker[kWorkerExitCode] = code
|
|
2856
|
+
|
|
2607
2857
|
if (worker[kWorkerStatus] === 'exited') {
|
|
2608
2858
|
return
|
|
2609
2859
|
}
|
|
2610
2860
|
|
|
2611
2861
|
const started = worker[kWorkerStatus] === 'started'
|
|
2862
|
+
|
|
2863
|
+
// A crashed worker leaves its replica empty until the replica is refilled below.
|
|
2864
|
+
// Workers which crash while starting are handled by #startWorker.
|
|
2865
|
+
const replica = started ? this.#getWorkerReplica(applicationId, worker) : undefined
|
|
2866
|
+
if (replica) {
|
|
2867
|
+
replica.worker = null
|
|
2868
|
+
}
|
|
2869
|
+
|
|
2612
2870
|
worker[kWorkerStatus] = 'exited'
|
|
2613
2871
|
this.emitAndNotify('application:worker:exited', eventPayload)
|
|
2614
2872
|
|
|
2615
|
-
const portOffset = this.#workerPortOffsets.get(worker[kFullId]) ?? index
|
|
2616
2873
|
this.#cleanupWorker(worker)
|
|
2617
2874
|
|
|
2618
2875
|
if (this.#status === 'stopping') {
|
|
@@ -2628,8 +2885,9 @@ export class Runtime extends EventEmitter {
|
|
|
2628
2885
|
this.logger.warn(`The ${errorLabel} unexpectedly exited with code ${code}.`)
|
|
2629
2886
|
}
|
|
2630
2887
|
|
|
2631
|
-
//
|
|
2632
|
-
|
|
2888
|
+
// Refill the replica of the worker if it was started. Workers which were not serving a replica anymore
|
|
2889
|
+
// (i.e. being retired after a restart) are not replaced, and a replica being replaced gets the replacement.
|
|
2890
|
+
if (started && this.#status === 'started' && replica && replica.pending !== 'replace') {
|
|
2633
2891
|
if (restartOnError > 0) {
|
|
2634
2892
|
if (restartOnError < IMMEDIATE_RESTART_MAX_THRESHOLD) {
|
|
2635
2893
|
this.logger.warn(`The ${errorLabel} is being restarted ...`)
|
|
@@ -2637,14 +2895,16 @@ export class Runtime extends EventEmitter {
|
|
|
2637
2895
|
this.logger.warn(`The ${errorLabel} will be restarted in ${restartOnError}ms ...`)
|
|
2638
2896
|
}
|
|
2639
2897
|
|
|
2640
|
-
this.#
|
|
2898
|
+
this.#refillReplica(config, applicationConfig, workersCount, applicationId, replica, false, 0).catch(
|
|
2641
2899
|
err => {
|
|
2642
2900
|
this.logger.error({ err: ensureLoggableError(err) }, `${errorLabel} could not be restarted.`)
|
|
2901
|
+
this.#detachReplica(applicationId, replica)
|
|
2643
2902
|
}
|
|
2644
2903
|
)
|
|
2645
2904
|
} else {
|
|
2646
2905
|
this.emitAndNotify('application:worker:unvailable', eventPayload)
|
|
2647
2906
|
this.logger.warn(`The ${errorLabel} is no longer available.`)
|
|
2907
|
+
this.#detachReplica(applicationId, replica)
|
|
2648
2908
|
}
|
|
2649
2909
|
}
|
|
2650
2910
|
})
|
|
@@ -2654,6 +2914,7 @@ export class Runtime extends EventEmitter {
|
|
|
2654
2914
|
worker[kFullId] = workerId
|
|
2655
2915
|
worker[kApplicationId] = applicationId
|
|
2656
2916
|
worker[kWorkerId] = index
|
|
2917
|
+
worker[kWorkerPortOffset] = portOffset
|
|
2657
2918
|
worker[kWorkerStatus] = 'boot'
|
|
2658
2919
|
|
|
2659
2920
|
if (inspectorOptions) {
|
|
@@ -2694,18 +2955,47 @@ export class Runtime extends EventEmitter {
|
|
|
2694
2955
|
// Forward events from the worker
|
|
2695
2956
|
// Do not use emitAndNotify here since we don't want to forward unknown events
|
|
2696
2957
|
worker[kITC].on('event', ({ event, payload }) => {
|
|
2958
|
+
if (event === 'serverOptions') {
|
|
2959
|
+
worker[kWorkerServerOptions] = payload[0]
|
|
2960
|
+
}
|
|
2961
|
+
|
|
2697
2962
|
event = `application:worker:event:${event}`
|
|
2698
2963
|
|
|
2699
2964
|
this.emit(event, ...payload, workerId, applicationId, index)
|
|
2700
2965
|
this.logger.trace({ event, payload, id: workerId, application: applicationId, worker: index }, 'Runtime event')
|
|
2701
2966
|
})
|
|
2702
2967
|
|
|
2968
|
+
worker[kITC].on('compile-cache:flushed', ({ flushed, source }) => {
|
|
2969
|
+
this.emit('application:worker:compile-cache:flushed', {
|
|
2970
|
+
application: applicationId,
|
|
2971
|
+
worker: index,
|
|
2972
|
+
workersCount,
|
|
2973
|
+
flushed,
|
|
2974
|
+
source
|
|
2975
|
+
})
|
|
2976
|
+
})
|
|
2977
|
+
|
|
2978
|
+
worker[kITC].on('compile-cache:unavailable', ({ source }) => {
|
|
2979
|
+
this.emit('application:worker:compile-cache:unavailable', {
|
|
2980
|
+
application: applicationId,
|
|
2981
|
+
worker: index,
|
|
2982
|
+
workersCount,
|
|
2983
|
+
source
|
|
2984
|
+
})
|
|
2985
|
+
})
|
|
2986
|
+
|
|
2703
2987
|
// The worker notifies us when its capability has spawned a child process
|
|
2704
2988
|
// (e.g. Next.js in dev mode). From that point on health metrics must come
|
|
2705
2989
|
// from the child via ITC; for thread-only workers we keep reading the
|
|
2706
2990
|
// handle directly in getWorkerHealth().
|
|
2707
|
-
worker[kITC].on('subprocess:started', () => {
|
|
2991
|
+
worker[kITC].on('subprocess:started', ({ pid } = {}) => {
|
|
2708
2992
|
worker[kIsSubprocessHost] = true
|
|
2993
|
+
worker[kSubprocessPid] = pid
|
|
2994
|
+
})
|
|
2995
|
+
worker[kITC].on('subprocess:exited', ({ pid }) => {
|
|
2996
|
+
if (worker[kSubprocessPid] === pid) {
|
|
2997
|
+
worker[kSubprocessPid] = undefined
|
|
2998
|
+
}
|
|
2709
2999
|
})
|
|
2710
3000
|
|
|
2711
3001
|
worker[kITC].on(openTelemetryITCMessage, resourceMetrics => {
|
|
@@ -2867,9 +3157,7 @@ export class Runtime extends EventEmitter {
|
|
|
2867
3157
|
this.logger.info(`The application "${applicationId}" has been successfully reloaded.`)
|
|
2868
3158
|
this.emitAndNotify('application:worker:reloaded', eventPayload)
|
|
2869
3159
|
|
|
2870
|
-
|
|
2871
|
-
this.#showUrl()
|
|
2872
|
-
}
|
|
3160
|
+
this.#showUrls(this.#workers.getKeys(applicationId))
|
|
2873
3161
|
} catch (e) {
|
|
2874
3162
|
this.logger.error(e)
|
|
2875
3163
|
}
|
|
@@ -2881,25 +3169,6 @@ export class Runtime extends EventEmitter {
|
|
|
2881
3169
|
this.#workers.set(workerId, worker)
|
|
2882
3170
|
}
|
|
2883
3171
|
|
|
2884
|
-
// Setup the interceptor
|
|
2885
|
-
// kInterceptorReadyPromise resolves when the worker
|
|
2886
|
-
// is ready to receive requests: after calling the replaceServer method
|
|
2887
|
-
//
|
|
2888
|
-
// It is stored, not awaited: #startWorker awaits it later, and only if the
|
|
2889
|
-
// worker is actually started. A worker that is discarded or torn down
|
|
2890
|
-
// before that — a replacement abandoned because the runtime stopped
|
|
2891
|
-
// mid-restart, for instance — leaves the promise with nobody to observe
|
|
2892
|
-
// it. Closing the mesh interceptor rejects any routing still in flight
|
|
2893
|
-
// ('The dispatcher has been closed.'), so that is a routine shutdown
|
|
2894
|
-
// outcome, not an exotic one, and it surfaced as an unhandledRejection
|
|
2895
|
-
// that could take the process down before graceful shutdown finished.
|
|
2896
|
-
//
|
|
2897
|
-
// Attaching a no-op handler marks the rejection observed WITHOUT
|
|
2898
|
-
// swallowing it: the await in #startWorker still sees and reports it.
|
|
2899
|
-
const interceptorReady = this.#meshInterceptor.route(applicationId, worker)
|
|
2900
|
-
interceptorReady.catch(() => {})
|
|
2901
|
-
worker[kInterceptorReadyPromise] = interceptorReady
|
|
2902
|
-
|
|
2903
3172
|
// Wait for initialization
|
|
2904
3173
|
try {
|
|
2905
3174
|
await waitEventFromITC(worker, 'init')
|
|
@@ -2918,11 +3187,16 @@ export class Runtime extends EventEmitter {
|
|
|
2918
3187
|
}
|
|
2919
3188
|
|
|
2920
3189
|
this.#workers.delete(workerId)
|
|
2921
|
-
return this.#setupWorker(
|
|
2922
|
-
|
|
2923
|
-
|
|
2924
|
-
|
|
2925
|
-
|
|
3190
|
+
return this.#setupWorker(
|
|
3191
|
+
config,
|
|
3192
|
+
applicationConfig,
|
|
3193
|
+
workersCount,
|
|
3194
|
+
applicationId,
|
|
3195
|
+
index,
|
|
3196
|
+
portOffset,
|
|
3197
|
+
enabled,
|
|
3198
|
+
attempt + 1
|
|
3199
|
+
)
|
|
2926
3200
|
}
|
|
2927
3201
|
|
|
2928
3202
|
worker[kConfig] = { ...applicationConfig, health, workers: workersCount }
|
|
@@ -3339,29 +3613,8 @@ export class Runtime extends EventEmitter {
|
|
|
3339
3613
|
|
|
3340
3614
|
const { url: workerUrl, scheduledTasks } = workerStartResult
|
|
3341
3615
|
worker[kWorkerScheduledTasks] = scheduledTasks
|
|
3342
|
-
if (workerUrl) {
|
|
3343
|
-
this.#url = workerUrl
|
|
3344
|
-
|
|
3345
|
-
// Pin the entrypoint port so that subsequent restarts (especially with
|
|
3346
|
-
// stopBeforeStart when reuseTcpPorts is false) bind to the same port
|
|
3347
|
-
// instead of getting a new random one.
|
|
3348
|
-
if (applicationConfig.entrypoint) {
|
|
3349
|
-
try {
|
|
3350
|
-
const boundPort = Number(new URL(workerUrl).port)
|
|
3351
|
-
if (boundPort) {
|
|
3352
|
-
this.#entrypointPort = boundPort
|
|
3353
|
-
}
|
|
3354
|
-
} catch {
|
|
3355
|
-
// URL parsing failed, leave unchanged
|
|
3356
|
-
}
|
|
3357
|
-
}
|
|
3358
|
-
}
|
|
3359
3616
|
|
|
3360
|
-
|
|
3361
|
-
const interceptorResult = await executeWithTimeout(worker[kInterceptorReadyPromise], config.startTimeout)
|
|
3362
|
-
if (interceptorResult === kTimeout) {
|
|
3363
|
-
throw new WorkerInterceptorJoinTimeoutError(label, config.startTimeout)
|
|
3364
|
-
}
|
|
3617
|
+
this.#recordWorkerUrl(worker, id, workerUrl)
|
|
3365
3618
|
|
|
3366
3619
|
worker[kWorkerStatus] = 'started'
|
|
3367
3620
|
worker[kWorkerStartTime] = Date.now()
|
|
@@ -3385,12 +3638,28 @@ export class Runtime extends EventEmitter {
|
|
|
3385
3638
|
).unref()
|
|
3386
3639
|
}
|
|
3387
3640
|
} catch (err) {
|
|
3388
|
-
|
|
3641
|
+
let error = ensureError(err)
|
|
3642
|
+
// Forced shutdown may terminate the thread before its start handler can report the abort.
|
|
3643
|
+
if (
|
|
3644
|
+
worker[kWorkerStartAborted] &&
|
|
3645
|
+
error.code === 'PLT_RUNTIME_APPLICATION_WORKER_EXIT'
|
|
3646
|
+
) {
|
|
3647
|
+
error = new RuntimeAbortedError({ cause: error })
|
|
3648
|
+
}
|
|
3389
3649
|
worker[kITC].notify('application:worker:start:processed')
|
|
3390
|
-
const portOffset = this.#workerPortOffsets.get(worker[kFullId]) ?? index
|
|
3391
3650
|
|
|
3392
|
-
|
|
3393
|
-
|
|
3651
|
+
if (error.code === 'EADDRINUSE' && Number.isInteger(Number(error.port))) {
|
|
3652
|
+
const port = Number(error.port)
|
|
3653
|
+
const owner = this.#getPortOwner(port, id)
|
|
3654
|
+
|
|
3655
|
+
if (owner) {
|
|
3656
|
+
error = new AddressInUseError(port, owner, id)
|
|
3657
|
+
} else if (this.#getPortOwner(port, id, undefined, true)) {
|
|
3658
|
+
error = new WorkerAddressInUseError(port, id)
|
|
3659
|
+
}
|
|
3660
|
+
// Otherwise the port is used by an external process: keep the original error, which already describes it
|
|
3661
|
+
}
|
|
3662
|
+
if (error.code === 'EACCES') throw error
|
|
3394
3663
|
|
|
3395
3664
|
this.#cleanupWorker(worker)
|
|
3396
3665
|
|
|
@@ -3414,7 +3683,16 @@ export class Runtime extends EventEmitter {
|
|
|
3414
3683
|
|
|
3415
3684
|
const restartOnError = this.#getApplicationRestartOnError(config, applicationConfig)
|
|
3416
3685
|
|
|
3417
|
-
if (
|
|
3686
|
+
if (
|
|
3687
|
+
disableRestartAttempts ||
|
|
3688
|
+
!restartOnError ||
|
|
3689
|
+
error.code === 'EACCES' ||
|
|
3690
|
+
error.code === 'EADDRINUSE' ||
|
|
3691
|
+
error.code === 'EADDRNOTAVAIL' ||
|
|
3692
|
+
error.code === 'PLT_RUNTIME_EADDR_IN_USE' ||
|
|
3693
|
+
error.code === 'PLT_RUNTIME_RUNTIME_ABORT' ||
|
|
3694
|
+
error.code === 'PLT_RUNTIME_WORKER_EADDR_IN_USE'
|
|
3695
|
+
) {
|
|
3418
3696
|
throw error
|
|
3419
3697
|
}
|
|
3420
3698
|
|
|
@@ -3434,7 +3712,64 @@ export class Runtime extends EventEmitter {
|
|
|
3434
3712
|
)
|
|
3435
3713
|
}
|
|
3436
3714
|
|
|
3437
|
-
|
|
3715
|
+
// The replica was removed while the worker was starting, there is nothing to refill
|
|
3716
|
+
const replica = this.#getWorkerReplica(id, worker)
|
|
3717
|
+
if (!replica) {
|
|
3718
|
+
throw error
|
|
3719
|
+
}
|
|
3720
|
+
replica.worker = null
|
|
3721
|
+
await this.#refillReplica(config, applicationConfig, workersCount, id, replica, silent, bootstrapAttempt)
|
|
3722
|
+
}
|
|
3723
|
+
}
|
|
3724
|
+
|
|
3725
|
+
async #startFirstWorkerWithCompileCache (config, applicationConfig, workers, id, silent) {
|
|
3726
|
+
const cacheFlushEvent = 'application:worker:compile-cache:flushed'
|
|
3727
|
+
const cacheUnavailableEvent = 'application:worker:compile-cache:unavailable'
|
|
3728
|
+
const { promise, resolve } = Promise.withResolvers()
|
|
3729
|
+
const cacheEvents = []
|
|
3730
|
+
const runtime = this
|
|
3731
|
+
let expectedSource
|
|
3732
|
+
|
|
3733
|
+
function removeCacheFlushListener () {
|
|
3734
|
+
runtime.off(cacheFlushEvent, cacheFlushListener)
|
|
3735
|
+
runtime.off(cacheUnavailableEvent, cacheUnavailableListener)
|
|
3736
|
+
}
|
|
3737
|
+
|
|
3738
|
+
function resolveIfReady () {
|
|
3739
|
+
if (expectedSource && cacheEvents.some(function (event) {
|
|
3740
|
+
return event.source === expectedSource
|
|
3741
|
+
})) {
|
|
3742
|
+
removeCacheFlushListener()
|
|
3743
|
+
resolve()
|
|
3744
|
+
}
|
|
3745
|
+
}
|
|
3746
|
+
|
|
3747
|
+
function cacheFlushListener (event) {
|
|
3748
|
+
if (event.application === id && event.worker === 0) {
|
|
3749
|
+
cacheEvents.push(event)
|
|
3750
|
+
resolveIfReady()
|
|
3751
|
+
}
|
|
3752
|
+
}
|
|
3753
|
+
|
|
3754
|
+
function cacheUnavailableListener (event) {
|
|
3755
|
+
if (event.application === id && event.worker === 0) {
|
|
3756
|
+
cacheEvents.push(event)
|
|
3757
|
+
resolveIfReady()
|
|
3758
|
+
}
|
|
3759
|
+
}
|
|
3760
|
+
|
|
3761
|
+
runtime.on(cacheFlushEvent, cacheFlushListener)
|
|
3762
|
+
runtime.on(cacheUnavailableEvent, cacheUnavailableListener)
|
|
3763
|
+
|
|
3764
|
+
try {
|
|
3765
|
+
await this.#startWorker(config, applicationConfig, workers, id, 0, silent)
|
|
3766
|
+
const firstWorker = this.#workers.get(`${id}:0`)
|
|
3767
|
+
expectedSource = firstWorker?.[kIsSubprocessHost] === true ? 'child-process' : 'worker'
|
|
3768
|
+
resolveIfReady()
|
|
3769
|
+
await promise
|
|
3770
|
+
} catch (err) {
|
|
3771
|
+
removeCacheFlushListener()
|
|
3772
|
+
throw err
|
|
3438
3773
|
}
|
|
3439
3774
|
}
|
|
3440
3775
|
|
|
@@ -3454,6 +3789,9 @@ export class Runtime extends EventEmitter {
|
|
|
3454
3789
|
|
|
3455
3790
|
const eventPayload = { application: id, worker: index, workersCount }
|
|
3456
3791
|
|
|
3792
|
+
if (worker[kWorkerStatus] === 'starting' && this.error) {
|
|
3793
|
+
worker[kWorkerStartAborted] = true
|
|
3794
|
+
}
|
|
3457
3795
|
worker[kWorkerStatus] = 'stopping'
|
|
3458
3796
|
worker[kITC].removeAllListeners('changed')
|
|
3459
3797
|
this.emitAndNotify('application:worker:stopping', eventPayload)
|
|
@@ -3469,13 +3807,18 @@ export class Runtime extends EventEmitter {
|
|
|
3469
3807
|
|
|
3470
3808
|
// Always send the stop message, it will shut down workers that only had ITC and interceptors setup
|
|
3471
3809
|
try {
|
|
3472
|
-
const res = await executeWithTimeout(
|
|
3810
|
+
const res = await executeWithTimeout(
|
|
3811
|
+
sendViaITC(worker, 'stop', { force: !!this.error, dependents, shutdownTimeout: exitTimeout }),
|
|
3812
|
+
exitTimeout
|
|
3813
|
+
)
|
|
3473
3814
|
|
|
3474
3815
|
if (res === kTimeout) {
|
|
3816
|
+
this.#terminateSubprocess(worker)
|
|
3475
3817
|
this.emitAndNotify('application:worker:stop:timeout', eventPayload)
|
|
3476
3818
|
this.logger.error(`Timeout while stopping ${label}. Killing a worker thread.`)
|
|
3477
3819
|
}
|
|
3478
3820
|
} catch (error) {
|
|
3821
|
+
this.#terminateSubprocess(worker)
|
|
3479
3822
|
this.emitAndNotify('application:worker:stop:error', eventPayload)
|
|
3480
3823
|
this.logger.error({ err: ensureLoggableError(error) }, `Failed to stop ${label}. Killing a worker thread.`)
|
|
3481
3824
|
} finally {
|
|
@@ -3503,13 +3846,32 @@ export class Runtime extends EventEmitter {
|
|
|
3503
3846
|
await this.#avoidOutOfOrderThreadLogs()
|
|
3504
3847
|
|
|
3505
3848
|
worker[kWorkerStatus] = 'stopped'
|
|
3849
|
+
worker[kWorkerUrl] = undefined
|
|
3506
3850
|
this.emitAndNotify('application:worker:stopped', eventPayload)
|
|
3507
3851
|
this.#broadcastWorkers()
|
|
3508
3852
|
}
|
|
3509
3853
|
|
|
3854
|
+
#terminateSubprocess (worker) {
|
|
3855
|
+
const pid = worker[kSubprocessPid]
|
|
3856
|
+
if (!Number.isInteger(pid) || pid <= 0) {
|
|
3857
|
+
return
|
|
3858
|
+
}
|
|
3859
|
+
try {
|
|
3860
|
+
// The main thread owns the fallback when a worker cannot supervise its child.
|
|
3861
|
+
this.logger.debug({ pid }, 'Terminating application child process from the runtime.')
|
|
3862
|
+
process.kill(pid, 'SIGKILL')
|
|
3863
|
+
} catch (error) {
|
|
3864
|
+
if (error.code === 'ESRCH') {
|
|
3865
|
+
// ESRCH means no such process: the child exited before the fallback signal arrived.
|
|
3866
|
+
this.logger.debug({ pid }, 'Application child process already exited before termination.')
|
|
3867
|
+
} else {
|
|
3868
|
+
this.logger.error({ err: ensureLoggableError(error) }, 'Failed to terminate application child process.')
|
|
3869
|
+
}
|
|
3870
|
+
}
|
|
3871
|
+
}
|
|
3872
|
+
|
|
3510
3873
|
#cleanupWorker (worker) {
|
|
3511
3874
|
clearTimeout(worker[kHealthCheckTimer])
|
|
3512
|
-
this.#workerPortOffsets.delete(worker[kFullId])
|
|
3513
3875
|
|
|
3514
3876
|
const currentWorker = this.#workers.get(worker[kFullId])
|
|
3515
3877
|
|
|
@@ -3521,8 +3883,8 @@ export class Runtime extends EventEmitter {
|
|
|
3521
3883
|
}
|
|
3522
3884
|
|
|
3523
3885
|
async #discardWorker (worker) {
|
|
3524
|
-
|
|
3525
|
-
worker
|
|
3886
|
+
// Marked synchronously, so that the exit handler ignores the worker and nobody else tries to stop it
|
|
3887
|
+
worker[kWorkerStatus] = 'exited'
|
|
3526
3888
|
await worker.terminate()
|
|
3527
3889
|
|
|
3528
3890
|
return this.#cleanupWorker(worker)
|
|
@@ -3538,6 +3900,57 @@ export class Runtime extends EventEmitter {
|
|
|
3538
3900
|
return index
|
|
3539
3901
|
}
|
|
3540
3902
|
|
|
3903
|
+
#getWorkerReplica (applicationId, worker) {
|
|
3904
|
+
return this.#replicas.get(applicationId)?.find(replica => replica.worker === worker)
|
|
3905
|
+
}
|
|
3906
|
+
|
|
3907
|
+
// Removes a replica and cancels the work in progress for it. Stopping its worker is up to the caller.
|
|
3908
|
+
async #detachReplica (applicationId, replica) {
|
|
3909
|
+
const replicas = this.#replicas.get(applicationId)
|
|
3910
|
+
const position = replicas?.indexOf(replica) ?? -1
|
|
3911
|
+
|
|
3912
|
+
if (position !== -1) {
|
|
3913
|
+
replicas.splice(position, 1)
|
|
3914
|
+
}
|
|
3915
|
+
|
|
3916
|
+
replica.abortController.abort()
|
|
3917
|
+
|
|
3918
|
+
if (replica.replacement) {
|
|
3919
|
+
await this.#discardWorker(replica.replacement)
|
|
3920
|
+
}
|
|
3921
|
+
}
|
|
3922
|
+
|
|
3923
|
+
// Returns the lowest port offset which is not used by any replica of the application.
|
|
3924
|
+
// Empty replicas keep their port offset, as they will be refilled.
|
|
3925
|
+
#getNextReplicaPortOffset (applicationId) {
|
|
3926
|
+
const used = new Set(this.#replicas.get(applicationId)?.map(replica => replica.portOffset))
|
|
3927
|
+
|
|
3928
|
+
let offset = 0
|
|
3929
|
+
while (used.has(offset)) {
|
|
3930
|
+
offset++
|
|
3931
|
+
}
|
|
3932
|
+
|
|
3933
|
+
return offset
|
|
3934
|
+
}
|
|
3935
|
+
|
|
3936
|
+
// Returns true if the application of the worker is configured to listen on a fixed port shared by all its workers
|
|
3937
|
+
async #usesSharedFixedPort (worker) {
|
|
3938
|
+
let server
|
|
3939
|
+
|
|
3940
|
+
try {
|
|
3941
|
+
server = (await sendViaITC(worker, 'getApplicationConfig'))?.server
|
|
3942
|
+
} catch {
|
|
3943
|
+
return false
|
|
3944
|
+
}
|
|
3945
|
+
|
|
3946
|
+
if (!server || server.portAssignment === 'perWorkerIncrement') {
|
|
3947
|
+
return false
|
|
3948
|
+
}
|
|
3949
|
+
|
|
3950
|
+
const port = Number(server.port)
|
|
3951
|
+
return Number.isInteger(port) && port > 0
|
|
3952
|
+
}
|
|
3953
|
+
|
|
3541
3954
|
// Returns the effective restartOnError value for an application: the application-level
|
|
3542
3955
|
// value, when defined, takes precedence over the runtime-level one.
|
|
3543
3956
|
// The value is normalized to a number: false becomes 0 (never restart),
|
|
@@ -3554,89 +3967,101 @@ export class Runtime extends EventEmitter {
|
|
|
3554
3967
|
return restartOnError
|
|
3555
3968
|
}
|
|
3556
3969
|
|
|
3557
|
-
|
|
3970
|
+
// Starts a new worker for a replica whose worker crashed (or failed to start), after the restartOnError delay
|
|
3971
|
+
async #refillReplica (config, applicationConfig, workersCount, id, replica, silent, bootstrapAttempt) {
|
|
3558
3972
|
const restartOnError = this.#getApplicationRestartOnError(config, applicationConfig)
|
|
3973
|
+
const { signal } = replica.abortController
|
|
3974
|
+
replica.pending = 'refill'
|
|
3559
3975
|
|
|
3560
|
-
|
|
3561
|
-
|
|
3562
|
-
|
|
3563
|
-
|
|
3564
|
-
|
|
3565
|
-
|
|
3566
|
-
return
|
|
3567
|
-
}
|
|
3568
|
-
|
|
3569
|
-
restartPromise = new Promise((resolve, reject) => {
|
|
3570
|
-
async function restart () {
|
|
3571
|
-
this.#restartingWorkers.delete(restartKey)
|
|
3572
|
-
|
|
3573
|
-
// If some processes were scheduled to restart
|
|
3574
|
-
// but the runtime is stopped, ignore it
|
|
3575
|
-
if (!this.#status.startsWith('start')) {
|
|
3576
|
-
return
|
|
3976
|
+
try {
|
|
3977
|
+
try {
|
|
3978
|
+
if (restartOnError < IMMEDIATE_RESTART_MAX_THRESHOLD) {
|
|
3979
|
+
await new Promise(resolve => process.nextTick(resolve))
|
|
3980
|
+
} else {
|
|
3981
|
+
await sleep(restartOnError, undefined, { signal })
|
|
3577
3982
|
}
|
|
3983
|
+
} catch {
|
|
3984
|
+
// The replica was removed in the meanwhile
|
|
3985
|
+
return
|
|
3986
|
+
}
|
|
3578
3987
|
|
|
3579
|
-
|
|
3580
|
-
|
|
3581
|
-
|
|
3582
|
-
|
|
3988
|
+
// If the replica was removed or the runtime was stopped in the meanwhile, ignore it
|
|
3989
|
+
if (signal.aborted || !this.#status.startsWith('start')) {
|
|
3990
|
+
return
|
|
3991
|
+
}
|
|
3583
3992
|
|
|
3584
|
-
|
|
3585
|
-
|
|
3586
|
-
await this.#startWorker(config, applicationConfig, workersCount, id, newIndex, silent, bootstrapAttempt)
|
|
3587
|
-
this.#incrementApplicationRestartCount(id)
|
|
3993
|
+
// Get a new unique index for the worker, which inherits the port offset of the replica
|
|
3994
|
+
const newIndex = this.#getNextWorkerIndex(id)
|
|
3588
3995
|
|
|
3589
|
-
|
|
3590
|
-
|
|
3591
|
-
|
|
3592
|
-
|
|
3593
|
-
|
|
3594
|
-
|
|
3996
|
+
try {
|
|
3997
|
+
const worker = await this.#setupWorker(
|
|
3998
|
+
config,
|
|
3999
|
+
applicationConfig,
|
|
4000
|
+
workersCount,
|
|
4001
|
+
id,
|
|
4002
|
+
newIndex,
|
|
4003
|
+
replica.portOffset
|
|
4004
|
+
)
|
|
3595
4005
|
|
|
3596
|
-
|
|
3597
|
-
|
|
3598
|
-
|
|
3599
|
-
|
|
4006
|
+
if (signal.aborted) {
|
|
4007
|
+
await this.#discardWorker(worker)
|
|
4008
|
+
return
|
|
4009
|
+
}
|
|
3600
4010
|
|
|
3601
|
-
|
|
4011
|
+
replica.worker = worker
|
|
4012
|
+
await this.#startWorker(config, applicationConfig, workersCount, id, newIndex, silent, bootstrapAttempt, worker)
|
|
4013
|
+
} catch (err) {
|
|
4014
|
+
// The replica was removed or the runtime was stopped while the worker was starting, ignore any error.
|
|
4015
|
+
if (signal.aborted || !this.#status.startsWith('start')) {
|
|
4016
|
+
return
|
|
3602
4017
|
}
|
|
3603
|
-
}
|
|
3604
4018
|
|
|
3605
|
-
|
|
3606
|
-
process.nextTick(restart.bind(this))
|
|
3607
|
-
} else {
|
|
3608
|
-
setTimeout(restart.bind(this), restartOnError)
|
|
4019
|
+
throw err
|
|
3609
4020
|
}
|
|
3610
|
-
})
|
|
3611
4021
|
|
|
3612
|
-
|
|
3613
|
-
|
|
4022
|
+
this.#incrementApplicationRestartCount(id)
|
|
4023
|
+
this.logger.info(`The ${this.#workerExtendedLabel(id, newIndex, workersCount)} has been successfully restarted ...`)
|
|
4024
|
+
} finally {
|
|
4025
|
+
if (replica.pending === 'refill') {
|
|
4026
|
+
replica.pending = null
|
|
4027
|
+
}
|
|
4028
|
+
}
|
|
3614
4029
|
}
|
|
3615
4030
|
|
|
3616
|
-
|
|
4031
|
+
// Replaces the worker of a replica with a new one. Only one operation at a time is performed on a replica.
|
|
4032
|
+
async #replaceWorker (
|
|
4033
|
+
config,
|
|
4034
|
+
applicationConfig,
|
|
4035
|
+
workersCount,
|
|
4036
|
+
applicationId,
|
|
4037
|
+
oldIndex,
|
|
4038
|
+
worker,
|
|
4039
|
+
silent,
|
|
4040
|
+
deferOldWorkerRetirement = false
|
|
4041
|
+
) {
|
|
4042
|
+
const replica = this.#getWorkerReplica(applicationId, worker)
|
|
4043
|
+
|
|
4044
|
+
// The worker is not serving a replica anymore, or the replica is already being refilled or replaced
|
|
4045
|
+
if (!replica || replica.pending) {
|
|
4046
|
+
return
|
|
4047
|
+
}
|
|
4048
|
+
|
|
4049
|
+
const { signal } = replica.abortController
|
|
3617
4050
|
const oldLabel = this.#workerExtendedLabel(applicationId, oldIndex, workersCount)
|
|
3618
4051
|
let newWorker
|
|
4052
|
+
let error
|
|
3619
4053
|
|
|
3620
|
-
// Get a new unique index for the replacement worker
|
|
4054
|
+
// Get a new unique index for the replacement worker, which inherits the port offset of the replica
|
|
3621
4055
|
const newIndex = this.#getNextWorkerIndex(applicationId)
|
|
3622
4056
|
const newWorkerId = `${applicationId}:${newIndex}`
|
|
3623
4057
|
const newLabel = this.#workerExtendedLabel(applicationId, newIndex, workersCount)
|
|
3624
|
-
const portOffset = this.#workerPortOffsets.get(`${applicationId}:${oldIndex}`)
|
|
3625
|
-
this.#workerPortOffsets.set(newWorkerId, portOffset)
|
|
3626
4058
|
|
|
3627
4059
|
const stopBeforeStart =
|
|
3628
|
-
|
|
4060
|
+
Boolean(worker[kWorkerUrl]) &&
|
|
4061
|
+
worker[kWorkerServerOptions]?.port !== 0 &&
|
|
3629
4062
|
(config.reuseTcpPorts === false || applicationConfig.reuseTcpPorts === false || !features.node.reusePort)
|
|
3630
4063
|
|
|
3631
|
-
|
|
3632
|
-
// port in the config so the replacement worker binds to the same port.
|
|
3633
|
-
// We only do this for stopBeforeStart because when reusePort is available
|
|
3634
|
-
// the new worker starts alongside the old one and must use port 0 to avoid
|
|
3635
|
-
// SO_REUSEPORT routing requests to the stale old worker.
|
|
3636
|
-
let configForNewWorker = config
|
|
3637
|
-
if (stopBeforeStart && this.#entrypointPort && config.server) {
|
|
3638
|
-
configForNewWorker = { ...config, server: { ...config.server, port: this.#entrypointPort } }
|
|
3639
|
-
}
|
|
4064
|
+
replica.pending = 'replace'
|
|
3640
4065
|
|
|
3641
4066
|
try {
|
|
3642
4067
|
if (!silent) {
|
|
@@ -3645,13 +4070,22 @@ export class Runtime extends EventEmitter {
|
|
|
3645
4070
|
|
|
3646
4071
|
if (stopBeforeStart) {
|
|
3647
4072
|
await this.#removeWorker(workersCount, applicationId, oldIndex, worker, silent, oldLabel)
|
|
4073
|
+
replica.worker = null
|
|
3648
4074
|
}
|
|
3649
4075
|
|
|
3650
|
-
|
|
3651
|
-
|
|
4076
|
+
newWorker = await this.#setupWorker(
|
|
4077
|
+
config,
|
|
4078
|
+
applicationConfig,
|
|
4079
|
+
workersCount,
|
|
4080
|
+
applicationId,
|
|
4081
|
+
newIndex,
|
|
4082
|
+
replica.portOffset,
|
|
4083
|
+
false
|
|
4084
|
+
)
|
|
4085
|
+
replica.replacement = newWorker
|
|
3652
4086
|
|
|
3653
|
-
// Make sure the runtime hasn't been stopped in the meanwhile
|
|
3654
|
-
if (this.#status !== 'started') {
|
|
4087
|
+
// Make sure the runtime hasn't been stopped and the replica hasn't been removed in the meanwhile
|
|
4088
|
+
if (this.#status !== 'started' || signal.aborted) {
|
|
3655
4089
|
return this.#discardWorker(newWorker)
|
|
3656
4090
|
}
|
|
3657
4091
|
|
|
@@ -3662,7 +4096,7 @@ export class Runtime extends EventEmitter {
|
|
|
3662
4096
|
|
|
3663
4097
|
// Add the worker to the mesh
|
|
3664
4098
|
await this.#startWorker(
|
|
3665
|
-
|
|
4099
|
+
config,
|
|
3666
4100
|
applicationConfig,
|
|
3667
4101
|
workersCount,
|
|
3668
4102
|
applicationId,
|
|
@@ -3673,27 +4107,68 @@ export class Runtime extends EventEmitter {
|
|
|
3673
4107
|
true
|
|
3674
4108
|
)
|
|
3675
4109
|
|
|
3676
|
-
// Make sure the runtime hasn't been stopped in the meanwhile
|
|
3677
|
-
if (this.#status !== 'started') {
|
|
4110
|
+
// Make sure the runtime hasn't been stopped and the replica hasn't been removed in the meanwhile
|
|
4111
|
+
if (this.#status !== 'started' || signal.aborted) {
|
|
3678
4112
|
return this.#discardWorker(newWorker)
|
|
3679
4113
|
}
|
|
3680
4114
|
} catch (e) {
|
|
3681
|
-
|
|
4115
|
+
error = e
|
|
4116
|
+
} finally {
|
|
4117
|
+
replica.pending = null
|
|
4118
|
+
replica.replacement = null
|
|
4119
|
+
}
|
|
3682
4120
|
|
|
4121
|
+
if (error) {
|
|
3683
4122
|
if (this.#workers.get(newWorkerId) === newWorker) {
|
|
3684
4123
|
this.#workers.delete(newWorkerId)
|
|
3685
4124
|
}
|
|
3686
4125
|
|
|
3687
4126
|
newWorker?.terminate?.()
|
|
3688
|
-
|
|
4127
|
+
|
|
4128
|
+
// The replica was removed in the meanwhile, the new worker was meant to be discarded anyway
|
|
4129
|
+
if (signal.aborted) {
|
|
4130
|
+
return
|
|
4131
|
+
}
|
|
4132
|
+
|
|
4133
|
+
// The replaced worker crashed (or was stopped before starting the new one): the replica must be refilled
|
|
4134
|
+
if (!replica.worker && this.#getApplicationRestartOnError(config, applicationConfig) > 0) {
|
|
4135
|
+
this.#refillReplica(config, applicationConfig, workersCount, applicationId, replica, true, 0).catch(err => {
|
|
4136
|
+
this.logger.error({ err: ensureLoggableError(err) }, `${oldLabel} could not be restarted.`)
|
|
4137
|
+
this.#detachReplica(applicationId, replica)
|
|
4138
|
+
})
|
|
4139
|
+
}
|
|
4140
|
+
|
|
4141
|
+
throw error
|
|
3689
4142
|
}
|
|
3690
4143
|
|
|
4144
|
+
// The replaced worker might have crashed in the meanwhile, in which case it is not in the replica anymore
|
|
4145
|
+
replica.worker = newWorker
|
|
4146
|
+
|
|
3691
4147
|
if (!stopBeforeStart) {
|
|
4148
|
+
if (deferOldWorkerRetirement) {
|
|
4149
|
+
return {
|
|
4150
|
+
discard: async () => {
|
|
4151
|
+
// Roll back: the replaced worker serves the replica again
|
|
4152
|
+
if (replica.worker === newWorker) {
|
|
4153
|
+
replica.worker = typeof worker[kWorkerExitCode] === 'number' ? null : worker
|
|
4154
|
+
}
|
|
4155
|
+
|
|
4156
|
+
await this.#discardWorker(newWorker)
|
|
4157
|
+
},
|
|
4158
|
+
retire: () => this.#removeWorker(workersCount, applicationId, oldIndex, worker, silent, oldLabel)
|
|
4159
|
+
}
|
|
4160
|
+
}
|
|
4161
|
+
|
|
3692
4162
|
await this.#removeWorker(workersCount, applicationId, oldIndex, worker, silent, oldLabel)
|
|
3693
4163
|
}
|
|
3694
4164
|
}
|
|
3695
4165
|
|
|
3696
4166
|
async #removeWorker (workersCount, applicationId, index, worker, silent, label) {
|
|
4167
|
+
// The worker crashed in the meanwhile, there is nothing left to remove
|
|
4168
|
+
if (typeof worker[kWorkerExitCode] === 'number') {
|
|
4169
|
+
return
|
|
4170
|
+
}
|
|
4171
|
+
|
|
3697
4172
|
if (!silent) {
|
|
3698
4173
|
this.logger.debug(`Preparing to stop the old version of ${label} ...`)
|
|
3699
4174
|
}
|
|
@@ -3759,7 +4234,13 @@ export class Runtime extends EventEmitter {
|
|
|
3759
4234
|
|
|
3760
4235
|
// Note that in this class "== null" is purposely used instead of "===" to check for both null and undefined
|
|
3761
4236
|
if (workerId == null) {
|
|
3762
|
-
|
|
4237
|
+
// During restarts and scaling the application also has workers which are still booting or being stopped:
|
|
4238
|
+
// only pick a worker which can serve the call. Booting workers cannot handle ITC calls yet,
|
|
4239
|
+
// stopping ones might exit before replying.
|
|
4240
|
+
worker = this.#workers.next(applicationId, worker => {
|
|
4241
|
+
const status = worker[kWorkerStatus]
|
|
4242
|
+
return status === 'started' || (!ensureStarted && (status === 'init' || status === 'starting'))
|
|
4243
|
+
})
|
|
3763
4244
|
} else {
|
|
3764
4245
|
worker = this.#workers.get(`${applicationId}:${workerId}`)
|
|
3765
4246
|
}
|
|
@@ -3771,6 +4252,11 @@ export class Runtime extends EventEmitter {
|
|
|
3771
4252
|
return null
|
|
3772
4253
|
}
|
|
3773
4254
|
|
|
4255
|
+
// The application has workers, but they are all booting or being stopped
|
|
4256
|
+
if (workerId == null && this.#workers.getKeys(applicationId).length > 0) {
|
|
4257
|
+
throw new ApplicationNotStartedError(applicationId)
|
|
4258
|
+
}
|
|
4259
|
+
|
|
3774
4260
|
if (applicationsIds.includes(applicationId)) {
|
|
3775
4261
|
const availableWorkers = this.#workers
|
|
3776
4262
|
.getKeys(applicationId)
|
|
@@ -3864,11 +4350,20 @@ export class Runtime extends EventEmitter {
|
|
|
3864
4350
|
}
|
|
3865
4351
|
|
|
3866
4352
|
async #getRuntimePackageJson () {
|
|
3867
|
-
const
|
|
3868
|
-
|
|
3869
|
-
|
|
3870
|
-
|
|
3871
|
-
|
|
4353
|
+
const packageJsonPath = join(this.#root, 'package.json')
|
|
4354
|
+
|
|
4355
|
+
/*
|
|
4356
|
+
A project need not have one, and a runtime that cannot report its metadata without one is a
|
|
4357
|
+
runtime whose `applications:add --save` stops working -- that command reads `projectDir`,
|
|
4358
|
+
`configPath` and `autoload` from this, and has done since `GET /config` was removed. The
|
|
4359
|
+
metadata already declares `packageName` and `packageVersion` as nullable, so absence is a
|
|
4360
|
+
state its shape allows.
|
|
4361
|
+
*/
|
|
4362
|
+
try {
|
|
4363
|
+
return JSON.parse(await readFile(packageJsonPath, 'utf8'))
|
|
4364
|
+
} catch {
|
|
4365
|
+
return {}
|
|
4366
|
+
}
|
|
3872
4367
|
}
|
|
3873
4368
|
|
|
3874
4369
|
#handleWorkerStandardStreams (worker, applicationId, workerId) {
|
|
@@ -4217,16 +4712,21 @@ export class Runtime extends EventEmitter {
|
|
|
4217
4712
|
await this.#updateApplicationConfigHealth(applicationId, health)
|
|
4218
4713
|
}
|
|
4219
4714
|
|
|
4220
|
-
//
|
|
4221
|
-
|
|
4222
|
-
for (const
|
|
4223
|
-
|
|
4715
|
+
// A copy: while the loop awaits, scale-downs remove replicas from the list, which would shift the remaining ones
|
|
4716
|
+
// and skip some. Replicas added in the meanwhile (scale-up) already use the new health config.
|
|
4717
|
+
for (const replica of [...(this.#replicas.get(applicationId) ?? [])]) {
|
|
4718
|
+
// The worker crashed, the replica is refilled with a fresh worker which uses the new health config
|
|
4719
|
+
const worker = replica.worker
|
|
4720
|
+
if (!worker) {
|
|
4721
|
+
continue
|
|
4722
|
+
}
|
|
4723
|
+
|
|
4724
|
+
const workerIndex = worker[kWorkerId]
|
|
4224
4725
|
this.logger.info(
|
|
4225
4726
|
{ health: { current: currentHealth, new: health } },
|
|
4226
4727
|
`Restarting application "${applicationId}" worker ${workerIndex} to update config health heap...`
|
|
4227
4728
|
)
|
|
4228
4729
|
|
|
4229
|
-
const worker = this.#workers.get(workerKey)
|
|
4230
4730
|
if (health.maxHeapTotal) {
|
|
4231
4731
|
worker[kConfig].health.maxHeapTotal = health.maxHeapTotal
|
|
4232
4732
|
}
|
|
@@ -4269,24 +4769,51 @@ export class Runtime extends EventEmitter {
|
|
|
4269
4769
|
|
|
4270
4770
|
if (currentWorkers < workers) {
|
|
4271
4771
|
report.started = []
|
|
4272
|
-
|
|
4772
|
+
|
|
4773
|
+
let pendingReplica
|
|
4273
4774
|
|
|
4274
4775
|
try {
|
|
4275
4776
|
for (let i = currentWorkers; i < workers; i++) {
|
|
4276
4777
|
const newIndex = this.#getNextWorkerIndex(applicationId)
|
|
4277
|
-
|
|
4278
|
-
|
|
4778
|
+
pendingReplica = await this.#setupReplica(
|
|
4779
|
+
config,
|
|
4780
|
+
applicationConfig,
|
|
4781
|
+
workers,
|
|
4782
|
+
applicationId,
|
|
4783
|
+
newIndex,
|
|
4784
|
+
this.#getNextReplicaPortOffset(applicationId)
|
|
4785
|
+
)
|
|
4786
|
+
|
|
4787
|
+
// The replica was removed in the meanwhile (i.e. scale-down or application stopped)
|
|
4788
|
+
if (!pendingReplica) {
|
|
4789
|
+
continue
|
|
4790
|
+
}
|
|
4279
4791
|
|
|
4280
|
-
await this.#
|
|
4281
|
-
|
|
4792
|
+
await this.#startWorker(
|
|
4793
|
+
config,
|
|
4794
|
+
applicationConfig,
|
|
4795
|
+
workers,
|
|
4796
|
+
applicationId,
|
|
4797
|
+
newIndex,
|
|
4798
|
+
false,
|
|
4799
|
+
0,
|
|
4800
|
+
pendingReplica.worker
|
|
4801
|
+
)
|
|
4282
4802
|
|
|
4283
|
-
|
|
4803
|
+
pendingReplica = undefined
|
|
4284
4804
|
report.started.push(newIndex)
|
|
4285
4805
|
startedWorkersCount++
|
|
4286
4806
|
}
|
|
4807
|
+
|
|
4808
|
+
// A worker added after boot re-runs the application's code, so it can answer differently
|
|
4809
|
+
// from the workers already running. Scale-up is the second place the answers must agree.
|
|
4810
|
+
await this.#collectServingState(applicationId)
|
|
4287
4811
|
report.success = true
|
|
4288
4812
|
} catch (err) {
|
|
4289
|
-
|
|
4813
|
+
// The worker of the replica could not be started
|
|
4814
|
+
if (pendingReplica && !pendingReplica.worker) {
|
|
4815
|
+
await this.#detachReplica(applicationId, pendingReplica)
|
|
4816
|
+
}
|
|
4290
4817
|
|
|
4291
4818
|
if (startedWorkersCount < 1) {
|
|
4292
4819
|
this.logger.error({ err }, 'Cannot start application workers, no worker started')
|
|
@@ -4303,31 +4830,30 @@ export class Runtime extends EventEmitter {
|
|
|
4303
4830
|
report.stopped = []
|
|
4304
4831
|
try {
|
|
4305
4832
|
const workersToStop = currentWorkers - workers
|
|
4306
|
-
|
|
4307
|
-
|
|
4308
|
-
|
|
4309
|
-
|
|
4310
|
-
|
|
4311
|
-
|
|
4312
|
-
|
|
4313
|
-
|
|
4314
|
-
|
|
4315
|
-
|
|
4316
|
-
|
|
4317
|
-
|
|
4318
|
-
|
|
4319
|
-
|
|
4320
|
-
|
|
4321
|
-
|
|
4833
|
+
// Remove the empty replicas first (their worker crashed and they are waiting to be refilled), then the ones
|
|
4834
|
+
// with the highest port offsets, so that applications using server.portAssignment=perWorkerIncrement keep
|
|
4835
|
+
// listening on a contiguous range of ports starting from the configured one.
|
|
4836
|
+
const removed = (this.#replicas.get(applicationId) ?? [])
|
|
4837
|
+
.toSorted((a, b) => Number(Boolean(a.worker)) - Number(Boolean(b.worker)) || b.portOffset - a.portOffset)
|
|
4838
|
+
.slice(0, workersToStop)
|
|
4839
|
+
|
|
4840
|
+
// Detached right away, so that concurrent operations see the requested workers.
|
|
4841
|
+
// This also cancels the work in progress for them, i.e. refills and replacements.
|
|
4842
|
+
const detached = Promise.all(removed.map(replica => this.#detachReplica(applicationId, replica)))
|
|
4843
|
+
|
|
4844
|
+
for (const replica of removed) {
|
|
4845
|
+
const worker = replica.worker
|
|
4846
|
+
if (worker && typeof worker[kWorkerExitCode] !== 'number') {
|
|
4847
|
+
await sendViaITC(worker, 'removeFromMesh')
|
|
4848
|
+
await this.#stopWorker(currentWorkers, applicationId, worker[kWorkerId], false, worker, [])
|
|
4849
|
+
report.stopped.push(worker[kWorkerId])
|
|
4850
|
+
}
|
|
4322
4851
|
|
|
4323
|
-
for (const workerIndex of workerIdsToStop.splice(0, workersToStop)) {
|
|
4324
|
-
const worker = this.#workers.get(`${applicationId}:${workerIndex}`)
|
|
4325
|
-
await sendViaITC(worker, 'removeFromMesh')
|
|
4326
|
-
await this.#stopWorker(currentWorkers, applicationId, workerIndex, false, worker, [])
|
|
4327
|
-
report.stopped.push(workerIndex)
|
|
4328
4852
|
stoppedWorkersCount++
|
|
4329
4853
|
}
|
|
4330
4854
|
|
|
4855
|
+
await detached
|
|
4856
|
+
|
|
4331
4857
|
report.success = true
|
|
4332
4858
|
} catch (err) {
|
|
4333
4859
|
if (stoppedWorkersCount < 1) {
|
|
@@ -4373,6 +4899,10 @@ export class Runtime extends EventEmitter {
|
|
|
4373
4899
|
}
|
|
4374
4900
|
}
|
|
4375
4901
|
|
|
4902
|
+
if (applicationConfig.sourcePath) {
|
|
4903
|
+
allows.add(`--allow-fs-read=${join(applicationConfig.sourcePath, '*')}`)
|
|
4904
|
+
}
|
|
4905
|
+
|
|
4376
4906
|
if (write?.length) {
|
|
4377
4907
|
for (const p of write) {
|
|
4378
4908
|
allows.add(`--allow-fs-write=${isAbsolute(p) ? p : join(applicationConfig.path, p)}`)
|
|
@@ -4418,6 +4948,11 @@ export class Runtime extends EventEmitter {
|
|
|
4418
4948
|
#processHealthSignals ({ workerId, signals }) {
|
|
4419
4949
|
const worker = this.#workers.get(workerId)
|
|
4420
4950
|
|
|
4951
|
+
// Signals can arrive from a worker which is not registered (anymore)
|
|
4952
|
+
if (!worker) {
|
|
4953
|
+
return
|
|
4954
|
+
}
|
|
4955
|
+
|
|
4421
4956
|
worker[kWorkerHealthSignals] ??= new HealthSignalsQueue()
|
|
4422
4957
|
worker[kWorkerHealthSignals].add(signals)
|
|
4423
4958
|
}
|
|
@@ -4493,18 +5028,16 @@ export class Runtime extends EventEmitter {
|
|
|
4493
5028
|
}
|
|
4494
5029
|
|
|
4495
5030
|
#isApplicationStarted (id) {
|
|
4496
|
-
|
|
4497
|
-
if (!applicationConfig) {
|
|
5031
|
+
if (!this.#applications.has(id)) {
|
|
4498
5032
|
return false
|
|
4499
5033
|
}
|
|
4500
5034
|
|
|
4501
|
-
|
|
4502
|
-
|
|
4503
|
-
|
|
4504
|
-
const status =
|
|
5035
|
+
// Workers replaced by restarts get new indexes, so check all the workers of the application.
|
|
5036
|
+
// Anything past boot/init means already started.
|
|
5037
|
+
for (const workerId of this.#workers.getKeys(id)) {
|
|
5038
|
+
const status = this.#workers.get(workerId)[kWorkerStatus]
|
|
4505
5039
|
|
|
4506
|
-
|
|
4507
|
-
if (status && status !== 'boot' && status !== 'init') {
|
|
5040
|
+
if (status !== 'boot' && status !== 'init') {
|
|
4508
5041
|
return true
|
|
4509
5042
|
}
|
|
4510
5043
|
}
|
|
@@ -4847,15 +5380,55 @@ export class Runtime extends EventEmitter {
|
|
|
4847
5380
|
this.#loggerContext.updatePrefixes(ids)
|
|
4848
5381
|
}
|
|
4849
5382
|
|
|
4850
|
-
#
|
|
4851
|
-
|
|
5383
|
+
#getPortOwner (port, applicationId, hostname, includeSameApplication = false) {
|
|
5384
|
+
if (!Number.isInteger(port) || port <= 0) {
|
|
5385
|
+
return null
|
|
5386
|
+
}
|
|
4852
5387
|
|
|
4853
|
-
this.
|
|
4854
|
-
|
|
4855
|
-
|
|
4856
|
-
|
|
5388
|
+
for (const worker of this.#workers.values()) {
|
|
5389
|
+
if (!worker[kWorkerUrl] || (!includeSameApplication && worker[kApplicationId] === applicationId)) {
|
|
5390
|
+
continue
|
|
5391
|
+
}
|
|
5392
|
+
|
|
5393
|
+
try {
|
|
5394
|
+
const url = new URL(worker[kWorkerUrl])
|
|
5395
|
+
const workerPort = Number(url.port || (url.protocol === 'https:' ? 443 : 80))
|
|
5396
|
+
const workerHost = url.hostname
|
|
5397
|
+
if (workerPort === port && this.#listenersOverlap(hostname ?? workerHost, workerHost)) {
|
|
5398
|
+
return worker[kApplicationId]
|
|
5399
|
+
}
|
|
5400
|
+
} catch {}
|
|
5401
|
+
}
|
|
5402
|
+
|
|
5403
|
+
return null
|
|
5404
|
+
}
|
|
5405
|
+
|
|
5406
|
+
#listenersOverlap (host, otherHost) {
|
|
5407
|
+
host = host.toLowerCase()
|
|
5408
|
+
const wildcards = new Set(['0.0.0.0', '::', '[::]'])
|
|
5409
|
+
otherHost = otherHost.toLowerCase()
|
|
5410
|
+
return host === otherHost || wildcards.has(host) || wildcards.has(otherHost)
|
|
5411
|
+
}
|
|
5412
|
+
|
|
5413
|
+
#recordWorkerUrl (worker, applicationId, workerUrl) {
|
|
5414
|
+
if (!workerUrl) {
|
|
5415
|
+
worker[kWorkerUrl] = undefined
|
|
5416
|
+
return
|
|
5417
|
+
}
|
|
5418
|
+
|
|
5419
|
+
let hostname
|
|
5420
|
+
let port
|
|
5421
|
+
try {
|
|
5422
|
+
const url = new URL(workerUrl)
|
|
5423
|
+
hostname = url.hostname
|
|
5424
|
+
port = Number(url.port || (url.protocol === 'https:' ? 443 : 80))
|
|
5425
|
+
} catch {}
|
|
5426
|
+
|
|
5427
|
+
const owner = this.#getPortOwner(port, applicationId, hostname)
|
|
5428
|
+
if (owner) {
|
|
5429
|
+
throw new AddressInUseError(port, owner, applicationId)
|
|
5430
|
+
}
|
|
4857
5431
|
|
|
4858
|
-
|
|
4859
|
-
worker.terminate()
|
|
5432
|
+
worker[kWorkerUrl] = workerUrl
|
|
4860
5433
|
}
|
|
4861
5434
|
}
|