@platformatic/runtime 3.0.0-alpha.5 → 3.0.0-alpha.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config.d.ts +11 -7
- package/eslint.config.js +2 -4
- package/index.d.ts +11 -11
- package/index.js +35 -46
- package/lib/config.js +159 -102
- package/lib/errors.js +65 -99
- package/lib/generator.js +160 -164
- package/lib/logger.js +6 -8
- package/lib/management-api.js +91 -63
- package/lib/prom-server.js +10 -14
- package/lib/runtime.js +815 -747
- package/lib/scheduler.js +13 -15
- package/lib/schema.js +11 -8
- package/lib/shared-http-cache.js +5 -9
- package/lib/upgrade.js +5 -9
- package/lib/utils.js +6 -14
- package/lib/version.js +7 -0
- package/lib/versions/v1.36.0.js +2 -4
- package/lib/versions/v1.5.0.js +2 -4
- package/lib/versions/v2.0.0.js +3 -5
- package/lib/versions/v3.0.0.js +16 -0
- package/lib/worker/{app.js → controller.js} +68 -63
- package/lib/worker/http-cache.js +11 -14
- package/lib/worker/interceptors.js +14 -18
- package/lib/worker/itc.js +85 -79
- package/lib/worker/main.js +49 -55
- package/lib/worker/messaging.js +23 -27
- package/lib/worker/round-robin-map.js +23 -19
- package/lib/worker/shared-context.js +2 -6
- package/lib/worker/symbols.js +12 -29
- package/package.json +24 -23
- package/schema.json +281 -20
- package/lib/dependencies.js +0 -65
package/lib/runtime.js
CHANGED
|
@@ -1,97 +1,110 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
const { ITC } = require('@platformatic/itc')
|
|
4
|
-
const {
|
|
5
|
-
features,
|
|
6
|
-
ensureLoggableError,
|
|
1
|
+
import {
|
|
2
|
+
deepmerge,
|
|
7
3
|
ensureError,
|
|
4
|
+
ensureLoggableError,
|
|
5
|
+
executeInParallel,
|
|
8
6
|
executeWithTimeout,
|
|
9
|
-
|
|
10
|
-
|
|
7
|
+
features,
|
|
8
|
+
kMetadata,
|
|
11
9
|
kTimeout,
|
|
12
|
-
|
|
13
|
-
}
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
10
|
+
parseMemorySize
|
|
11
|
+
} from '@platformatic/foundation'
|
|
12
|
+
import { ITC } from '@platformatic/itc'
|
|
13
|
+
import fastify from 'fastify'
|
|
14
|
+
import { EventEmitter, once } from 'node:events'
|
|
15
|
+
import { existsSync } from 'node:fs'
|
|
16
|
+
import { readFile } from 'node:fs/promises'
|
|
17
|
+
import { STATUS_CODES } from 'node:http'
|
|
18
|
+
import { createRequire } from 'node:module'
|
|
19
|
+
import { join } from 'node:path'
|
|
20
|
+
import { setImmediate as immediate, setTimeout as sleep } from 'node:timers/promises'
|
|
21
|
+
import { pathToFileURL } from 'node:url'
|
|
22
|
+
import { Worker } from 'node:worker_threads'
|
|
23
|
+
import SonicBoom from 'sonic-boom'
|
|
24
|
+
import { Agent, request, interceptors as undiciInterceptors } from 'undici'
|
|
25
|
+
import { createThreadInterceptor } from 'undici-thread-interceptor'
|
|
26
|
+
import {
|
|
27
|
+
ApplicationAlreadyStartedError,
|
|
28
|
+
ApplicationNotFoundError,
|
|
29
|
+
ApplicationNotStartedError,
|
|
30
|
+
ApplicationStartTimeoutError,
|
|
31
|
+
InvalidArgumentError,
|
|
32
|
+
MessagingError,
|
|
33
|
+
MissingEntrypointError,
|
|
34
|
+
RuntimeAbortedError,
|
|
35
|
+
RuntimeExitedError,
|
|
36
|
+
WorkerNotFoundError
|
|
37
|
+
} from './errors.js'
|
|
38
|
+
import { abstractLogger, createLogger } from './logger.js'
|
|
39
|
+
import { startManagementApi } from './management-api.js'
|
|
40
|
+
import { startPrometheusServer } from './prom-server.js'
|
|
41
|
+
import { startScheduler } from './scheduler.js'
|
|
42
|
+
import { createSharedStore } from './shared-http-cache.js'
|
|
43
|
+
import { version } from './version.js'
|
|
44
|
+
import { sendViaITC, waitEventFromITC } from './worker/itc.js'
|
|
45
|
+
import { RoundRobinMap } from './worker/round-robin-map.js'
|
|
46
|
+
import {
|
|
47
|
+
kApplicationId,
|
|
48
|
+
kConfig,
|
|
37
49
|
kFullId,
|
|
38
|
-
kServiceId,
|
|
39
|
-
kWorkerId,
|
|
40
|
-
kITC,
|
|
41
50
|
kHealthCheckTimer,
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
kStderrMarker,
|
|
51
|
+
kId,
|
|
52
|
+
kITC,
|
|
45
53
|
kLastELU,
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
const kWorkerFile = join(__dirname, 'worker/main.js')
|
|
54
|
+
kStderrMarker,
|
|
55
|
+
kWorkerId,
|
|
56
|
+
kWorkersBroadcast,
|
|
57
|
+
kWorkerStatus
|
|
58
|
+
} from './worker/symbols.js'
|
|
52
59
|
|
|
60
|
+
const kWorkerFile = join(import.meta.dirname, 'worker/main.js')
|
|
53
61
|
const kInspectorOptions = Symbol('plt.runtime.worker.inspectorOptions')
|
|
54
|
-
const kForwardEvents = Symbol('plt.runtime.worker.forwardEvents')
|
|
55
62
|
|
|
56
63
|
const MAX_LISTENERS_COUNT = 100
|
|
57
64
|
const MAX_METRICS_QUEUE_LENGTH = 5 * 60 // 5 minutes in seconds
|
|
58
65
|
const COLLECT_METRICS_TIMEOUT = 1000
|
|
59
66
|
|
|
67
|
+
const MAX_CONCURRENCY = 5
|
|
60
68
|
const MAX_BOOTSTRAP_ATTEMPTS = 5
|
|
61
69
|
const IMMEDIATE_RESTART_MAX_THRESHOLD = 10
|
|
62
70
|
const MAX_WORKERS = 100
|
|
63
71
|
|
|
64
|
-
|
|
65
|
-
|
|
72
|
+
export class Runtime extends EventEmitter {
|
|
73
|
+
logger
|
|
74
|
+
error
|
|
66
75
|
|
|
67
|
-
|
|
76
|
+
#loggerDestination
|
|
77
|
+
#stdio
|
|
78
|
+
|
|
79
|
+
#status // starting, started, stopping, stopped, closed
|
|
68
80
|
#root
|
|
69
81
|
#config
|
|
70
82
|
#env
|
|
71
83
|
#context
|
|
84
|
+
#sharedContext
|
|
72
85
|
#isProduction
|
|
73
|
-
#
|
|
74
|
-
#servicesIds
|
|
86
|
+
#concurrency
|
|
75
87
|
#entrypointId
|
|
76
88
|
#url
|
|
77
|
-
|
|
89
|
+
|
|
78
90
|
#metrics
|
|
79
91
|
#metricsTimeout
|
|
80
|
-
|
|
92
|
+
|
|
81
93
|
#meshInterceptor
|
|
82
94
|
#dispatcher
|
|
95
|
+
|
|
83
96
|
#managementApi
|
|
84
97
|
#prometheusServer
|
|
85
98
|
#inspectorServer
|
|
99
|
+
|
|
100
|
+
#applicationsConfigsPatches
|
|
86
101
|
#workers
|
|
87
102
|
#workersBroadcastChannel
|
|
88
103
|
#workerITCHandlers
|
|
89
104
|
#restartingWorkers
|
|
105
|
+
|
|
90
106
|
#sharedHttpCache
|
|
91
|
-
servicesConfigsPatches
|
|
92
107
|
#scheduler
|
|
93
|
-
#stdio
|
|
94
|
-
#sharedContext
|
|
95
108
|
|
|
96
109
|
constructor (config, context) {
|
|
97
110
|
super()
|
|
@@ -102,19 +115,15 @@ class Runtime extends EventEmitter {
|
|
|
102
115
|
this.#env = config[kMetadata].env
|
|
103
116
|
this.#context = context ?? {}
|
|
104
117
|
this.#isProduction = this.#context.isProduction ?? this.#context.production ?? false
|
|
105
|
-
this.#
|
|
118
|
+
this.#concurrency = this.#context.concurrency ?? MAX_CONCURRENCY
|
|
106
119
|
this.#workers = new RoundRobinMap()
|
|
107
|
-
this.#servicesIds = []
|
|
108
120
|
this.#url = undefined
|
|
109
|
-
this.#meshInterceptor = createThreadInterceptor({
|
|
110
|
-
domain: '.plt.local',
|
|
111
|
-
timeout: this.#config.serviceTimeout
|
|
112
|
-
})
|
|
121
|
+
this.#meshInterceptor = createThreadInterceptor({ domain: '.plt.local', timeout: this.#config.applicationTimeout })
|
|
113
122
|
this.logger = abstractLogger // This is replaced by the real logger in init() and eventually removed in close()
|
|
114
123
|
this.#status = undefined
|
|
115
124
|
this.#restartingWorkers = new Map()
|
|
116
125
|
this.#sharedHttpCache = null
|
|
117
|
-
this
|
|
126
|
+
this.#applicationsConfigsPatches = new Map()
|
|
118
127
|
|
|
119
128
|
if (!this.#config.logger.captureStdio) {
|
|
120
129
|
this.#stdio = {
|
|
@@ -124,9 +133,9 @@ class Runtime extends EventEmitter {
|
|
|
124
133
|
}
|
|
125
134
|
|
|
126
135
|
this.#workerITCHandlers = {
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
136
|
+
getApplicationMeta: this.getApplicationMeta.bind(this),
|
|
137
|
+
listApplications: this.getApplicationsIds.bind(this),
|
|
138
|
+
getApplications: this.getApplications.bind(this),
|
|
130
139
|
getWorkers: this.getWorkers.bind(this),
|
|
131
140
|
getWorkerMessagingChannel: this.#getWorkerMessagingChannel.bind(this),
|
|
132
141
|
getHttpCacheValue: this.#getHttpCacheValue.bind(this),
|
|
@@ -145,10 +154,6 @@ class Runtime extends EventEmitter {
|
|
|
145
154
|
}
|
|
146
155
|
|
|
147
156
|
const config = this.#config
|
|
148
|
-
const autoloadEnabled = config.autoload
|
|
149
|
-
|
|
150
|
-
// This cannot be transferred to worker threads
|
|
151
|
-
delete config.configManager
|
|
152
157
|
|
|
153
158
|
if (config.managementApi) {
|
|
154
159
|
this.#managementApi = await startManagementApi(this, this.#root)
|
|
@@ -163,19 +168,18 @@ class Runtime extends EventEmitter {
|
|
|
163
168
|
this.logger = logger
|
|
164
169
|
this.#loggerDestination = destination
|
|
165
170
|
|
|
166
|
-
this.#servicesIds = config.services.map(service => service.id)
|
|
167
171
|
this.#createWorkersBroadcastChannel()
|
|
168
172
|
|
|
169
173
|
const workersConfig = []
|
|
170
|
-
for (const
|
|
171
|
-
const count =
|
|
172
|
-
if (count > 1 &&
|
|
174
|
+
for (const application of config.applications) {
|
|
175
|
+
const count = application.workers ?? this.#config.workers
|
|
176
|
+
if (count > 1 && application.entrypoint && !features.node.reusePort) {
|
|
173
177
|
this.logger.warn(
|
|
174
|
-
`"${
|
|
178
|
+
`"${application.id}" is set as the entrypoint, but reusePort is not available in your OS; setting workers to 1 instead of ${count}`
|
|
175
179
|
)
|
|
176
|
-
workersConfig.push({ id:
|
|
180
|
+
workersConfig.push({ id: application.id, workers: 1 })
|
|
177
181
|
} else {
|
|
178
|
-
workersConfig.push({ id:
|
|
182
|
+
workersConfig.push({ id: application.id, workers: count })
|
|
179
183
|
}
|
|
180
184
|
}
|
|
181
185
|
|
|
@@ -189,64 +193,7 @@ class Runtime extends EventEmitter {
|
|
|
189
193
|
this.#env['PLT_ENVIRONMENT'] = 'development'
|
|
190
194
|
}
|
|
191
195
|
|
|
192
|
-
|
|
193
|
-
for (const serviceConfig of config.services) {
|
|
194
|
-
// If there is no service path, check if the service was resolved
|
|
195
|
-
if (!serviceConfig.path) {
|
|
196
|
-
if (serviceConfig.url) {
|
|
197
|
-
// Try to backfill the path for external services
|
|
198
|
-
serviceConfig.path = join(this.#root, config.resolvedServicesBasePath, serviceConfig.id)
|
|
199
|
-
|
|
200
|
-
if (!existsSync(serviceConfig.path)) {
|
|
201
|
-
const executable = globalThis.platformatic?.executable ?? 'platformatic'
|
|
202
|
-
this.logger.error(
|
|
203
|
-
`The path for service "%s" does not exist. Please run "${executable} resolve" and try again.`,
|
|
204
|
-
serviceConfig.id
|
|
205
|
-
)
|
|
206
|
-
|
|
207
|
-
await this.closeAndThrow(new errors.RuntimeAbortedError())
|
|
208
|
-
}
|
|
209
|
-
} else {
|
|
210
|
-
this.logger.error(
|
|
211
|
-
'The service "%s" has no path defined. Please check your configuration and try again.',
|
|
212
|
-
serviceConfig.id
|
|
213
|
-
)
|
|
214
|
-
|
|
215
|
-
await this.closeAndThrow(new errors.RuntimeAbortedError())
|
|
216
|
-
}
|
|
217
|
-
}
|
|
218
|
-
|
|
219
|
-
await this.#setupService(serviceConfig)
|
|
220
|
-
}
|
|
221
|
-
|
|
222
|
-
try {
|
|
223
|
-
checkDependencies(config.services)
|
|
224
|
-
|
|
225
|
-
// Make sure the list exists before computing the dependencies, otherwise some services might not be stopped
|
|
226
|
-
if (autoloadEnabled) {
|
|
227
|
-
this.#workers = topologicalSort(this.#workers, config)
|
|
228
|
-
}
|
|
229
|
-
|
|
230
|
-
// Recompute the list of services after sorting
|
|
231
|
-
this.#servicesIds = config.services.map(service => service.id)
|
|
232
|
-
|
|
233
|
-
// When autoloading is disabled, add a warning if a service is defined before its dependencies
|
|
234
|
-
if (!autoloadEnabled) {
|
|
235
|
-
for (let i = 0; i < config.services.length; i++) {
|
|
236
|
-
const current = config.services[i]
|
|
237
|
-
|
|
238
|
-
for (const dep of current.dependencies ?? []) {
|
|
239
|
-
if (config.services.findIndex(s => s.id === dep.id) > i) {
|
|
240
|
-
this.logger.warn(
|
|
241
|
-
`Service "${current.id}" depends on service "${dep.id}", but it is defined and it will be started before it. Please check your configuration file.`
|
|
242
|
-
)
|
|
243
|
-
}
|
|
244
|
-
}
|
|
245
|
-
}
|
|
246
|
-
}
|
|
247
|
-
} catch (e) {
|
|
248
|
-
await this.closeAndThrow(e)
|
|
249
|
-
}
|
|
196
|
+
await this.#setupApplications()
|
|
250
197
|
|
|
251
198
|
await this.#setDispatcher(config.undici)
|
|
252
199
|
|
|
@@ -263,17 +210,19 @@ class Runtime extends EventEmitter {
|
|
|
263
210
|
}
|
|
264
211
|
|
|
265
212
|
if (typeof this.#config.entrypoint === 'undefined') {
|
|
266
|
-
throw new
|
|
213
|
+
throw new MissingEntrypointError()
|
|
267
214
|
}
|
|
268
215
|
this.#updateStatus('starting')
|
|
269
216
|
this.#createWorkersBroadcastChannel()
|
|
270
217
|
|
|
271
|
-
// Important: do not use Promise.all here since it won't properly manage dependencies
|
|
272
218
|
try {
|
|
273
|
-
|
|
274
|
-
|
|
219
|
+
const startInvocations = []
|
|
220
|
+
for (const application of this.getApplicationsIds()) {
|
|
221
|
+
startInvocations.push([application, silent])
|
|
275
222
|
}
|
|
276
223
|
|
|
224
|
+
await executeInParallel(this.startApplication.bind(this), startInvocations, this.#concurrency)
|
|
225
|
+
|
|
277
226
|
if (this.#config.inspectorOptions) {
|
|
278
227
|
const { port } = this.#config.inspectorOptions
|
|
279
228
|
|
|
@@ -304,7 +253,7 @@ class Runtime extends EventEmitter {
|
|
|
304
253
|
|
|
305
254
|
await server.listen({ port })
|
|
306
255
|
this.logger.info(
|
|
307
|
-
'The inspector server is now listening for all
|
|
256
|
+
'The inspector server is now listening for all applications. Open `chrome://inspect` in Google Chrome to connect.'
|
|
308
257
|
)
|
|
309
258
|
this.#inspectorServer = server
|
|
310
259
|
}
|
|
@@ -339,19 +288,38 @@ class Runtime extends EventEmitter {
|
|
|
339
288
|
|
|
340
289
|
// Stop the entrypoint first so that no new requests are accepted
|
|
341
290
|
if (this.#entrypointId) {
|
|
342
|
-
await this.
|
|
291
|
+
await this.stopApplication(this.#entrypointId, silent)
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
const stopInvocations = []
|
|
295
|
+
|
|
296
|
+
const allApplications = await this.getApplications(true)
|
|
297
|
+
|
|
298
|
+
// Construct the reverse dependency graph
|
|
299
|
+
const dependents = {}
|
|
300
|
+
for (const application of allApplications.applications) {
|
|
301
|
+
for (const dependency of application.dependencies ?? []) {
|
|
302
|
+
let applicationDependents = dependents[dependency]
|
|
303
|
+
if (!applicationDependents) {
|
|
304
|
+
applicationDependents = new Set()
|
|
305
|
+
dependents[dependency] = applicationDependents
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
applicationDependents.add(application.id)
|
|
309
|
+
}
|
|
343
310
|
}
|
|
344
311
|
|
|
345
|
-
|
|
346
|
-
for (const service of this.#servicesIds.reverse()) {
|
|
312
|
+
for (const application of this.getApplicationsIds()) {
|
|
347
313
|
// The entrypoint has been stopped above
|
|
348
|
-
if (
|
|
314
|
+
if (application === this.#entrypointId) {
|
|
349
315
|
continue
|
|
350
316
|
}
|
|
351
317
|
|
|
352
|
-
|
|
318
|
+
stopInvocations.push([application, silent, Array.from(dependents[application] ?? [])])
|
|
353
319
|
}
|
|
354
320
|
|
|
321
|
+
await executeInParallel(this.stopApplication.bind(this), stopInvocations, this.#concurrency)
|
|
322
|
+
|
|
355
323
|
await this.#meshInterceptor.close()
|
|
356
324
|
this.#workersBroadcastChannel?.close()
|
|
357
325
|
|
|
@@ -370,16 +338,11 @@ class Runtime extends EventEmitter {
|
|
|
370
338
|
return this.#url
|
|
371
339
|
}
|
|
372
340
|
|
|
373
|
-
getRuntimeStatus () {
|
|
374
|
-
return this.#status
|
|
375
|
-
}
|
|
376
|
-
|
|
377
341
|
async close (silent = false) {
|
|
378
|
-
this.#updateStatus('closing')
|
|
379
|
-
|
|
380
342
|
clearInterval(this.#metricsTimeout)
|
|
381
343
|
|
|
382
344
|
await this.stop(silent)
|
|
345
|
+
this.#updateStatus('closing')
|
|
383
346
|
|
|
384
347
|
// The management API autocloses by itself via event in management-api.js.
|
|
385
348
|
// This is needed to let management API stop endpoint to reply.
|
|
@@ -404,6 +367,7 @@ class Runtime extends EventEmitter {
|
|
|
404
367
|
|
|
405
368
|
async closeAndThrow (error) {
|
|
406
369
|
this.#updateStatus('errored', error)
|
|
370
|
+
this.error = error
|
|
407
371
|
|
|
408
372
|
// Wait for the next tick so that any pending logging is properly flushed
|
|
409
373
|
await sleep(1)
|
|
@@ -412,71 +376,9 @@ class Runtime extends EventEmitter {
|
|
|
412
376
|
throw error
|
|
413
377
|
}
|
|
414
378
|
|
|
415
|
-
async startService (id, silent = false) {
|
|
416
|
-
// Since when a service is stopped the worker is deleted, we consider a service start if its first service
|
|
417
|
-
// is no longer in the init phase
|
|
418
|
-
const firstWorker = this.#workers.get(`${id}:0`)
|
|
419
|
-
if (firstWorker && firstWorker[kWorkerStatus] !== 'boot' && firstWorker[kWorkerStatus] !== 'init') {
|
|
420
|
-
throw new errors.ApplicationAlreadyStartedError()
|
|
421
|
-
}
|
|
422
|
-
|
|
423
|
-
const config = this.#config
|
|
424
|
-
const serviceConfig = config.services.find(s => s.id === id)
|
|
425
|
-
|
|
426
|
-
if (!serviceConfig) {
|
|
427
|
-
throw new errors.ServiceNotFoundError(id, Array.from(this.#servicesIds).join(', '))
|
|
428
|
-
}
|
|
429
|
-
|
|
430
|
-
const workersCount = await this.#workers.getCount(serviceConfig.id)
|
|
431
|
-
|
|
432
|
-
this.emit('service:starting', id)
|
|
433
|
-
|
|
434
|
-
for (let i = 0; i < workersCount; i++) {
|
|
435
|
-
await this.#startWorker(config, serviceConfig, workersCount, id, i, silent)
|
|
436
|
-
}
|
|
437
|
-
|
|
438
|
-
this.emit('service:started', id)
|
|
439
|
-
}
|
|
440
|
-
|
|
441
|
-
async stopService (id, silent = false) {
|
|
442
|
-
const config = this.#config
|
|
443
|
-
const serviceConfig = config.services.find(s => s.id === id)
|
|
444
|
-
|
|
445
|
-
if (!serviceConfig) {
|
|
446
|
-
throw new errors.ServiceNotFoundError(id, Array.from(this.#servicesIds).join(', '))
|
|
447
|
-
}
|
|
448
|
-
|
|
449
|
-
const workersCount = await this.#workers.getCount(serviceConfig.id)
|
|
450
|
-
|
|
451
|
-
this.emit('service:stopping', id)
|
|
452
|
-
|
|
453
|
-
for (let i = 0; i < workersCount; i++) {
|
|
454
|
-
await this.#stopWorker(workersCount, id, i, silent)
|
|
455
|
-
}
|
|
456
|
-
|
|
457
|
-
this.emit('service:stopped', id)
|
|
458
|
-
}
|
|
459
|
-
|
|
460
|
-
async buildService (id) {
|
|
461
|
-
const service = await this.#getServiceById(id)
|
|
462
|
-
|
|
463
|
-
this.emit('service:building', id)
|
|
464
|
-
try {
|
|
465
|
-
await sendViaITC(service, 'build')
|
|
466
|
-
this.emit('service:built', id)
|
|
467
|
-
} catch (e) {
|
|
468
|
-
// The service exports no meta, return an empty object
|
|
469
|
-
if (e.code === 'PLT_ITC_HANDLER_NOT_FOUND') {
|
|
470
|
-
return {}
|
|
471
|
-
}
|
|
472
|
-
|
|
473
|
-
throw e
|
|
474
|
-
}
|
|
475
|
-
}
|
|
476
|
-
|
|
477
379
|
async inject (id, injectParams) {
|
|
478
|
-
// Make sure the
|
|
479
|
-
await this.#
|
|
380
|
+
// Make sure the application exists
|
|
381
|
+
await this.#getApplicationById(id, true)
|
|
480
382
|
|
|
481
383
|
if (typeof injectParams === 'string') {
|
|
482
384
|
injectParams = { url: injectParams }
|
|
@@ -519,6 +421,109 @@ class Runtime extends EventEmitter {
|
|
|
519
421
|
}
|
|
520
422
|
}
|
|
521
423
|
|
|
424
|
+
emit (event, payload) {
|
|
425
|
+
for (const worker of this.#workers.values()) {
|
|
426
|
+
worker[kITC].notify('runtime:event', { event, payload })
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
this.logger.trace({ event, payload }, 'Runtime event')
|
|
430
|
+
return super.emit(event, payload)
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
async sendCommandToApplication (id, name, message) {
|
|
434
|
+
const application = await this.#getApplicationById(id)
|
|
435
|
+
|
|
436
|
+
try {
|
|
437
|
+
return await sendViaITC(application, name, message)
|
|
438
|
+
} catch (e) {
|
|
439
|
+
// The application exports no meta, return an empty object
|
|
440
|
+
if (e.code === 'PLT_ITC_HANDLER_NOT_FOUND') {
|
|
441
|
+
return {}
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
throw e
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
async startApplication (id, silent = false) {
|
|
449
|
+
// Since when an application is stopped the worker is deleted, we consider an application start if its first application
|
|
450
|
+
// is no longer in the init phase
|
|
451
|
+
const firstWorker = this.#workers.get(`${id}:0`)
|
|
452
|
+
if (firstWorker && firstWorker[kWorkerStatus] !== 'boot' && firstWorker[kWorkerStatus] !== 'init') {
|
|
453
|
+
throw new ApplicationAlreadyStartedError()
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
const config = this.#config
|
|
457
|
+
const applicationConfig = config.applications.find(s => s.id === id)
|
|
458
|
+
|
|
459
|
+
if (!applicationConfig) {
|
|
460
|
+
throw new ApplicationNotFoundError(id, this.getApplicationsIds().join(', '))
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
const workersCount = await this.#workers.getCount(applicationConfig.id)
|
|
464
|
+
|
|
465
|
+
this.emit('application:starting', id)
|
|
466
|
+
|
|
467
|
+
for (let i = 0; i < workersCount; i++) {
|
|
468
|
+
await this.#startWorker(config, applicationConfig, workersCount, id, i, silent)
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
this.emit('application:started', id)
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
async stopApplication (id, silent = false, dependents = []) {
|
|
475
|
+
const config = this.#config
|
|
476
|
+
const applicationConfig = config.applications.find(s => s.id === id)
|
|
477
|
+
|
|
478
|
+
if (!applicationConfig) {
|
|
479
|
+
throw new ApplicationNotFoundError(id, this.getApplicationsIds().join(', '))
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
const workersCount = await this.#workers.getCount(applicationConfig.id)
|
|
483
|
+
|
|
484
|
+
this.emit('application:stopping', id)
|
|
485
|
+
|
|
486
|
+
if (typeof workersCount === 'number') {
|
|
487
|
+
const stopInvocations = []
|
|
488
|
+
for (let i = 0; i < workersCount; i++) {
|
|
489
|
+
stopInvocations.push([workersCount, id, i, silent, undefined, dependents])
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
await executeInParallel(this.#stopWorker.bind(this), stopInvocations, this.#concurrency)
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
this.emit('application:stopped', id)
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
async buildApplication (id) {
|
|
499
|
+
const application = await this.#getApplicationById(id)
|
|
500
|
+
|
|
501
|
+
this.emit('application:building', id)
|
|
502
|
+
try {
|
|
503
|
+
await sendViaITC(application, 'build')
|
|
504
|
+
this.emit('application:built', id)
|
|
505
|
+
} catch (e) {
|
|
506
|
+
// The application exports no meta, return an empty object
|
|
507
|
+
if (e.code === 'PLT_ITC_HANDLER_NOT_FOUND') {
|
|
508
|
+
return {}
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
throw e
|
|
512
|
+
}
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
async startApplicationProfiling (id, options = {}, ensureStarted = true) {
|
|
516
|
+
const service = await this.#getApplicationById(id, ensureStarted)
|
|
517
|
+
|
|
518
|
+
return sendViaITC(service, 'startProfiling', options)
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
async stopApplicationProfiling (id, ensureStarted = true) {
|
|
522
|
+
const service = await this.#getApplicationById(id, ensureStarted)
|
|
523
|
+
|
|
524
|
+
return sendViaITC(service, 'stopProfiling')
|
|
525
|
+
}
|
|
526
|
+
|
|
522
527
|
async updateUndiciInterceptors (undiciConfig) {
|
|
523
528
|
this.#config.undici = undiciConfig
|
|
524
529
|
|
|
@@ -546,7 +551,7 @@ class Runtime extends EventEmitter {
|
|
|
546
551
|
try {
|
|
547
552
|
metrics = await this.getFormattedMetrics()
|
|
548
553
|
} catch (error) {
|
|
549
|
-
if (!(error instanceof
|
|
554
|
+
if (!(error instanceof RuntimeExitedError)) {
|
|
550
555
|
this.logger.error({ err: ensureLoggableError(error) }, 'Error collecting metrics')
|
|
551
556
|
}
|
|
552
557
|
return
|
|
@@ -560,6 +565,25 @@ class Runtime extends EventEmitter {
|
|
|
560
565
|
}, COLLECT_METRICS_TIMEOUT).unref()
|
|
561
566
|
}
|
|
562
567
|
|
|
568
|
+
invalidateHttpCache (options = {}) {
|
|
569
|
+
const { keys, tags } = options
|
|
570
|
+
|
|
571
|
+
if (!this.#sharedHttpCache) {
|
|
572
|
+
return
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
const promises = []
|
|
576
|
+
if (keys && keys.length > 0) {
|
|
577
|
+
promises.push(this.#sharedHttpCache.deleteKeys(keys))
|
|
578
|
+
}
|
|
579
|
+
|
|
580
|
+
if (tags && tags.length > 0) {
|
|
581
|
+
promises.push(this.#sharedHttpCache.deleteTags(tags))
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
return Promise.all(promises)
|
|
585
|
+
}
|
|
586
|
+
|
|
563
587
|
async addLoggerDestination (writableStream) {
|
|
564
588
|
// Add the stream - We output everything we get
|
|
565
589
|
this.#loggerDestination.add({ stream: writableStream, level: 1 })
|
|
@@ -579,10 +603,153 @@ class Runtime extends EventEmitter {
|
|
|
579
603
|
this.on('closed', onClose)
|
|
580
604
|
}
|
|
581
605
|
|
|
606
|
+
async updateSharedContext (options = {}) {
|
|
607
|
+
const { context, overwrite = false } = options
|
|
608
|
+
|
|
609
|
+
const sharedContext = overwrite ? {} : this.#sharedContext
|
|
610
|
+
Object.assign(sharedContext, context)
|
|
611
|
+
|
|
612
|
+
this.#sharedContext = sharedContext
|
|
613
|
+
|
|
614
|
+
const promises = []
|
|
615
|
+
for (const worker of this.#workers.values()) {
|
|
616
|
+
promises.push(sendViaITC(worker, 'setSharedContext', sharedContext))
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
const results = await Promise.allSettled(promises)
|
|
620
|
+
for (const result of results) {
|
|
621
|
+
if (result.status === 'rejected') {
|
|
622
|
+
this.logger.error({ err: result.reason }, 'Cannot update shared context')
|
|
623
|
+
}
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
return sharedContext
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
setApplicationConfigPatch (id, patch) {
|
|
630
|
+
this.#applicationsConfigsPatches.set(id, patch)
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
removeApplicationConfigPatch (id) {
|
|
634
|
+
this.#applicationsConfigsPatches.delete(id)
|
|
635
|
+
}
|
|
636
|
+
|
|
637
|
+
/**
|
|
638
|
+
* Updates the resources of the applications, such as the number of workers and health configurations (e.g., heap memory settings).
|
|
639
|
+
*
|
|
640
|
+
* This function handles three update scenarios for each application:
|
|
641
|
+
* 1. **Updating workers only**: Adjusts the number of workers for the application.
|
|
642
|
+
* 2. **Updating health configurations only**: Updates health parameters like `maxHeapTotal` or `maxYoungGeneration`.
|
|
643
|
+
* 3. **Updating both workers and health configurations**: Scales the workers and also applies health settings.
|
|
644
|
+
*
|
|
645
|
+
* When updating both workers and health:
|
|
646
|
+
* - **Scaling down workers**: Stops extra workers, then restarts the remaining workers with the previous settings.
|
|
647
|
+
* - **Scaling up workers**: Starts new workers with the updated heap settings, then restarts the old workers with the updated settings.
|
|
648
|
+
*
|
|
649
|
+
* Scaling up new resources (workers and/or heap memory) may fails due to insufficient memory, in this case the operation may fail partially or entirely.
|
|
650
|
+
* Scaling down is expected to succeed without issues.
|
|
651
|
+
*
|
|
652
|
+
* @param {Array<Object>} updates - An array of objects that define the updates for each application.
|
|
653
|
+
* @param {string} updates[].application - The ID of the application to update.
|
|
654
|
+
* @param {number} [updates[].workers] - The desired number of workers for the application. If omitted, workers will not be updated.
|
|
655
|
+
* @param {Object} [updates[].health] - The health configuration to update for the application, which may include:
|
|
656
|
+
* @param {string|number} [updates[].health.maxHeapTotal] - The maximum heap memory for the application. Can be a valid memory string (e.g., '1G', '512MB') or a number representing bytes.
|
|
657
|
+
* @param {string|number} [updates[].health.maxYoungGeneration] - The maximum young generation memory for the application. Can be a valid memory string (e.g., '128MB') or a number representing bytes.
|
|
658
|
+
*
|
|
659
|
+
* @returns {Promise<Array<Object>>} - A promise that resolves to an array of reports for each application, detailing the success or failure of the operations:
|
|
660
|
+
* - `application`: The application ID.
|
|
661
|
+
* - `workers`: The workers' update report, including the current, new number of workers, started workers, and success status.
|
|
662
|
+
* - `health`: The health update report, showing the current and new heap settings, updated workers, and success status.
|
|
663
|
+
*
|
|
664
|
+
* @example
|
|
665
|
+
* await runtime.updateApplicationsResources([
|
|
666
|
+
* { application: 'application-1', workers: 2, health: { maxHeapTotal: '1G', maxYoungGeneration: '128 MB' } },
|
|
667
|
+
* { application: 'application-2', health: { maxHeapTotal: '1G' } },
|
|
668
|
+
* { application: 'application-3', workers: 2 },
|
|
669
|
+
* ])
|
|
670
|
+
*
|
|
671
|
+
* In this example:
|
|
672
|
+
* - `application-1` will have 2 workers and updated heap memory configurations.
|
|
673
|
+
* - `application-2` will have updated heap memory settings (without changing workers).
|
|
674
|
+
* - `application-3` will have its workers set to 2 but no change in memory settings.
|
|
675
|
+
*
|
|
676
|
+
* @throws {InvalidArgumentError} - Throws if any update parameter is invalid, such as:
|
|
677
|
+
* - Missing application ID.
|
|
678
|
+
* - Invalid worker count (not a positive integer).
|
|
679
|
+
* - Invalid memory size format for `maxHeapTotal` or `maxYoungGeneration`.
|
|
680
|
+
* @throws {ApplicationNotFoundError} - Throws if the specified application ID does not exist in the current application configuration.
|
|
681
|
+
*/
|
|
682
|
+
async updateApplicationsResources (updates) {
|
|
683
|
+
if (this.#status === 'stopping' || this.#status === 'closed') {
|
|
684
|
+
this.logger.warn('Cannot update application resources when the runtime is stopping or closed')
|
|
685
|
+
return
|
|
686
|
+
}
|
|
687
|
+
|
|
688
|
+
const ups = await this.#validateUpdateApplicationResources(updates)
|
|
689
|
+
const config = this.#config
|
|
690
|
+
|
|
691
|
+
const report = []
|
|
692
|
+
for (const update of ups) {
|
|
693
|
+
const { applicationId, config: applicationConfig, workers, health, currentWorkers, currentHealth } = update
|
|
694
|
+
|
|
695
|
+
if (workers && health) {
|
|
696
|
+
const r = await this.#updateApplicationWorkersAndHealth(
|
|
697
|
+
applicationId,
|
|
698
|
+
config,
|
|
699
|
+
applicationConfig,
|
|
700
|
+
workers,
|
|
701
|
+
health,
|
|
702
|
+
currentWorkers,
|
|
703
|
+
currentHealth
|
|
704
|
+
)
|
|
705
|
+
report.push({
|
|
706
|
+
application: applicationId,
|
|
707
|
+
workers: r.workers,
|
|
708
|
+
health: r.health
|
|
709
|
+
})
|
|
710
|
+
} else if (health) {
|
|
711
|
+
const r = await this.#updateApplicationHealth(
|
|
712
|
+
applicationId,
|
|
713
|
+
config,
|
|
714
|
+
applicationConfig,
|
|
715
|
+
currentWorkers,
|
|
716
|
+
currentHealth,
|
|
717
|
+
health
|
|
718
|
+
)
|
|
719
|
+
report.push({
|
|
720
|
+
application: applicationId,
|
|
721
|
+
health: r.health
|
|
722
|
+
})
|
|
723
|
+
} else if (workers) {
|
|
724
|
+
const r = await this.#updateApplicationWorkers(
|
|
725
|
+
applicationId,
|
|
726
|
+
config,
|
|
727
|
+
applicationConfig,
|
|
728
|
+
workers,
|
|
729
|
+
currentWorkers
|
|
730
|
+
)
|
|
731
|
+
report.push({
|
|
732
|
+
application: applicationId,
|
|
733
|
+
workers: r.workers
|
|
734
|
+
})
|
|
735
|
+
}
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
return report
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
setConcurrency (concurrency) {
|
|
742
|
+
this.#concurrency = concurrency
|
|
743
|
+
}
|
|
744
|
+
|
|
582
745
|
async getUrl () {
|
|
583
746
|
return this.#url
|
|
584
747
|
}
|
|
585
748
|
|
|
749
|
+
getRuntimeStatus () {
|
|
750
|
+
return this.#status
|
|
751
|
+
}
|
|
752
|
+
|
|
586
753
|
async getRuntimeMetadata () {
|
|
587
754
|
const packageJson = await this.#getRuntimePackageJson()
|
|
588
755
|
const entrypointDetails = await this.getEntrypointDetails()
|
|
@@ -598,7 +765,7 @@ class Runtime extends EventEmitter {
|
|
|
598
765
|
packageName: packageJson.name ?? null,
|
|
599
766
|
packageVersion: packageJson.version ?? null,
|
|
600
767
|
url: entrypointDetails?.url ?? null,
|
|
601
|
-
platformaticVersion
|
|
768
|
+
platformaticVersion: version
|
|
602
769
|
}
|
|
603
770
|
}
|
|
604
771
|
|
|
@@ -628,47 +795,19 @@ class Runtime extends EventEmitter {
|
|
|
628
795
|
}
|
|
629
796
|
|
|
630
797
|
getManagementApiUrl () {
|
|
631
|
-
return this.#managementApi?.server.address()
|
|
798
|
+
return this.#managementApi?.server.address() ?? null
|
|
632
799
|
}
|
|
633
800
|
|
|
634
801
|
async getEntrypointDetails () {
|
|
635
|
-
return this.
|
|
636
|
-
}
|
|
637
|
-
|
|
638
|
-
async getServices () {
|
|
639
|
-
return {
|
|
640
|
-
entrypoint: this.#entrypointId,
|
|
641
|
-
production: this.#isProduction,
|
|
642
|
-
services: await Promise.all(this.#servicesIds.map(id => this.getServiceDetails(id)))
|
|
643
|
-
}
|
|
644
|
-
}
|
|
645
|
-
|
|
646
|
-
async getWorkers () {
|
|
647
|
-
const status = {}
|
|
648
|
-
|
|
649
|
-
for (const [service, { count }] of Object.entries(this.#workers.configuration)) {
|
|
650
|
-
for (let i = 0; i < count; i++) {
|
|
651
|
-
const label = `${service}:${i}`
|
|
652
|
-
const worker = this.#workers.get(label)
|
|
653
|
-
|
|
654
|
-
status[label] = {
|
|
655
|
-
service,
|
|
656
|
-
worker: i,
|
|
657
|
-
status: worker?.[kWorkerStatus] ?? 'exited',
|
|
658
|
-
thread: worker?.threadId
|
|
659
|
-
}
|
|
660
|
-
}
|
|
661
|
-
}
|
|
662
|
-
|
|
663
|
-
return status
|
|
802
|
+
return this.getApplicationDetails(this.#entrypointId)
|
|
664
803
|
}
|
|
665
804
|
|
|
666
805
|
async getCustomHealthChecks () {
|
|
667
806
|
const status = {}
|
|
668
807
|
|
|
669
|
-
for (const [
|
|
808
|
+
for (const [application, { count }] of Object.entries(this.#workers.configuration)) {
|
|
670
809
|
for (let i = 0; i < count; i++) {
|
|
671
|
-
const label = `${
|
|
810
|
+
const label = `${application}:${i}`
|
|
672
811
|
const worker = this.#workers.get(label)
|
|
673
812
|
|
|
674
813
|
status[label] = await sendViaITC(worker, 'getCustomHealthCheck')
|
|
@@ -681,83 +820,16 @@ class Runtime extends EventEmitter {
|
|
|
681
820
|
async getCustomReadinessChecks () {
|
|
682
821
|
const status = {}
|
|
683
822
|
|
|
684
|
-
for (const [
|
|
685
|
-
for (let i = 0; i < count; i++) {
|
|
686
|
-
const label = `${
|
|
687
|
-
const worker = this.#workers.get(label)
|
|
688
|
-
|
|
689
|
-
status[label] = await sendViaITC(worker, 'getCustomReadinessCheck')
|
|
690
|
-
}
|
|
691
|
-
}
|
|
692
|
-
|
|
693
|
-
return status
|
|
694
|
-
}
|
|
695
|
-
|
|
696
|
-
async getServiceDetails (id, allowUnloaded = false) {
|
|
697
|
-
let service
|
|
698
|
-
|
|
699
|
-
try {
|
|
700
|
-
service = await this.#getServiceById(id)
|
|
701
|
-
} catch (e) {
|
|
702
|
-
if (allowUnloaded) {
|
|
703
|
-
return { id, status: 'stopped' }
|
|
704
|
-
}
|
|
705
|
-
|
|
706
|
-
throw e
|
|
707
|
-
}
|
|
708
|
-
|
|
709
|
-
const { entrypoint, dependencies, localUrl } = service[kConfig]
|
|
710
|
-
|
|
711
|
-
const status = await sendViaITC(service, 'getStatus')
|
|
712
|
-
const { type, version } = await sendViaITC(service, 'getServiceInfo')
|
|
713
|
-
|
|
714
|
-
const serviceDetails = {
|
|
715
|
-
id,
|
|
716
|
-
type,
|
|
717
|
-
status,
|
|
718
|
-
version,
|
|
719
|
-
localUrl,
|
|
720
|
-
entrypoint,
|
|
721
|
-
dependencies
|
|
722
|
-
}
|
|
723
|
-
|
|
724
|
-
if (this.#isProduction) {
|
|
725
|
-
serviceDetails.workers = this.#workers.getCount(id)
|
|
726
|
-
}
|
|
727
|
-
|
|
728
|
-
if (entrypoint) {
|
|
729
|
-
serviceDetails.url = status === 'started' ? this.#url : null
|
|
730
|
-
}
|
|
731
|
-
|
|
732
|
-
return serviceDetails
|
|
733
|
-
}
|
|
734
|
-
|
|
735
|
-
async getService (id, ensureStarted = true) {
|
|
736
|
-
return this.#getServiceById(id, ensureStarted)
|
|
737
|
-
}
|
|
738
|
-
|
|
739
|
-
async getServiceConfig (id, ensureStarted = true) {
|
|
740
|
-
const service = await this.#getServiceById(id, ensureStarted)
|
|
741
|
-
|
|
742
|
-
return sendViaITC(service, 'getServiceConfig')
|
|
743
|
-
}
|
|
744
|
-
|
|
745
|
-
async getServiceEnv (id, ensureStarted = true) {
|
|
746
|
-
const service = await this.#getServiceById(id, ensureStarted)
|
|
747
|
-
|
|
748
|
-
return sendViaITC(service, 'getServiceEnv')
|
|
749
|
-
}
|
|
750
|
-
|
|
751
|
-
async getServiceOpenapiSchema (id) {
|
|
752
|
-
const service = await this.#getServiceById(id, true)
|
|
753
|
-
|
|
754
|
-
return sendViaITC(service, 'getServiceOpenAPISchema')
|
|
755
|
-
}
|
|
823
|
+
for (const [application, { count }] of Object.entries(this.#workers.configuration)) {
|
|
824
|
+
for (let i = 0; i < count; i++) {
|
|
825
|
+
const label = `${application}:${i}`
|
|
826
|
+
const worker = this.#workers.get(label)
|
|
756
827
|
|
|
757
|
-
|
|
758
|
-
|
|
828
|
+
status[label] = await sendViaITC(worker, 'getCustomReadinessCheck')
|
|
829
|
+
}
|
|
830
|
+
}
|
|
759
831
|
|
|
760
|
-
return
|
|
832
|
+
return status
|
|
761
833
|
}
|
|
762
834
|
|
|
763
835
|
async getMetrics (format = 'json') {
|
|
@@ -765,26 +837,30 @@ class Runtime extends EventEmitter {
|
|
|
765
837
|
|
|
766
838
|
for (const worker of this.#workers.values()) {
|
|
767
839
|
try {
|
|
768
|
-
// The
|
|
840
|
+
// The application might be temporarily unavailable
|
|
769
841
|
if (worker[kWorkerStatus] !== 'started') {
|
|
770
842
|
continue
|
|
771
843
|
}
|
|
772
844
|
|
|
773
|
-
const
|
|
774
|
-
if (
|
|
845
|
+
const applicationMetrics = await sendViaITC(worker, 'getMetrics', format)
|
|
846
|
+
if (applicationMetrics) {
|
|
775
847
|
if (metrics === null) {
|
|
776
848
|
metrics = format === 'json' ? [] : ''
|
|
777
849
|
}
|
|
778
850
|
|
|
779
851
|
if (format === 'json') {
|
|
780
|
-
metrics.push(...
|
|
852
|
+
metrics.push(...applicationMetrics)
|
|
781
853
|
} else {
|
|
782
|
-
metrics +=
|
|
854
|
+
metrics += applicationMetrics
|
|
783
855
|
}
|
|
784
856
|
}
|
|
785
857
|
} catch (e) {
|
|
786
|
-
// The
|
|
787
|
-
if (
|
|
858
|
+
// The application exited while we were sending the ITC, skip it
|
|
859
|
+
if (
|
|
860
|
+
e.code === 'PLT_RUNTIME_APPLICATION_NOT_STARTED' ||
|
|
861
|
+
e.code === 'PLT_RUNTIME_APPLICATION_EXIT' ||
|
|
862
|
+
e.code === 'PLT_RUNTIME_APPLICATION_WORKER_EXIT'
|
|
863
|
+
) {
|
|
788
864
|
continue
|
|
789
865
|
}
|
|
790
866
|
|
|
@@ -817,7 +893,7 @@ class Runtime extends EventEmitter {
|
|
|
817
893
|
'http_request_all_summary_seconds'
|
|
818
894
|
]
|
|
819
895
|
|
|
820
|
-
const
|
|
896
|
+
const applicationsMetrics = {}
|
|
821
897
|
|
|
822
898
|
for (const metric of metrics) {
|
|
823
899
|
const { name, values } = metric
|
|
@@ -826,15 +902,15 @@ class Runtime extends EventEmitter {
|
|
|
826
902
|
if (!values || values.length === 0) continue
|
|
827
903
|
|
|
828
904
|
const labels = values[0].labels
|
|
829
|
-
const
|
|
905
|
+
const applicationId = labels?.applicationId
|
|
830
906
|
|
|
831
|
-
if (!
|
|
832
|
-
throw new Error('Missing
|
|
907
|
+
if (!applicationId) {
|
|
908
|
+
throw new Error('Missing applicationId label in metrics')
|
|
833
909
|
}
|
|
834
910
|
|
|
835
|
-
let
|
|
836
|
-
if (!
|
|
837
|
-
|
|
911
|
+
let applicationMetrics = applicationsMetrics[applicationId]
|
|
912
|
+
if (!applicationMetrics) {
|
|
913
|
+
applicationMetrics = {
|
|
838
914
|
cpu: 0,
|
|
839
915
|
rss: 0,
|
|
840
916
|
totalHeapSize: 0,
|
|
@@ -849,45 +925,45 @@ class Runtime extends EventEmitter {
|
|
|
849
925
|
p99: 0
|
|
850
926
|
}
|
|
851
927
|
}
|
|
852
|
-
|
|
928
|
+
applicationsMetrics[applicationId] = applicationMetrics
|
|
853
929
|
}
|
|
854
930
|
|
|
855
|
-
parsePromMetric(
|
|
931
|
+
parsePromMetric(applicationMetrics, metric)
|
|
856
932
|
}
|
|
857
933
|
|
|
858
|
-
function parsePromMetric (
|
|
934
|
+
function parsePromMetric (applicationMetrics, promMetric) {
|
|
859
935
|
const { name } = promMetric
|
|
860
936
|
|
|
861
937
|
if (name === 'process_cpu_percent_usage') {
|
|
862
|
-
|
|
938
|
+
applicationMetrics.cpu = promMetric.values[0].value
|
|
863
939
|
return
|
|
864
940
|
}
|
|
865
941
|
if (name === 'process_resident_memory_bytes') {
|
|
866
|
-
|
|
942
|
+
applicationMetrics.rss = promMetric.values[0].value
|
|
867
943
|
return
|
|
868
944
|
}
|
|
869
945
|
if (name === 'nodejs_heap_size_total_bytes') {
|
|
870
|
-
|
|
946
|
+
applicationMetrics.totalHeapSize = promMetric.values[0].value
|
|
871
947
|
return
|
|
872
948
|
}
|
|
873
949
|
if (name === 'nodejs_heap_size_used_bytes') {
|
|
874
|
-
|
|
950
|
+
applicationMetrics.usedHeapSize = promMetric.values[0].value
|
|
875
951
|
return
|
|
876
952
|
}
|
|
877
953
|
if (name === 'nodejs_heap_space_size_total_bytes') {
|
|
878
954
|
const newSpaceSize = promMetric.values.find(value => value.labels.space === 'new')
|
|
879
955
|
const oldSpaceSize = promMetric.values.find(value => value.labels.space === 'old')
|
|
880
956
|
|
|
881
|
-
|
|
882
|
-
|
|
957
|
+
applicationMetrics.newSpaceSize = newSpaceSize.value
|
|
958
|
+
applicationMetrics.oldSpaceSize = oldSpaceSize.value
|
|
883
959
|
return
|
|
884
960
|
}
|
|
885
961
|
if (name === 'nodejs_eventloop_utilization') {
|
|
886
|
-
|
|
962
|
+
applicationMetrics.elu = promMetric.values[0].value
|
|
887
963
|
return
|
|
888
964
|
}
|
|
889
965
|
if (name === 'http_request_all_summary_seconds') {
|
|
890
|
-
|
|
966
|
+
applicationMetrics.latency = {
|
|
891
967
|
p50: promMetric.values.find(value => value.labels.quantile === 0.5)?.value || 0,
|
|
892
968
|
p90: promMetric.values.find(value => value.labels.quantile === 0.9)?.value || 0,
|
|
893
969
|
p95: promMetric.values.find(value => value.labels.quantile === 0.95)?.value || 0,
|
|
@@ -899,7 +975,7 @@ class Runtime extends EventEmitter {
|
|
|
899
975
|
return {
|
|
900
976
|
version: 1,
|
|
901
977
|
date: new Date().toISOString(),
|
|
902
|
-
|
|
978
|
+
applications: applicationsMetrics
|
|
903
979
|
}
|
|
904
980
|
} catch (err) {
|
|
905
981
|
// If any metric is missing, return nothing
|
|
@@ -909,123 +985,157 @@ class Runtime extends EventEmitter {
|
|
|
909
985
|
}
|
|
910
986
|
}
|
|
911
987
|
|
|
912
|
-
|
|
913
|
-
|
|
988
|
+
getSharedContext () {
|
|
989
|
+
return this.#sharedContext
|
|
990
|
+
}
|
|
914
991
|
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
} catch (e) {
|
|
918
|
-
// The service exports no meta, return an empty object
|
|
919
|
-
if (e.code === 'PLT_ITC_HANDLER_NOT_FOUND') {
|
|
920
|
-
return {}
|
|
921
|
-
}
|
|
992
|
+
async getApplicationResourcesInfo (id) {
|
|
993
|
+
const workers = this.#workers.getCount(id)
|
|
922
994
|
|
|
923
|
-
|
|
924
|
-
|
|
995
|
+
const worker = await this.#getWorkerById(id, 0, false, false)
|
|
996
|
+
const health = worker[kConfig].health
|
|
997
|
+
|
|
998
|
+
return { workers, health }
|
|
925
999
|
}
|
|
926
1000
|
|
|
927
|
-
|
|
928
|
-
this.
|
|
1001
|
+
getApplicationsIds () {
|
|
1002
|
+
return this.#config.applications.map(application => application.id)
|
|
929
1003
|
}
|
|
930
1004
|
|
|
931
|
-
|
|
932
|
-
|
|
1005
|
+
async getApplications (allowUnloaded = false) {
|
|
1006
|
+
return {
|
|
1007
|
+
entrypoint: this.#entrypointId,
|
|
1008
|
+
production: this.#isProduction,
|
|
1009
|
+
applications: await Promise.all(
|
|
1010
|
+
this.getApplicationsIds().map(id => this.getApplicationDetails(id, allowUnloaded))
|
|
1011
|
+
)
|
|
1012
|
+
}
|
|
933
1013
|
}
|
|
934
1014
|
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
1015
|
+
async getWorkers () {
|
|
1016
|
+
const status = {}
|
|
1017
|
+
|
|
1018
|
+
for (const [application, { count }] of Object.entries(this.#workers.configuration)) {
|
|
1019
|
+
for (let i = 0; i < count; i++) {
|
|
1020
|
+
const label = `${application}:${i}`
|
|
1021
|
+
const worker = this.#workers.get(label)
|
|
1022
|
+
|
|
1023
|
+
status[label] = {
|
|
1024
|
+
application,
|
|
1025
|
+
worker: i,
|
|
1026
|
+
status: worker?.[kWorkerStatus] ?? 'exited',
|
|
1027
|
+
thread: worker?.threadId
|
|
1028
|
+
}
|
|
1029
|
+
}
|
|
938
1030
|
}
|
|
939
1031
|
|
|
940
|
-
return
|
|
1032
|
+
return status
|
|
941
1033
|
}
|
|
942
1034
|
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
return
|
|
946
|
-
}
|
|
1035
|
+
async getApplicationMeta (id) {
|
|
1036
|
+
const application = await this.#getApplicationById(id)
|
|
947
1037
|
|
|
948
|
-
|
|
1038
|
+
try {
|
|
1039
|
+
return await sendViaITC(application, 'getApplicationMeta')
|
|
1040
|
+
} catch (e) {
|
|
1041
|
+
// The application exports no meta, return an empty object
|
|
1042
|
+
if (e.code === 'PLT_ITC_HANDLER_NOT_FOUND') {
|
|
1043
|
+
return {}
|
|
1044
|
+
}
|
|
1045
|
+
|
|
1046
|
+
throw e
|
|
1047
|
+
}
|
|
949
1048
|
}
|
|
950
1049
|
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
1050
|
+
async getApplicationDetails (id, allowUnloaded = false) {
|
|
1051
|
+
let application
|
|
1052
|
+
|
|
1053
|
+
try {
|
|
1054
|
+
application = await this.#getApplicationById(id)
|
|
1055
|
+
} catch (e) {
|
|
1056
|
+
if (allowUnloaded) {
|
|
1057
|
+
return { id, status: 'stopped' }
|
|
1058
|
+
}
|
|
1059
|
+
|
|
1060
|
+
throw e
|
|
954
1061
|
}
|
|
955
1062
|
|
|
956
|
-
|
|
957
|
-
}
|
|
1063
|
+
const { entrypoint, localUrl } = application[kConfig]
|
|
958
1064
|
|
|
959
|
-
|
|
960
|
-
const {
|
|
1065
|
+
const status = await sendViaITC(application, 'getStatus')
|
|
1066
|
+
const { type, version, dependencies } = await sendViaITC(application, 'getApplicationInfo')
|
|
961
1067
|
|
|
962
|
-
|
|
963
|
-
|
|
1068
|
+
const applicationDetails = {
|
|
1069
|
+
id,
|
|
1070
|
+
type,
|
|
1071
|
+
status,
|
|
1072
|
+
dependencies,
|
|
1073
|
+
version,
|
|
1074
|
+
localUrl,
|
|
1075
|
+
entrypoint
|
|
964
1076
|
}
|
|
965
1077
|
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
promises.push(this.#sharedHttpCache.deleteKeys(keys))
|
|
1078
|
+
if (this.#isProduction) {
|
|
1079
|
+
applicationDetails.workers = this.#workers.getCount(id)
|
|
969
1080
|
}
|
|
970
1081
|
|
|
971
|
-
if (
|
|
972
|
-
|
|
1082
|
+
if (entrypoint) {
|
|
1083
|
+
applicationDetails.url = status === 'started' ? this.#url : null
|
|
973
1084
|
}
|
|
974
1085
|
|
|
975
|
-
return
|
|
1086
|
+
return applicationDetails
|
|
976
1087
|
}
|
|
977
1088
|
|
|
978
|
-
async
|
|
979
|
-
|
|
1089
|
+
async getApplication (id, ensureStarted = true) {
|
|
1090
|
+
return this.#getApplicationById(id, ensureStarted)
|
|
1091
|
+
}
|
|
980
1092
|
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
} catch (e) {
|
|
984
|
-
// The service exports no meta, return an empty object
|
|
985
|
-
if (e.code === 'PLT_ITC_HANDLER_NOT_FOUND') {
|
|
986
|
-
return {}
|
|
987
|
-
}
|
|
1093
|
+
async getApplicationConfig (id, ensureStarted = true) {
|
|
1094
|
+
const application = await this.#getApplicationById(id, ensureStarted)
|
|
988
1095
|
|
|
989
|
-
|
|
990
|
-
}
|
|
1096
|
+
return sendViaITC(application, 'getApplicationConfig')
|
|
991
1097
|
}
|
|
992
1098
|
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
if (worker[kForwardEvents]) {
|
|
996
|
-
worker[kITC].notify('runtime:event', { event, payload })
|
|
997
|
-
}
|
|
998
|
-
}
|
|
1099
|
+
async getApplicationEnv (id, ensureStarted = true) {
|
|
1100
|
+
const application = await this.#getApplicationById(id, ensureStarted)
|
|
999
1101
|
|
|
1000
|
-
|
|
1001
|
-
return super.emit(event, payload)
|
|
1102
|
+
return sendViaITC(application, 'getApplicationEnv')
|
|
1002
1103
|
}
|
|
1003
1104
|
|
|
1004
|
-
async
|
|
1005
|
-
const
|
|
1105
|
+
async getApplicationOpenapiSchema (id) {
|
|
1106
|
+
const application = await this.#getApplicationById(id, true)
|
|
1006
1107
|
|
|
1007
|
-
|
|
1008
|
-
|
|
1108
|
+
return sendViaITC(application, 'getApplicationOpenAPISchema')
|
|
1109
|
+
}
|
|
1009
1110
|
|
|
1010
|
-
|
|
1111
|
+
async getApplicationGraphqlSchema (id) {
|
|
1112
|
+
const application = await this.#getApplicationById(id, true)
|
|
1011
1113
|
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1114
|
+
return sendViaITC(application, 'getApplicationGraphQLSchema')
|
|
1115
|
+
}
|
|
1116
|
+
|
|
1117
|
+
#getHttpCacheValue ({ request }) {
|
|
1118
|
+
if (!this.#sharedHttpCache) {
|
|
1119
|
+
return
|
|
1015
1120
|
}
|
|
1016
1121
|
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1122
|
+
return this.#sharedHttpCache.getValue(request)
|
|
1123
|
+
}
|
|
1124
|
+
|
|
1125
|
+
#setHttpCacheValue ({ request, response, payload }) {
|
|
1126
|
+
if (!this.#sharedHttpCache) {
|
|
1127
|
+
return
|
|
1022
1128
|
}
|
|
1023
1129
|
|
|
1024
|
-
return
|
|
1130
|
+
return this.#sharedHttpCache.setValue(request, response, payload)
|
|
1025
1131
|
}
|
|
1026
1132
|
|
|
1027
|
-
|
|
1028
|
-
|
|
1133
|
+
#deleteHttpCacheValue ({ request }) {
|
|
1134
|
+
if (!this.#sharedHttpCache) {
|
|
1135
|
+
return
|
|
1136
|
+
}
|
|
1137
|
+
|
|
1138
|
+
return this.#sharedHttpCache.delete(request)
|
|
1029
1139
|
}
|
|
1030
1140
|
|
|
1031
1141
|
async #setDispatcher (undiciConfig) {
|
|
@@ -1055,23 +1165,65 @@ class Runtime extends EventEmitter {
|
|
|
1055
1165
|
this.logger.info(`Platformatic is now listening at ${this.#url}`)
|
|
1056
1166
|
}
|
|
1057
1167
|
|
|
1058
|
-
async #
|
|
1059
|
-
|
|
1168
|
+
async #setupApplications () {
|
|
1169
|
+
const config = this.#config
|
|
1170
|
+
const setupInvocations = []
|
|
1171
|
+
|
|
1172
|
+
// Parse all applications and verify we're not missing any path or resolved application
|
|
1173
|
+
for (const applicationConfig of config.applications) {
|
|
1174
|
+
// If there is no application path, check if the application was resolved
|
|
1175
|
+
if (!applicationConfig.path) {
|
|
1176
|
+
if (applicationConfig.url) {
|
|
1177
|
+
// Try to backfill the path for external applications
|
|
1178
|
+
applicationConfig.path = join(this.#root, config.resolvedApplicationsBasePath, applicationConfig.id)
|
|
1179
|
+
|
|
1180
|
+
if (!existsSync(applicationConfig.path)) {
|
|
1181
|
+
const executable = globalThis.platformatic?.executable ?? 'platformatic'
|
|
1182
|
+
this.logger.error(
|
|
1183
|
+
`The path for application "%s" does not exist. Please run "${executable} resolve" and try again.`,
|
|
1184
|
+
applicationConfig.id
|
|
1185
|
+
)
|
|
1186
|
+
|
|
1187
|
+
await this.closeAndThrow(new RuntimeAbortedError())
|
|
1188
|
+
}
|
|
1189
|
+
} else {
|
|
1190
|
+
this.logger.error(
|
|
1191
|
+
'The application "%s" has no path defined. Please check your configuration and try again.',
|
|
1192
|
+
applicationConfig.id
|
|
1193
|
+
)
|
|
1194
|
+
|
|
1195
|
+
await this.closeAndThrow(new RuntimeAbortedError())
|
|
1196
|
+
}
|
|
1197
|
+
}
|
|
1198
|
+
|
|
1199
|
+
setupInvocations.push([applicationConfig])
|
|
1200
|
+
}
|
|
1201
|
+
|
|
1202
|
+
await executeInParallel(this.#setupApplication.bind(this), setupInvocations, this.#concurrency)
|
|
1203
|
+
}
|
|
1204
|
+
|
|
1205
|
+
async #setupApplication (applicationConfig) {
|
|
1206
|
+
if (this.#status === 'stopping' || this.#status === 'closed') {
|
|
1207
|
+
return
|
|
1208
|
+
}
|
|
1060
1209
|
|
|
1061
1210
|
const config = this.#config
|
|
1062
|
-
const workersCount = await this.#workers.getCount(
|
|
1063
|
-
const id =
|
|
1211
|
+
const workersCount = await this.#workers.getCount(applicationConfig.id)
|
|
1212
|
+
const id = applicationConfig.id
|
|
1213
|
+
const setupInvocations = []
|
|
1064
1214
|
|
|
1065
1215
|
for (let i = 0; i < workersCount; i++) {
|
|
1066
|
-
|
|
1216
|
+
setupInvocations.push([config, applicationConfig, workersCount, id, i])
|
|
1067
1217
|
}
|
|
1068
1218
|
|
|
1069
|
-
this.
|
|
1219
|
+
await executeInParallel(this.#setupWorker.bind(this), setupInvocations, this.#concurrency)
|
|
1220
|
+
|
|
1221
|
+
this.emit('application:init', id)
|
|
1070
1222
|
}
|
|
1071
1223
|
|
|
1072
|
-
async #setupWorker (config,
|
|
1224
|
+
async #setupWorker (config, applicationConfig, workersCount, applicationId, index, enabled = true) {
|
|
1073
1225
|
const { restartOnError } = config
|
|
1074
|
-
const workerId = `${
|
|
1226
|
+
const workerId = `${applicationId}:${index}`
|
|
1075
1227
|
|
|
1076
1228
|
// Handle inspector
|
|
1077
1229
|
let inspectorOptions
|
|
@@ -1085,36 +1237,40 @@ class Runtime extends EventEmitter {
|
|
|
1085
1237
|
}
|
|
1086
1238
|
|
|
1087
1239
|
if (config.telemetry) {
|
|
1088
|
-
|
|
1240
|
+
applicationConfig.telemetry = {
|
|
1089
1241
|
...config.telemetry,
|
|
1090
|
-
...
|
|
1091
|
-
|
|
1242
|
+
...applicationConfig.telemetry,
|
|
1243
|
+
applicationName: `${config.telemetry.applicationName}-${applicationConfig.id}`
|
|
1092
1244
|
}
|
|
1093
1245
|
}
|
|
1094
1246
|
|
|
1095
|
-
const errorLabel = this.#workerExtendedLabel(
|
|
1096
|
-
const health = deepmerge(config.health ?? {},
|
|
1247
|
+
const errorLabel = this.#workerExtendedLabel(applicationId, index, workersCount)
|
|
1248
|
+
const health = deepmerge(config.health ?? {}, applicationConfig.health ?? {})
|
|
1097
1249
|
|
|
1098
1250
|
const execArgv = []
|
|
1099
1251
|
|
|
1100
|
-
if (!
|
|
1252
|
+
if (!applicationConfig.skipTelemetryHooks && config.telemetry && config.telemetry.enabled !== false) {
|
|
1253
|
+
const require = createRequire(import.meta.url)
|
|
1254
|
+
const telemetryPath = require.resolve('@platformatic/telemetry')
|
|
1255
|
+
const openTelemetrySetupPath = join(telemetryPath, '..', 'lib', 'node-telemetry.js')
|
|
1101
1256
|
const hookUrl = pathToFileURL(require.resolve('@opentelemetry/instrumentation/hook.mjs'))
|
|
1257
|
+
|
|
1102
1258
|
// We need the following because otherwise some open telemetry instrumentations won't work with ESM (like express)
|
|
1103
1259
|
// see: https://github.com/open-telemetry/opentelemetry-js/blob/main/doc/esm-support.md#instrumentation-hook-required-for-esm
|
|
1104
1260
|
execArgv.push('--import', `data:text/javascript, import { register } from 'node:module'; register('${hookUrl}')`)
|
|
1105
1261
|
execArgv.push('--import', pathToFileURL(openTelemetrySetupPath))
|
|
1106
1262
|
}
|
|
1107
1263
|
|
|
1108
|
-
if ((
|
|
1264
|
+
if ((applicationConfig.sourceMaps ?? config.sourceMaps) === true) {
|
|
1109
1265
|
execArgv.push('--enable-source-maps')
|
|
1110
1266
|
}
|
|
1111
1267
|
|
|
1112
1268
|
const workerEnv = structuredClone(this.#env)
|
|
1113
1269
|
|
|
1114
|
-
if (
|
|
1270
|
+
if (applicationConfig.nodeOptions?.trim().length > 0) {
|
|
1115
1271
|
const originalNodeOptions = workerEnv['NODE_OPTIONS'] ?? ''
|
|
1116
1272
|
|
|
1117
|
-
workerEnv['NODE_OPTIONS'] = `${originalNodeOptions} ${
|
|
1273
|
+
workerEnv['NODE_OPTIONS'] = `${originalNodeOptions} ${applicationConfig.nodeOptions}`.trim()
|
|
1118
1274
|
}
|
|
1119
1275
|
|
|
1120
1276
|
const maxHeapTotal =
|
|
@@ -1132,10 +1288,10 @@ class Runtime extends EventEmitter {
|
|
|
1132
1288
|
const worker = new Worker(kWorkerFile, {
|
|
1133
1289
|
workerData: {
|
|
1134
1290
|
config,
|
|
1135
|
-
|
|
1136
|
-
...
|
|
1291
|
+
applicationConfig: {
|
|
1292
|
+
...applicationConfig,
|
|
1137
1293
|
isProduction: this.#isProduction,
|
|
1138
|
-
configPatch: this.
|
|
1294
|
+
configPatch: this.#applicationsConfigsPatches.get(applicationId)
|
|
1139
1295
|
},
|
|
1140
1296
|
worker: {
|
|
1141
1297
|
id: workerId,
|
|
@@ -1145,7 +1301,7 @@ class Runtime extends EventEmitter {
|
|
|
1145
1301
|
inspectorOptions,
|
|
1146
1302
|
dirname: this.#root
|
|
1147
1303
|
},
|
|
1148
|
-
argv:
|
|
1304
|
+
argv: applicationConfig.arguments,
|
|
1149
1305
|
execArgv,
|
|
1150
1306
|
env: workerEnv,
|
|
1151
1307
|
resourceLimits: {
|
|
@@ -1156,13 +1312,14 @@ class Runtime extends EventEmitter {
|
|
|
1156
1312
|
stderr: true
|
|
1157
1313
|
})
|
|
1158
1314
|
|
|
1159
|
-
this.#handleWorkerStandardStreams(worker,
|
|
1315
|
+
this.#handleWorkerStandardStreams(worker, applicationId, workersCount > 1 ? index : undefined)
|
|
1160
1316
|
|
|
1161
1317
|
// Make sure the listener can handle a lot of API requests at once before raising a warning
|
|
1162
1318
|
worker.setMaxListeners(1e3)
|
|
1163
1319
|
|
|
1164
|
-
// Track
|
|
1165
|
-
const eventPayload = {
|
|
1320
|
+
// Track application exiting
|
|
1321
|
+
const eventPayload = { application: applicationId, worker: index, workersCount }
|
|
1322
|
+
|
|
1166
1323
|
worker.once('exit', code => {
|
|
1167
1324
|
if (worker[kWorkerStatus] === 'exited') {
|
|
1168
1325
|
return
|
|
@@ -1170,7 +1327,7 @@ class Runtime extends EventEmitter {
|
|
|
1170
1327
|
|
|
1171
1328
|
const started = worker[kWorkerStatus] === 'started'
|
|
1172
1329
|
worker[kWorkerStatus] = 'exited'
|
|
1173
|
-
this.emit('
|
|
1330
|
+
this.emit('application:worker:exited', eventPayload)
|
|
1174
1331
|
|
|
1175
1332
|
this.#cleanupWorker(worker)
|
|
1176
1333
|
|
|
@@ -1181,13 +1338,13 @@ class Runtime extends EventEmitter {
|
|
|
1181
1338
|
// Wait for the next tick so that crashed from the thread are logged first
|
|
1182
1339
|
setImmediate(() => {
|
|
1183
1340
|
if (started && (!config.watch || code !== 0)) {
|
|
1184
|
-
this.emit('
|
|
1341
|
+
this.emit('application:worker:error', { ...eventPayload, code })
|
|
1185
1342
|
this.#broadcastWorkers()
|
|
1186
1343
|
|
|
1187
1344
|
this.logger.warn(`The ${errorLabel} unexpectedly exited with code ${code}.`)
|
|
1188
1345
|
}
|
|
1189
1346
|
|
|
1190
|
-
// Restart the
|
|
1347
|
+
// Restart the application if it was started
|
|
1191
1348
|
if (started && this.#status === 'started') {
|
|
1192
1349
|
if (restartOnError > 0) {
|
|
1193
1350
|
if (restartOnError < IMMEDIATE_RESTART_MAX_THRESHOLD) {
|
|
@@ -1196,28 +1353,29 @@ class Runtime extends EventEmitter {
|
|
|
1196
1353
|
this.logger.warn(`The ${errorLabel} will be restarted in ${restartOnError}ms ...`)
|
|
1197
1354
|
}
|
|
1198
1355
|
|
|
1199
|
-
this.#restartCrashedWorker(config,
|
|
1200
|
-
|
|
1201
|
-
|
|
1356
|
+
this.#restartCrashedWorker(config, applicationConfig, workersCount, applicationId, index, false, 0).catch(
|
|
1357
|
+
err => {
|
|
1358
|
+
this.logger.error({ err: ensureLoggableError(err) }, `${errorLabel} could not be restarted.`)
|
|
1359
|
+
}
|
|
1360
|
+
)
|
|
1202
1361
|
} else {
|
|
1203
|
-
this.emit('
|
|
1362
|
+
this.emit('application:worker:unvailable', eventPayload)
|
|
1204
1363
|
this.logger.warn(`The ${errorLabel} is no longer available.`)
|
|
1205
1364
|
}
|
|
1206
1365
|
}
|
|
1207
1366
|
})
|
|
1208
1367
|
})
|
|
1209
1368
|
|
|
1210
|
-
worker[kId] = workersCount > 1 ? workerId :
|
|
1369
|
+
worker[kId] = workersCount > 1 ? workerId : applicationId
|
|
1211
1370
|
worker[kFullId] = workerId
|
|
1212
|
-
worker[
|
|
1371
|
+
worker[kApplicationId] = applicationId
|
|
1213
1372
|
worker[kWorkerId] = workersCount > 1 ? index : undefined
|
|
1214
1373
|
worker[kWorkerStatus] = 'boot'
|
|
1215
|
-
worker[kForwardEvents] = false
|
|
1216
1374
|
|
|
1217
1375
|
if (inspectorOptions) {
|
|
1218
1376
|
worker[kInspectorOptions] = {
|
|
1219
1377
|
port: inspectorOptions.port,
|
|
1220
|
-
id:
|
|
1378
|
+
id: applicationId,
|
|
1221
1379
|
dirname: this.#root
|
|
1222
1380
|
}
|
|
1223
1381
|
}
|
|
@@ -1226,41 +1384,36 @@ class Runtime extends EventEmitter {
|
|
|
1226
1384
|
worker[kITC] = new ITC({
|
|
1227
1385
|
name: workerId + '-runtime',
|
|
1228
1386
|
port: worker,
|
|
1229
|
-
handlers:
|
|
1230
|
-
...this.#workerITCHandlers,
|
|
1231
|
-
setEventsForwarding (value) {
|
|
1232
|
-
worker[kForwardEvents] = value
|
|
1233
|
-
}
|
|
1234
|
-
}
|
|
1387
|
+
handlers: this.#workerITCHandlers
|
|
1235
1388
|
})
|
|
1236
1389
|
worker[kITC].listen()
|
|
1237
1390
|
|
|
1238
1391
|
// Forward events from the worker
|
|
1239
1392
|
worker[kITC].on('event', ({ event, payload }) => {
|
|
1240
|
-
this.emit(`
|
|
1393
|
+
this.emit(`application:worker:event:${event}`, { ...eventPayload, payload })
|
|
1241
1394
|
})
|
|
1242
1395
|
|
|
1243
1396
|
// Only activate watch for the first instance
|
|
1244
1397
|
if (index === 0) {
|
|
1245
|
-
// Handle
|
|
1398
|
+
// Handle applications changes
|
|
1246
1399
|
// This is not purposely activated on when this.#config.watch === true
|
|
1247
|
-
// so that
|
|
1248
|
-
// used by the
|
|
1400
|
+
// so that applications can eventually manually trigger a restart. This mechanism is current
|
|
1401
|
+
// used by the gateway.
|
|
1249
1402
|
worker[kITC].on('changed', async () => {
|
|
1250
|
-
this.emit('
|
|
1403
|
+
this.emit('application:worker:changed', eventPayload)
|
|
1251
1404
|
|
|
1252
1405
|
try {
|
|
1253
1406
|
const wasStarted = worker[kWorkerStatus].startsWith('start')
|
|
1254
|
-
await this.
|
|
1407
|
+
await this.stopApplication(applicationId)
|
|
1255
1408
|
|
|
1256
1409
|
if (wasStarted) {
|
|
1257
|
-
await this.
|
|
1410
|
+
await this.startApplication(applicationId)
|
|
1258
1411
|
}
|
|
1259
1412
|
|
|
1260
|
-
this.logger.info(`The
|
|
1261
|
-
this.emit('
|
|
1413
|
+
this.logger.info(`The application "${applicationId}" has been successfully reloaded ...`)
|
|
1414
|
+
this.emit('application:worker:reloaded', eventPayload)
|
|
1262
1415
|
|
|
1263
|
-
if (
|
|
1416
|
+
if (applicationConfig.entrypoint) {
|
|
1264
1417
|
this.#showUrl()
|
|
1265
1418
|
}
|
|
1266
1419
|
} catch (e) {
|
|
@@ -1274,27 +1427,19 @@ class Runtime extends EventEmitter {
|
|
|
1274
1427
|
this.#workers.set(workerId, worker)
|
|
1275
1428
|
|
|
1276
1429
|
// Setup the interceptor
|
|
1277
|
-
this.#meshInterceptor.route(
|
|
1430
|
+
this.#meshInterceptor.route(applicationId, worker)
|
|
1278
1431
|
}
|
|
1279
1432
|
|
|
1280
|
-
//
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
if (serviceConfig.entrypoint) {
|
|
1284
|
-
this.#entrypointId = serviceId
|
|
1285
|
-
}
|
|
1433
|
+
// Wait for initialization
|
|
1434
|
+
await waitEventFromITC(worker, 'init')
|
|
1286
1435
|
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
if (envVar) {
|
|
1290
|
-
serviceConfig.localServiceEnvVars.set(envVar, url)
|
|
1291
|
-
}
|
|
1436
|
+
if (applicationConfig.entrypoint) {
|
|
1437
|
+
this.#entrypointId = applicationId
|
|
1292
1438
|
}
|
|
1293
1439
|
|
|
1294
|
-
|
|
1295
|
-
worker[kConfig] = { ...serviceConfig, health, workers: workersCount }
|
|
1440
|
+
worker[kConfig] = { ...applicationConfig, health, workers: workersCount }
|
|
1296
1441
|
worker[kWorkerStatus] = 'init'
|
|
1297
|
-
this.emit('
|
|
1442
|
+
this.emit('application:worker:init', eventPayload)
|
|
1298
1443
|
|
|
1299
1444
|
return worker
|
|
1300
1445
|
}
|
|
@@ -1312,7 +1457,7 @@ class Runtime extends EventEmitter {
|
|
|
1312
1457
|
return health
|
|
1313
1458
|
}
|
|
1314
1459
|
|
|
1315
|
-
#setupHealthCheck (config,
|
|
1460
|
+
#setupHealthCheck (config, applicationConfig, workersCount, id, index, worker, errorLabel, timeout) {
|
|
1316
1461
|
// Clear the timeout when exiting
|
|
1317
1462
|
worker.on('exit', () => clearTimeout(worker[kHealthCheckTimer]))
|
|
1318
1463
|
|
|
@@ -1338,9 +1483,9 @@ class Runtime extends EventEmitter {
|
|
|
1338
1483
|
health = { elu: -1, heapUsed: -1, heapTotal: -1 }
|
|
1339
1484
|
}
|
|
1340
1485
|
|
|
1341
|
-
this.emit('
|
|
1486
|
+
this.emit('application:worker:health', {
|
|
1342
1487
|
id: worker[kId],
|
|
1343
|
-
|
|
1488
|
+
application: id,
|
|
1344
1489
|
worker: index,
|
|
1345
1490
|
currentHealth: health,
|
|
1346
1491
|
unhealthy,
|
|
@@ -1367,14 +1512,14 @@ class Runtime extends EventEmitter {
|
|
|
1367
1512
|
|
|
1368
1513
|
if (unhealthyChecks === maxUnhealthyChecks) {
|
|
1369
1514
|
try {
|
|
1370
|
-
this.emit('
|
|
1515
|
+
this.emit('application:worker:unhealthy', { application: id, worker: index })
|
|
1371
1516
|
|
|
1372
1517
|
this.logger.error(
|
|
1373
1518
|
{ elu: health.elu, maxELU, memoryUsage: health.heapUsed, maxMemoryUsage: maxHeapUsed },
|
|
1374
1519
|
`The ${errorLabel} is unhealthy. Replacing it ...`
|
|
1375
1520
|
)
|
|
1376
1521
|
|
|
1377
|
-
await this.#replaceWorker(config,
|
|
1522
|
+
await this.#replaceWorker(config, applicationConfig, workersCount, id, index, worker)
|
|
1378
1523
|
} catch (e) {
|
|
1379
1524
|
this.logger.error(
|
|
1380
1525
|
{ elu: health.elu, maxELU, memoryUsage: health.heapUsed, maxMemoryUsage: maxHeapUsed },
|
|
@@ -1391,7 +1536,7 @@ class Runtime extends EventEmitter {
|
|
|
1391
1536
|
|
|
1392
1537
|
async #startWorker (
|
|
1393
1538
|
config,
|
|
1394
|
-
|
|
1539
|
+
applicationConfig,
|
|
1395
1540
|
workersCount,
|
|
1396
1541
|
id,
|
|
1397
1542
|
index,
|
|
@@ -1410,16 +1555,16 @@ class Runtime extends EventEmitter {
|
|
|
1410
1555
|
worker = await this.#getWorkerById(id, index, false, false)
|
|
1411
1556
|
}
|
|
1412
1557
|
|
|
1413
|
-
const eventPayload = {
|
|
1558
|
+
const eventPayload = { application: id, worker: index, workersCount }
|
|
1414
1559
|
|
|
1415
|
-
// The
|
|
1560
|
+
// The application was stopped, recreate the thread
|
|
1416
1561
|
if (!worker) {
|
|
1417
|
-
await this.#
|
|
1562
|
+
await this.#setupApplication(applicationConfig, index)
|
|
1418
1563
|
worker = await this.#getWorkerById(id, index)
|
|
1419
1564
|
}
|
|
1420
1565
|
|
|
1421
1566
|
worker[kWorkerStatus] = 'starting'
|
|
1422
|
-
this.emit('
|
|
1567
|
+
this.emit('application:worker:starting', eventPayload)
|
|
1423
1568
|
|
|
1424
1569
|
try {
|
|
1425
1570
|
let workerUrl
|
|
@@ -1427,10 +1572,10 @@ class Runtime extends EventEmitter {
|
|
|
1427
1572
|
workerUrl = await executeWithTimeout(sendViaITC(worker, 'start'), config.startTimeout)
|
|
1428
1573
|
|
|
1429
1574
|
if (workerUrl === kTimeout) {
|
|
1430
|
-
this.emit('
|
|
1575
|
+
this.emit('application:worker:startTimeout', eventPayload)
|
|
1431
1576
|
this.logger.info(`The ${label} failed to start in ${config.startTimeout}ms. Forcefully killing the thread.`)
|
|
1432
1577
|
worker.terminate()
|
|
1433
|
-
throw new
|
|
1578
|
+
throw new ApplicationStartTimeoutError(id, config.startTimeout)
|
|
1434
1579
|
}
|
|
1435
1580
|
} else {
|
|
1436
1581
|
workerUrl = await sendViaITC(worker, 'start')
|
|
@@ -1443,7 +1588,7 @@ class Runtime extends EventEmitter {
|
|
|
1443
1588
|
}
|
|
1444
1589
|
|
|
1445
1590
|
worker[kWorkerStatus] = 'started'
|
|
1446
|
-
this.emit('
|
|
1591
|
+
this.emit('application:worker:started', eventPayload)
|
|
1447
1592
|
this.#broadcastWorkers()
|
|
1448
1593
|
|
|
1449
1594
|
if (!silent) {
|
|
@@ -1456,7 +1601,7 @@ class Runtime extends EventEmitter {
|
|
|
1456
1601
|
// however, the health event will start when the worker is started
|
|
1457
1602
|
this.#setupHealthCheck(
|
|
1458
1603
|
config,
|
|
1459
|
-
|
|
1604
|
+
applicationConfig,
|
|
1460
1605
|
workersCount,
|
|
1461
1606
|
id,
|
|
1462
1607
|
index,
|
|
@@ -1467,6 +1612,7 @@ class Runtime extends EventEmitter {
|
|
|
1467
1612
|
}
|
|
1468
1613
|
} catch (err) {
|
|
1469
1614
|
const error = ensureError(err)
|
|
1615
|
+
worker[kITC].notify('application:worker:start:processed')
|
|
1470
1616
|
|
|
1471
1617
|
// TODO: handle port allocation error here
|
|
1472
1618
|
if (error.code === 'EADDRINUSE' || error.code === 'EACCES') throw error
|
|
@@ -1474,21 +1620,21 @@ class Runtime extends EventEmitter {
|
|
|
1474
1620
|
this.#cleanupWorker(worker)
|
|
1475
1621
|
|
|
1476
1622
|
if (worker[kWorkerStatus] !== 'exited') {
|
|
1477
|
-
// This prevent the exit handler to restart
|
|
1623
|
+
// This prevent the exit handler to restart application
|
|
1478
1624
|
worker[kWorkerStatus] = 'exited'
|
|
1479
1625
|
|
|
1480
1626
|
// Wait for the worker to exit gracefully, otherwise we terminate it
|
|
1481
|
-
const waitTimeout = await executeWithTimeout(once(worker, 'exit'), config.gracefulShutdown.
|
|
1627
|
+
const waitTimeout = await executeWithTimeout(once(worker, 'exit'), config.gracefulShutdown.application)
|
|
1482
1628
|
|
|
1483
1629
|
if (waitTimeout === kTimeout) {
|
|
1484
1630
|
await worker.terminate()
|
|
1485
1631
|
}
|
|
1486
1632
|
}
|
|
1487
1633
|
|
|
1488
|
-
this.emit('
|
|
1634
|
+
this.emit('application:worker:start:error', { ...eventPayload, error })
|
|
1489
1635
|
|
|
1490
|
-
if (error.code !== '
|
|
1491
|
-
this.logger.error({ err: ensureLoggableError(error) }, `Failed to start ${label}
|
|
1636
|
+
if (error.code !== 'PLT_RUNTIME_APPLICATION_START_TIMEOUT') {
|
|
1637
|
+
this.logger.error({ err: ensureLoggableError(error) }, `Failed to start ${label}: ${error.message}`)
|
|
1492
1638
|
}
|
|
1493
1639
|
|
|
1494
1640
|
const restartOnError = config.restartOnError
|
|
@@ -1499,6 +1645,7 @@ class Runtime extends EventEmitter {
|
|
|
1499
1645
|
|
|
1500
1646
|
if (bootstrapAttempt++ >= MAX_BOOTSTRAP_ATTEMPTS || restartOnError === 0) {
|
|
1501
1647
|
this.logger.error(`Failed to start ${label} after ${MAX_BOOTSTRAP_ATTEMPTS} attempts.`)
|
|
1648
|
+
this.emit('application:worker:start:failed', { ...eventPayload, error })
|
|
1502
1649
|
throw error
|
|
1503
1650
|
}
|
|
1504
1651
|
|
|
@@ -1512,11 +1659,11 @@ class Runtime extends EventEmitter {
|
|
|
1512
1659
|
)
|
|
1513
1660
|
}
|
|
1514
1661
|
|
|
1515
|
-
await this.#restartCrashedWorker(config,
|
|
1662
|
+
await this.#restartCrashedWorker(config, applicationConfig, workersCount, id, index, silent, bootstrapAttempt)
|
|
1516
1663
|
}
|
|
1517
1664
|
}
|
|
1518
1665
|
|
|
1519
|
-
async #stopWorker (workersCount, id, index, silent, worker
|
|
1666
|
+
async #stopWorker (workersCount, id, index, silent, worker, dependents) {
|
|
1520
1667
|
if (!worker) {
|
|
1521
1668
|
worker = await this.#getWorkerById(id, index, false, false)
|
|
1522
1669
|
}
|
|
@@ -1530,11 +1677,11 @@ class Runtime extends EventEmitter {
|
|
|
1530
1677
|
return this.#discardWorker(worker)
|
|
1531
1678
|
}
|
|
1532
1679
|
|
|
1533
|
-
const eventPayload = {
|
|
1680
|
+
const eventPayload = { application: id, worker: index, workersCount }
|
|
1534
1681
|
|
|
1535
1682
|
worker[kWorkerStatus] = 'stopping'
|
|
1536
1683
|
worker[kITC].removeAllListeners('changed')
|
|
1537
|
-
this.emit('
|
|
1684
|
+
this.emit('application:worker:stopping', eventPayload)
|
|
1538
1685
|
|
|
1539
1686
|
const label = this.#workerExtendedLabel(id, index, workersCount)
|
|
1540
1687
|
|
|
@@ -1547,11 +1694,15 @@ class Runtime extends EventEmitter {
|
|
|
1547
1694
|
|
|
1548
1695
|
// Always send the stop message, it will shut down workers that only had ITC and interceptors setup
|
|
1549
1696
|
try {
|
|
1550
|
-
await executeWithTimeout(sendViaITC(worker, 'stop'), exitTimeout)
|
|
1697
|
+
await executeWithTimeout(sendViaITC(worker, 'stop', { force: !!this.error, dependents }), exitTimeout)
|
|
1551
1698
|
} catch (error) {
|
|
1552
|
-
this.emit('
|
|
1699
|
+
this.emit('application:worker:stop:error', eventPayload)
|
|
1553
1700
|
this.logger.info({ error: ensureLoggableError(error) }, `Failed to stop ${label}. Killing a worker thread.`)
|
|
1554
1701
|
} finally {
|
|
1702
|
+
worker[kITC].notify('application:worker:stop:processed')
|
|
1703
|
+
// Wait for the processed message to be received
|
|
1704
|
+
await sleep(1)
|
|
1705
|
+
|
|
1555
1706
|
worker[kITC].close()
|
|
1556
1707
|
}
|
|
1557
1708
|
|
|
@@ -1559,19 +1710,19 @@ class Runtime extends EventEmitter {
|
|
|
1559
1710
|
this.logger.info(`Stopped the ${label}...`)
|
|
1560
1711
|
}
|
|
1561
1712
|
|
|
1562
|
-
// Wait for the worker thread to finish, we're going to create a new one if the
|
|
1713
|
+
// Wait for the worker thread to finish, we're going to create a new one if the application is ever restarted
|
|
1563
1714
|
const res = await executeWithTimeout(exitPromise, exitTimeout)
|
|
1564
1715
|
|
|
1565
1716
|
// If the worker didn't exit in time, kill it
|
|
1566
1717
|
if (res === kTimeout) {
|
|
1567
|
-
this.emit('
|
|
1718
|
+
this.emit('application:worker:exit:timeout', eventPayload)
|
|
1568
1719
|
await worker.terminate()
|
|
1569
1720
|
}
|
|
1570
1721
|
|
|
1571
1722
|
await this.#avoidOutOfOrderThreadLogs()
|
|
1572
1723
|
|
|
1573
1724
|
worker[kWorkerStatus] = 'stopped'
|
|
1574
|
-
this.emit('
|
|
1725
|
+
this.emit('application:worker:stopped', eventPayload)
|
|
1575
1726
|
this.#broadcastWorkers()
|
|
1576
1727
|
}
|
|
1577
1728
|
|
|
@@ -1588,18 +1739,20 @@ class Runtime extends EventEmitter {
|
|
|
1588
1739
|
}
|
|
1589
1740
|
|
|
1590
1741
|
async #discardWorker (worker) {
|
|
1591
|
-
this.#meshInterceptor.unroute(worker[
|
|
1742
|
+
this.#meshInterceptor.unroute(worker[kApplicationId], worker, true)
|
|
1592
1743
|
worker.removeAllListeners('exit')
|
|
1593
1744
|
await worker.terminate()
|
|
1594
1745
|
|
|
1595
1746
|
return this.#cleanupWorker(worker)
|
|
1596
1747
|
}
|
|
1597
1748
|
|
|
1598
|
-
#workerExtendedLabel (
|
|
1599
|
-
return workersCount > 1
|
|
1749
|
+
#workerExtendedLabel (applicationId, workerId, workersCount) {
|
|
1750
|
+
return workersCount > 1
|
|
1751
|
+
? `worker ${workerId} of the application "${applicationId}"`
|
|
1752
|
+
: `application "${applicationId}"`
|
|
1600
1753
|
}
|
|
1601
1754
|
|
|
1602
|
-
async #restartCrashedWorker (config,
|
|
1755
|
+
async #restartCrashedWorker (config, applicationConfig, workersCount, id, index, silent, bootstrapAttempt) {
|
|
1603
1756
|
const workerId = `${id}:${index}`
|
|
1604
1757
|
|
|
1605
1758
|
let restartPromise = this.#restartingWorkers.get(workerId)
|
|
@@ -1619,8 +1772,8 @@ class Runtime extends EventEmitter {
|
|
|
1619
1772
|
}
|
|
1620
1773
|
|
|
1621
1774
|
try {
|
|
1622
|
-
await this.#setupWorker(config,
|
|
1623
|
-
await this.#startWorker(config,
|
|
1775
|
+
await this.#setupWorker(config, applicationConfig, workersCount, id, index)
|
|
1776
|
+
await this.#startWorker(config, applicationConfig, workersCount, id, index, silent, bootstrapAttempt)
|
|
1624
1777
|
|
|
1625
1778
|
this.logger.info(
|
|
1626
1779
|
`The ${this.#workerExtendedLabel(id, index, workersCount)} has been successfully restarted ...`
|
|
@@ -1647,13 +1800,13 @@ class Runtime extends EventEmitter {
|
|
|
1647
1800
|
await restartPromise
|
|
1648
1801
|
}
|
|
1649
1802
|
|
|
1650
|
-
async #replaceWorker (config,
|
|
1651
|
-
const workerId = `${
|
|
1803
|
+
async #replaceWorker (config, applicationConfig, workersCount, applicationId, index, worker) {
|
|
1804
|
+
const workerId = `${applicationId}:${index}`
|
|
1652
1805
|
let newWorker
|
|
1653
1806
|
|
|
1654
1807
|
try {
|
|
1655
1808
|
// Create a new worker
|
|
1656
|
-
newWorker = await this.#setupWorker(config,
|
|
1809
|
+
newWorker = await this.#setupWorker(config, applicationConfig, workersCount, applicationId, index, false)
|
|
1657
1810
|
|
|
1658
1811
|
// Make sure the runtime hasn't been stopped in the meanwhile
|
|
1659
1812
|
if (this.#status !== 'started') {
|
|
@@ -1661,7 +1814,7 @@ class Runtime extends EventEmitter {
|
|
|
1661
1814
|
}
|
|
1662
1815
|
|
|
1663
1816
|
// Add the worker to the mesh
|
|
1664
|
-
await this.#startWorker(config,
|
|
1817
|
+
await this.#startWorker(config, applicationConfig, workersCount, applicationId, index, false, 0, newWorker, true)
|
|
1665
1818
|
|
|
1666
1819
|
// Make sure the runtime hasn't been stopped in the meanwhile
|
|
1667
1820
|
if (this.#status !== 'started') {
|
|
@@ -1669,7 +1822,7 @@ class Runtime extends EventEmitter {
|
|
|
1669
1822
|
}
|
|
1670
1823
|
|
|
1671
1824
|
this.#workers.set(workerId, newWorker)
|
|
1672
|
-
this.#meshInterceptor.route(
|
|
1825
|
+
this.#meshInterceptor.route(applicationId, newWorker)
|
|
1673
1826
|
|
|
1674
1827
|
// Remove the old worker and then kill it
|
|
1675
1828
|
await sendViaITC(worker, 'removeFromMesh')
|
|
@@ -1678,52 +1831,54 @@ class Runtime extends EventEmitter {
|
|
|
1678
1831
|
throw e
|
|
1679
1832
|
}
|
|
1680
1833
|
|
|
1681
|
-
await this.#stopWorker(workersCount,
|
|
1834
|
+
await this.#stopWorker(workersCount, applicationId, index, false, worker, [])
|
|
1682
1835
|
}
|
|
1683
1836
|
|
|
1684
|
-
async #
|
|
1685
|
-
// If the
|
|
1837
|
+
async #getApplicationById (applicationId, ensureStarted = false, mustExist = true) {
|
|
1838
|
+
// If the applicationId includes the worker, properly split
|
|
1686
1839
|
let workerId
|
|
1687
|
-
const matched =
|
|
1840
|
+
const matched = applicationId.match(/^(.+):(\d+)$/)
|
|
1688
1841
|
|
|
1689
1842
|
if (matched) {
|
|
1690
|
-
|
|
1843
|
+
applicationId = matched[1]
|
|
1691
1844
|
workerId = matched[2]
|
|
1692
1845
|
}
|
|
1693
1846
|
|
|
1694
|
-
return this.#getWorkerById(
|
|
1847
|
+
return this.#getWorkerById(applicationId, workerId, ensureStarted, mustExist)
|
|
1695
1848
|
}
|
|
1696
1849
|
|
|
1697
|
-
async #getWorkerById (
|
|
1850
|
+
async #getWorkerById (applicationId, workerId, ensureStarted = false, mustExist = true) {
|
|
1698
1851
|
let worker
|
|
1699
1852
|
|
|
1700
1853
|
if (typeof workerId !== 'undefined') {
|
|
1701
|
-
worker = this.#workers.get(`${
|
|
1854
|
+
worker = this.#workers.get(`${applicationId}:${workerId}`)
|
|
1702
1855
|
} else {
|
|
1703
|
-
worker = this.#workers.next(
|
|
1856
|
+
worker = this.#workers.next(applicationId)
|
|
1704
1857
|
}
|
|
1705
1858
|
|
|
1859
|
+
const applicationsIds = this.getApplicationsIds()
|
|
1860
|
+
|
|
1706
1861
|
if (!worker) {
|
|
1707
|
-
if (!mustExist &&
|
|
1862
|
+
if (!mustExist && applicationsIds.includes(applicationId)) {
|
|
1708
1863
|
return null
|
|
1709
1864
|
}
|
|
1710
1865
|
|
|
1711
|
-
if (
|
|
1866
|
+
if (applicationsIds.includes(applicationId)) {
|
|
1712
1867
|
const availableWorkers = Array.from(this.#workers.keys())
|
|
1713
|
-
.filter(key => key.startsWith(
|
|
1868
|
+
.filter(key => key.startsWith(applicationId + ':'))
|
|
1714
1869
|
.map(key => key.split(':')[1])
|
|
1715
1870
|
.join(', ')
|
|
1716
|
-
throw new
|
|
1871
|
+
throw new WorkerNotFoundError(workerId, applicationId, availableWorkers)
|
|
1717
1872
|
} else {
|
|
1718
|
-
throw new
|
|
1873
|
+
throw new ApplicationNotFoundError(applicationId, applicationsIds.join(', '))
|
|
1719
1874
|
}
|
|
1720
1875
|
}
|
|
1721
1876
|
|
|
1722
1877
|
if (ensureStarted) {
|
|
1723
|
-
const
|
|
1878
|
+
const applicationStatus = await sendViaITC(worker, 'getStatus')
|
|
1724
1879
|
|
|
1725
|
-
if (
|
|
1726
|
-
throw new
|
|
1880
|
+
if (applicationStatus !== 'started') {
|
|
1881
|
+
throw new ApplicationNotStartedError(applicationId)
|
|
1727
1882
|
}
|
|
1728
1883
|
}
|
|
1729
1884
|
|
|
@@ -1744,17 +1899,17 @@ class Runtime extends EventEmitter {
|
|
|
1744
1899
|
continue
|
|
1745
1900
|
}
|
|
1746
1901
|
|
|
1747
|
-
const
|
|
1748
|
-
let
|
|
1902
|
+
const application = worker[kApplicationId]
|
|
1903
|
+
let applicationWorkers = workers.get(application)
|
|
1749
1904
|
|
|
1750
|
-
if (!
|
|
1751
|
-
|
|
1752
|
-
workers.set(
|
|
1905
|
+
if (!applicationWorkers) {
|
|
1906
|
+
applicationWorkers = []
|
|
1907
|
+
workers.set(application, applicationWorkers)
|
|
1753
1908
|
}
|
|
1754
1909
|
|
|
1755
|
-
|
|
1910
|
+
applicationWorkers.push({
|
|
1756
1911
|
id: worker[kId],
|
|
1757
|
-
|
|
1912
|
+
application: worker[kApplicationId],
|
|
1758
1913
|
worker: worker[kWorkerId],
|
|
1759
1914
|
thread: worker.threadId
|
|
1760
1915
|
})
|
|
@@ -1767,8 +1922,8 @@ class Runtime extends EventEmitter {
|
|
|
1767
1922
|
}
|
|
1768
1923
|
}
|
|
1769
1924
|
|
|
1770
|
-
async #getWorkerMessagingChannel ({
|
|
1771
|
-
const target = await this.#getWorkerById(
|
|
1925
|
+
async #getWorkerMessagingChannel ({ application, worker }, context) {
|
|
1926
|
+
const target = await this.#getWorkerById(application, worker, true, true)
|
|
1772
1927
|
|
|
1773
1928
|
const { port1, port2 } = new MessageChannel()
|
|
1774
1929
|
|
|
@@ -1779,11 +1934,11 @@ class Runtime extends EventEmitter {
|
|
|
1779
1934
|
)
|
|
1780
1935
|
|
|
1781
1936
|
if (response === kTimeout) {
|
|
1782
|
-
throw new
|
|
1937
|
+
throw new MessagingError(application, 'Timeout while establishing a communication channel.')
|
|
1783
1938
|
}
|
|
1784
1939
|
|
|
1785
1940
|
context.transferList = [port2]
|
|
1786
|
-
this.emit('
|
|
1941
|
+
this.emit('application:worker:messagingChannel', { application, worker })
|
|
1787
1942
|
return port2
|
|
1788
1943
|
}
|
|
1789
1944
|
|
|
@@ -1795,8 +1950,8 @@ class Runtime extends EventEmitter {
|
|
|
1795
1950
|
return packageJson
|
|
1796
1951
|
}
|
|
1797
1952
|
|
|
1798
|
-
#handleWorkerStandardStreams (worker,
|
|
1799
|
-
const binding = { name:
|
|
1953
|
+
#handleWorkerStandardStreams (worker, applicationId, workerId) {
|
|
1954
|
+
const binding = { name: applicationId }
|
|
1800
1955
|
|
|
1801
1956
|
if (typeof workerId !== 'undefined') {
|
|
1802
1957
|
binding.worker = workerId
|
|
@@ -1893,187 +2048,83 @@ class Runtime extends EventEmitter {
|
|
|
1893
2048
|
}
|
|
1894
2049
|
}
|
|
1895
2050
|
|
|
1896
|
-
async
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
const worker = await this.#getWorkerById(id, 0, false, false)
|
|
1900
|
-
const health = worker[kConfig].health
|
|
1901
|
-
|
|
1902
|
-
return { workers, health }
|
|
1903
|
-
}
|
|
1904
|
-
|
|
1905
|
-
async #updateServiceConfigWorkers (serviceId, workers) {
|
|
1906
|
-
this.logger.info(`Updating service "${serviceId}" config workers to ${workers}`)
|
|
2051
|
+
async #updateApplicationConfigWorkers (applicationId, workers) {
|
|
2052
|
+
this.logger.info(`Updating application "${applicationId}" config workers to ${workers}`)
|
|
1907
2053
|
|
|
1908
|
-
this.#config.
|
|
1909
|
-
const
|
|
1910
|
-
this.#workers.setCount(
|
|
1911
|
-
|
|
2054
|
+
this.#config.applications.find(s => s.id === applicationId).workers = workers
|
|
2055
|
+
const application = await this.#getApplicationById(applicationId)
|
|
2056
|
+
this.#workers.setCount(applicationId, workers)
|
|
2057
|
+
application[kConfig].workers = workers
|
|
1912
2058
|
|
|
1913
2059
|
const promises = []
|
|
1914
2060
|
for (const [workerId, worker] of this.#workers.entries()) {
|
|
1915
|
-
if (workerId.startsWith(`${
|
|
1916
|
-
promises.push(sendViaITC(worker, 'updateWorkersCount', {
|
|
2061
|
+
if (workerId.startsWith(`${applicationId}:`)) {
|
|
2062
|
+
promises.push(sendViaITC(worker, 'updateWorkersCount', { applicationId, workers }))
|
|
1917
2063
|
}
|
|
1918
2064
|
}
|
|
1919
2065
|
|
|
1920
2066
|
const results = await Promise.allSettled(promises)
|
|
1921
2067
|
for (const result of results) {
|
|
1922
2068
|
if (result.status === 'rejected') {
|
|
1923
|
-
this.logger.error({ err: result.reason }, `Cannot update
|
|
2069
|
+
this.logger.error({ err: result.reason }, `Cannot update application "${applicationId}" workers`)
|
|
1924
2070
|
throw result.reason
|
|
1925
2071
|
}
|
|
1926
2072
|
}
|
|
1927
2073
|
}
|
|
1928
2074
|
|
|
1929
|
-
async #
|
|
1930
|
-
this.logger.info(`Updating
|
|
2075
|
+
async #updateApplicationConfigHealth (applicationId, health) {
|
|
2076
|
+
this.logger.info(`Updating application "${applicationId}" config health heap to ${JSON.stringify(health)}`)
|
|
1931
2077
|
const { maxHeapTotal, maxYoungGeneration } = health
|
|
1932
2078
|
|
|
1933
|
-
const
|
|
2079
|
+
const application = this.#config.applications.find(s => s.id === applicationId)
|
|
1934
2080
|
if (maxHeapTotal) {
|
|
1935
|
-
|
|
2081
|
+
application.health.maxHeapTotal = maxHeapTotal
|
|
1936
2082
|
}
|
|
1937
2083
|
if (maxYoungGeneration) {
|
|
1938
|
-
|
|
1939
|
-
}
|
|
1940
|
-
}
|
|
1941
|
-
|
|
1942
|
-
/**
|
|
1943
|
-
* Updates the resources of the services, such as the number of workers and health configurations (e.g., heap memory settings).
|
|
1944
|
-
*
|
|
1945
|
-
* This function handles three update scenarios for each service:
|
|
1946
|
-
* 1. **Updating workers only**: Adjusts the number of workers for the service.
|
|
1947
|
-
* 2. **Updating health configurations only**: Updates health parameters like `maxHeapTotal` or `maxYoungGeneration`.
|
|
1948
|
-
* 3. **Updating both workers and health configurations**: Scales the workers and also applies health settings.
|
|
1949
|
-
*
|
|
1950
|
-
* When updating both workers and health:
|
|
1951
|
-
* - **Scaling down workers**: Stops extra workers, then restarts the remaining workers with the previous settings.
|
|
1952
|
-
* - **Scaling up workers**: Starts new workers with the updated heap settings, then restarts the old workers with the updated settings.
|
|
1953
|
-
*
|
|
1954
|
-
* Scaling up new resources (workers and/or heap memory) may fails due to insufficient memory, in this case the operation may fail partially or entirely.
|
|
1955
|
-
* Scaling down is expected to succeed without issues.
|
|
1956
|
-
*
|
|
1957
|
-
* @param {Array<Object>} updates - An array of objects that define the updates for each service.
|
|
1958
|
-
* @param {string} updates[].service - The ID of the service to update.
|
|
1959
|
-
* @param {number} [updates[].workers] - The desired number of workers for the service. If omitted, workers will not be updated.
|
|
1960
|
-
* @param {Object} [updates[].health] - The health configuration to update for the service, which may include:
|
|
1961
|
-
* @param {string|number} [updates[].health.maxHeapTotal] - The maximum heap memory for the service. Can be a valid memory string (e.g., '1G', '512MB') or a number representing bytes.
|
|
1962
|
-
* @param {string|number} [updates[].health.maxYoungGeneration] - The maximum young generation memory for the service. Can be a valid memory string (e.g., '128MB') or a number representing bytes.
|
|
1963
|
-
*
|
|
1964
|
-
* @returns {Promise<Array<Object>>} - A promise that resolves to an array of reports for each service, detailing the success or failure of the operations:
|
|
1965
|
-
* - `service`: The service ID.
|
|
1966
|
-
* - `workers`: The workers' update report, including the current, new number of workers, started workers, and success status.
|
|
1967
|
-
* - `health`: The health update report, showing the current and new heap settings, updated workers, and success status.
|
|
1968
|
-
*
|
|
1969
|
-
* @example
|
|
1970
|
-
* await runtime.updateServicesResources([
|
|
1971
|
-
* { service: 'service-1', workers: 2, health: { maxHeapTotal: '1G', maxYoungGeneration: '128 MB' } },
|
|
1972
|
-
* { service: 'service-2', health: { maxHeapTotal: '1G' } },
|
|
1973
|
-
* { service: 'service-3', workers: 2 },
|
|
1974
|
-
* ])
|
|
1975
|
-
*
|
|
1976
|
-
* In this example:
|
|
1977
|
-
* - `service-1` will have 2 workers and updated heap memory configurations.
|
|
1978
|
-
* - `service-2` will have updated heap memory settings (without changing workers).
|
|
1979
|
-
* - `service-3` will have its workers set to 2 but no change in memory settings.
|
|
1980
|
-
*
|
|
1981
|
-
* @throws {InvalidArgumentError} - Throws if any update parameter is invalid, such as:
|
|
1982
|
-
* - Missing service ID.
|
|
1983
|
-
* - Invalid worker count (not a positive integer).
|
|
1984
|
-
* - Invalid memory size format for `maxHeapTotal` or `maxYoungGeneration`.
|
|
1985
|
-
* @throws {ServiceNotFoundError} - Throws if the specified service ID does not exist in the current service configuration.
|
|
1986
|
-
*/
|
|
1987
|
-
async updateServicesResources (updates) {
|
|
1988
|
-
if (this.#status === 'stopping' || this.#status === 'closed') {
|
|
1989
|
-
this.logger.warn('Cannot update service resources when the runtime is stopping or closed')
|
|
1990
|
-
return
|
|
1991
|
-
}
|
|
1992
|
-
|
|
1993
|
-
const ups = await this.#validateUpdateServiceResources(updates)
|
|
1994
|
-
const config = this.#config
|
|
1995
|
-
|
|
1996
|
-
const report = []
|
|
1997
|
-
for (const update of ups) {
|
|
1998
|
-
const { serviceId, config: serviceConfig, workers, health, currentWorkers, currentHealth } = update
|
|
1999
|
-
|
|
2000
|
-
if (workers && health) {
|
|
2001
|
-
const r = await this.#updateServiceWorkersAndHealth(
|
|
2002
|
-
serviceId,
|
|
2003
|
-
config,
|
|
2004
|
-
serviceConfig,
|
|
2005
|
-
workers,
|
|
2006
|
-
health,
|
|
2007
|
-
currentWorkers,
|
|
2008
|
-
currentHealth
|
|
2009
|
-
)
|
|
2010
|
-
report.push({
|
|
2011
|
-
service: serviceId,
|
|
2012
|
-
workers: r.workers,
|
|
2013
|
-
health: r.health
|
|
2014
|
-
})
|
|
2015
|
-
} else if (health) {
|
|
2016
|
-
const r = await this.#updateServiceHealth(
|
|
2017
|
-
serviceId,
|
|
2018
|
-
config,
|
|
2019
|
-
serviceConfig,
|
|
2020
|
-
currentWorkers,
|
|
2021
|
-
currentHealth,
|
|
2022
|
-
health
|
|
2023
|
-
)
|
|
2024
|
-
report.push({
|
|
2025
|
-
service: serviceId,
|
|
2026
|
-
health: r.health
|
|
2027
|
-
})
|
|
2028
|
-
} else if (workers) {
|
|
2029
|
-
const r = await this.#updateServiceWorkers(serviceId, config, serviceConfig, workers, currentWorkers)
|
|
2030
|
-
report.push({
|
|
2031
|
-
service: serviceId,
|
|
2032
|
-
workers: r.workers
|
|
2033
|
-
})
|
|
2034
|
-
}
|
|
2084
|
+
application.health.maxYoungGeneration = maxYoungGeneration
|
|
2035
2085
|
}
|
|
2036
|
-
|
|
2037
|
-
return report
|
|
2038
2086
|
}
|
|
2039
2087
|
|
|
2040
|
-
async #
|
|
2088
|
+
async #validateUpdateApplicationResources (updates) {
|
|
2041
2089
|
if (!Array.isArray(updates)) {
|
|
2042
|
-
throw new
|
|
2090
|
+
throw new InvalidArgumentError('updates', 'must be an array')
|
|
2043
2091
|
}
|
|
2044
2092
|
if (updates.length === 0) {
|
|
2045
|
-
throw new
|
|
2093
|
+
throw new InvalidArgumentError('updates', 'must have at least one element')
|
|
2046
2094
|
}
|
|
2047
2095
|
|
|
2048
2096
|
const config = this.#config
|
|
2049
2097
|
const validatedUpdates = []
|
|
2050
2098
|
for (const update of updates) {
|
|
2051
|
-
const {
|
|
2099
|
+
const { application: applicationId } = update
|
|
2052
2100
|
|
|
2053
|
-
if (!
|
|
2054
|
-
throw new
|
|
2101
|
+
if (!applicationId) {
|
|
2102
|
+
throw new InvalidArgumentError('application', 'must be a string')
|
|
2055
2103
|
}
|
|
2056
|
-
const
|
|
2057
|
-
if (!
|
|
2058
|
-
throw new
|
|
2104
|
+
const applicationConfig = config.applications.find(s => s.id === applicationId)
|
|
2105
|
+
if (!applicationConfig) {
|
|
2106
|
+
throw new ApplicationNotFoundError(applicationId, Array.from(this.getApplicationsIds()).join(', '))
|
|
2059
2107
|
}
|
|
2060
2108
|
|
|
2061
|
-
const { workers: currentWorkers, health: currentHealth } = await this.
|
|
2109
|
+
const { workers: currentWorkers, health: currentHealth } = await this.getApplicationResourcesInfo(applicationId)
|
|
2062
2110
|
|
|
2063
2111
|
let workers
|
|
2064
2112
|
if (update.workers !== undefined) {
|
|
2065
2113
|
if (typeof update.workers !== 'number') {
|
|
2066
|
-
throw new
|
|
2114
|
+
throw new InvalidArgumentError('workers', 'must be a number')
|
|
2067
2115
|
}
|
|
2068
2116
|
if (update.workers <= 0) {
|
|
2069
|
-
throw new
|
|
2117
|
+
throw new InvalidArgumentError('workers', 'must be greater than 0')
|
|
2070
2118
|
}
|
|
2071
2119
|
if (update.workers > MAX_WORKERS) {
|
|
2072
|
-
throw new
|
|
2120
|
+
throw new InvalidArgumentError('workers', `must be less than ${MAX_WORKERS}`)
|
|
2073
2121
|
}
|
|
2074
2122
|
|
|
2075
2123
|
if (currentWorkers === update.workers) {
|
|
2076
|
-
this.logger.warn(
|
|
2124
|
+
this.logger.warn(
|
|
2125
|
+
{ applicationId, workers: update.workers },
|
|
2126
|
+
'No change in the number of workers for application'
|
|
2127
|
+
)
|
|
2077
2128
|
} else {
|
|
2078
2129
|
workers = update.workers
|
|
2079
2130
|
}
|
|
@@ -2086,22 +2137,19 @@ class Runtime extends EventEmitter {
|
|
|
2086
2137
|
try {
|
|
2087
2138
|
maxHeapTotal = parseMemorySize(update.health.maxHeapTotal)
|
|
2088
2139
|
} catch {
|
|
2089
|
-
throw new
|
|
2140
|
+
throw new InvalidArgumentError('maxHeapTotal', 'must be a valid memory size')
|
|
2090
2141
|
}
|
|
2091
2142
|
} else if (typeof update.health.maxHeapTotal === 'number') {
|
|
2092
2143
|
maxHeapTotal = update.health.maxHeapTotal
|
|
2093
2144
|
if (update.health.maxHeapTotal <= 0) {
|
|
2094
|
-
throw new
|
|
2145
|
+
throw new InvalidArgumentError('maxHeapTotal', 'must be greater than 0')
|
|
2095
2146
|
}
|
|
2096
2147
|
} else {
|
|
2097
|
-
throw new
|
|
2098
|
-
'maxHeapTotal',
|
|
2099
|
-
'must be a number or a string representing a memory size'
|
|
2100
|
-
)
|
|
2148
|
+
throw new InvalidArgumentError('maxHeapTotal', 'must be a number or a string representing a memory size')
|
|
2101
2149
|
}
|
|
2102
2150
|
|
|
2103
2151
|
if (currentHealth.maxHeapTotal === maxHeapTotal) {
|
|
2104
|
-
this.logger.warn({
|
|
2152
|
+
this.logger.warn({ applicationId, maxHeapTotal }, 'No change in the max heap total for application')
|
|
2105
2153
|
maxHeapTotal = undefined
|
|
2106
2154
|
}
|
|
2107
2155
|
}
|
|
@@ -2111,22 +2159,25 @@ class Runtime extends EventEmitter {
|
|
|
2111
2159
|
try {
|
|
2112
2160
|
maxYoungGeneration = parseMemorySize(update.health.maxYoungGeneration)
|
|
2113
2161
|
} catch {
|
|
2114
|
-
throw new
|
|
2162
|
+
throw new InvalidArgumentError('maxYoungGeneration', 'must be a valid memory size')
|
|
2115
2163
|
}
|
|
2116
2164
|
} else if (typeof update.health.maxYoungGeneration === 'number') {
|
|
2117
2165
|
maxYoungGeneration = update.health.maxYoungGeneration
|
|
2118
2166
|
if (update.health.maxYoungGeneration <= 0) {
|
|
2119
|
-
throw new
|
|
2167
|
+
throw new InvalidArgumentError('maxYoungGeneration', 'must be greater than 0')
|
|
2120
2168
|
}
|
|
2121
2169
|
} else {
|
|
2122
|
-
throw new
|
|
2170
|
+
throw new InvalidArgumentError(
|
|
2123
2171
|
'maxYoungGeneration',
|
|
2124
2172
|
'must be a number or a string representing a memory size'
|
|
2125
2173
|
)
|
|
2126
2174
|
}
|
|
2127
2175
|
|
|
2128
2176
|
if (currentHealth.maxYoungGeneration && currentHealth.maxYoungGeneration === maxYoungGeneration) {
|
|
2129
|
-
this.logger.warn(
|
|
2177
|
+
this.logger.warn(
|
|
2178
|
+
{ applicationId, maxYoungGeneration },
|
|
2179
|
+
'No change in the max young generation for application'
|
|
2180
|
+
)
|
|
2130
2181
|
maxYoungGeneration = undefined
|
|
2131
2182
|
}
|
|
2132
2183
|
}
|
|
@@ -2143,17 +2194,24 @@ class Runtime extends EventEmitter {
|
|
|
2143
2194
|
health.maxYoungGeneration = maxYoungGeneration
|
|
2144
2195
|
}
|
|
2145
2196
|
}
|
|
2146
|
-
validatedUpdates.push({
|
|
2197
|
+
validatedUpdates.push({
|
|
2198
|
+
applicationId,
|
|
2199
|
+
config: applicationConfig,
|
|
2200
|
+
workers,
|
|
2201
|
+
health,
|
|
2202
|
+
currentWorkers,
|
|
2203
|
+
currentHealth
|
|
2204
|
+
})
|
|
2147
2205
|
}
|
|
2148
2206
|
}
|
|
2149
2207
|
|
|
2150
2208
|
return validatedUpdates
|
|
2151
2209
|
}
|
|
2152
2210
|
|
|
2153
|
-
async #
|
|
2154
|
-
|
|
2211
|
+
async #updateApplicationWorkersAndHealth (
|
|
2212
|
+
applicationId,
|
|
2155
2213
|
config,
|
|
2156
|
-
|
|
2214
|
+
applicationConfig,
|
|
2157
2215
|
workers,
|
|
2158
2216
|
health,
|
|
2159
2217
|
currentWorkers,
|
|
@@ -2161,12 +2219,18 @@ class Runtime extends EventEmitter {
|
|
|
2161
2219
|
) {
|
|
2162
2220
|
if (currentWorkers > workers) {
|
|
2163
2221
|
// stop workers
|
|
2164
|
-
const reportWorkers = await this.#
|
|
2222
|
+
const reportWorkers = await this.#updateApplicationWorkers(
|
|
2223
|
+
applicationId,
|
|
2224
|
+
config,
|
|
2225
|
+
applicationConfig,
|
|
2226
|
+
workers,
|
|
2227
|
+
currentWorkers
|
|
2228
|
+
)
|
|
2165
2229
|
// update heap for current workers
|
|
2166
|
-
const reportHealth = await this.#
|
|
2167
|
-
|
|
2230
|
+
const reportHealth = await this.#updateApplicationHealth(
|
|
2231
|
+
applicationId,
|
|
2168
2232
|
config,
|
|
2169
|
-
|
|
2233
|
+
applicationConfig,
|
|
2170
2234
|
workers,
|
|
2171
2235
|
currentHealth,
|
|
2172
2236
|
health
|
|
@@ -2174,15 +2238,21 @@ class Runtime extends EventEmitter {
|
|
|
2174
2238
|
|
|
2175
2239
|
return { workers: reportWorkers, health: reportHealth }
|
|
2176
2240
|
} else {
|
|
2177
|
-
// update
|
|
2178
|
-
await this.#
|
|
2241
|
+
// update application heap
|
|
2242
|
+
await this.#updateApplicationConfigHealth(applicationId, health)
|
|
2179
2243
|
// start new workers with new heap
|
|
2180
|
-
const reportWorkers = await this.#
|
|
2244
|
+
const reportWorkers = await this.#updateApplicationWorkers(
|
|
2245
|
+
applicationId,
|
|
2246
|
+
config,
|
|
2247
|
+
applicationConfig,
|
|
2248
|
+
workers,
|
|
2249
|
+
currentWorkers
|
|
2250
|
+
)
|
|
2181
2251
|
// update heap for current workers
|
|
2182
|
-
const reportHealth = await this.#
|
|
2183
|
-
|
|
2252
|
+
const reportHealth = await this.#updateApplicationHealth(
|
|
2253
|
+
applicationId,
|
|
2184
2254
|
config,
|
|
2185
|
-
|
|
2255
|
+
applicationConfig,
|
|
2186
2256
|
currentWorkers,
|
|
2187
2257
|
currentHealth,
|
|
2188
2258
|
health,
|
|
@@ -2193,10 +2263,10 @@ class Runtime extends EventEmitter {
|
|
|
2193
2263
|
}
|
|
2194
2264
|
}
|
|
2195
2265
|
|
|
2196
|
-
async #
|
|
2197
|
-
|
|
2266
|
+
async #updateApplicationHealth (
|
|
2267
|
+
applicationId,
|
|
2198
2268
|
config,
|
|
2199
|
-
|
|
2269
|
+
applicationConfig,
|
|
2200
2270
|
currentWorkers,
|
|
2201
2271
|
currentHealth,
|
|
2202
2272
|
health,
|
|
@@ -2209,16 +2279,16 @@ class Runtime extends EventEmitter {
|
|
|
2209
2279
|
}
|
|
2210
2280
|
try {
|
|
2211
2281
|
if (updateConfig) {
|
|
2212
|
-
await this.#
|
|
2282
|
+
await this.#updateApplicationConfigHealth(applicationId, health)
|
|
2213
2283
|
}
|
|
2214
2284
|
|
|
2215
2285
|
for (let i = 0; i < currentWorkers; i++) {
|
|
2216
2286
|
this.logger.info(
|
|
2217
2287
|
{ health: { current: currentHealth, new: health } },
|
|
2218
|
-
`Restarting
|
|
2288
|
+
`Restarting application "${applicationId}" worker ${i} to update config health heap...`
|
|
2219
2289
|
)
|
|
2220
2290
|
|
|
2221
|
-
const worker = await this.#getWorkerById(
|
|
2291
|
+
const worker = await this.#getWorkerById(applicationId, i)
|
|
2222
2292
|
if (health.maxHeapTotal) {
|
|
2223
2293
|
worker[kConfig].health.maxHeapTotal = health.maxHeapTotal
|
|
2224
2294
|
}
|
|
@@ -2226,22 +2296,22 @@ class Runtime extends EventEmitter {
|
|
|
2226
2296
|
worker[kConfig].health.maxYoungGeneration = health.maxYoungGeneration
|
|
2227
2297
|
}
|
|
2228
2298
|
|
|
2229
|
-
await this.#replaceWorker(config,
|
|
2299
|
+
await this.#replaceWorker(config, applicationConfig, currentWorkers, applicationId, i, worker)
|
|
2230
2300
|
report.updated.push(i)
|
|
2231
2301
|
this.logger.info(
|
|
2232
2302
|
{ health: { current: currentHealth, new: health } },
|
|
2233
|
-
`Restarted
|
|
2303
|
+
`Restarted application "${applicationId}" worker ${i}`
|
|
2234
2304
|
)
|
|
2235
2305
|
}
|
|
2236
2306
|
report.success = true
|
|
2237
2307
|
} catch (err) {
|
|
2238
2308
|
if (report.updated.length < 1) {
|
|
2239
|
-
this.logger.error({ err }, 'Cannot update
|
|
2240
|
-
await this.#
|
|
2309
|
+
this.logger.error({ err }, 'Cannot update application health heap, no worker updated')
|
|
2310
|
+
await this.#updateApplicationConfigHealth(applicationId, currentHealth)
|
|
2241
2311
|
} else {
|
|
2242
2312
|
this.logger.error(
|
|
2243
2313
|
{ err },
|
|
2244
|
-
`Cannot update
|
|
2314
|
+
`Cannot update application health heap, updated workers: ${report.updated.length} out of ${currentWorkers}`
|
|
2245
2315
|
)
|
|
2246
2316
|
}
|
|
2247
2317
|
report.success = false
|
|
@@ -2249,7 +2319,7 @@ class Runtime extends EventEmitter {
|
|
|
2249
2319
|
return report
|
|
2250
2320
|
}
|
|
2251
2321
|
|
|
2252
|
-
async #
|
|
2322
|
+
async #updateApplicationWorkers (applicationId, config, applicationConfig, workers, currentWorkers) {
|
|
2253
2323
|
const report = {
|
|
2254
2324
|
current: currentWorkers,
|
|
2255
2325
|
new: workers
|
|
@@ -2257,47 +2327,47 @@ class Runtime extends EventEmitter {
|
|
|
2257
2327
|
if (currentWorkers < workers) {
|
|
2258
2328
|
report.started = []
|
|
2259
2329
|
try {
|
|
2260
|
-
await this.#
|
|
2330
|
+
await this.#updateApplicationConfigWorkers(applicationId, workers)
|
|
2261
2331
|
for (let i = currentWorkers; i < workers; i++) {
|
|
2262
|
-
await this.#setupWorker(config,
|
|
2263
|
-
await this.#startWorker(config,
|
|
2332
|
+
await this.#setupWorker(config, applicationConfig, workers, applicationId, i)
|
|
2333
|
+
await this.#startWorker(config, applicationConfig, workers, applicationId, i, false, 0)
|
|
2264
2334
|
report.started.push(i)
|
|
2265
2335
|
}
|
|
2266
2336
|
report.success = true
|
|
2267
2337
|
} catch (err) {
|
|
2268
2338
|
if (report.started.length < 1) {
|
|
2269
|
-
this.logger.error({ err }, 'Cannot start
|
|
2270
|
-
await this.#
|
|
2339
|
+
this.logger.error({ err }, 'Cannot start application workers, no worker started')
|
|
2340
|
+
await this.#updateApplicationConfigWorkers(applicationId, currentWorkers)
|
|
2271
2341
|
} else {
|
|
2272
2342
|
this.logger.error(
|
|
2273
2343
|
{ err },
|
|
2274
|
-
`Cannot start
|
|
2344
|
+
`Cannot start application workers, started workers: ${report.started.length} out of ${workers}`
|
|
2275
2345
|
)
|
|
2276
|
-
await this.#
|
|
2346
|
+
await this.#updateApplicationConfigWorkers(applicationId, currentWorkers + report.started.length)
|
|
2277
2347
|
}
|
|
2278
2348
|
report.success = false
|
|
2279
2349
|
}
|
|
2280
2350
|
} else {
|
|
2281
|
-
// keep the current workers count until all the
|
|
2351
|
+
// keep the current workers count until all the application workers are all stopped
|
|
2282
2352
|
report.stopped = []
|
|
2283
2353
|
try {
|
|
2284
2354
|
for (let i = currentWorkers - 1; i >= workers; i--) {
|
|
2285
|
-
const worker = await this.#getWorkerById(
|
|
2355
|
+
const worker = await this.#getWorkerById(applicationId, i, false, false)
|
|
2286
2356
|
await sendViaITC(worker, 'removeFromMesh')
|
|
2287
|
-
await this.#stopWorker(currentWorkers,
|
|
2357
|
+
await this.#stopWorker(currentWorkers, applicationId, i, false, worker, [])
|
|
2288
2358
|
report.stopped.push(i)
|
|
2289
2359
|
}
|
|
2290
|
-
await this.#
|
|
2360
|
+
await this.#updateApplicationConfigWorkers(applicationId, workers)
|
|
2291
2361
|
report.success = true
|
|
2292
2362
|
} catch (err) {
|
|
2293
2363
|
if (report.stopped.length < 1) {
|
|
2294
|
-
this.logger.error({ err }, 'Cannot stop
|
|
2364
|
+
this.logger.error({ err }, 'Cannot stop application workers, no worker stopped')
|
|
2295
2365
|
} else {
|
|
2296
2366
|
this.logger.error(
|
|
2297
2367
|
{ err },
|
|
2298
|
-
`Cannot stop
|
|
2368
|
+
`Cannot stop application workers, stopped workers: ${report.stopped.length} out of ${workers}`
|
|
2299
2369
|
)
|
|
2300
|
-
await this.#
|
|
2370
|
+
await this.#updateApplicationConfigWorkers(applicationId, currentWorkers - report.stopped)
|
|
2301
2371
|
}
|
|
2302
2372
|
report.success = false
|
|
2303
2373
|
}
|
|
@@ -2305,5 +2375,3 @@ class Runtime extends EventEmitter {
|
|
|
2305
2375
|
return report
|
|
2306
2376
|
}
|
|
2307
2377
|
}
|
|
2308
|
-
|
|
2309
|
-
module.exports = { Runtime }
|