@nsshunt/stsappframework 3.2.1 → 3.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commonTypes.js +69 -0
- package/dist/commonTypes.js.map +1 -0
- package/dist/controller/stscontrollerbase.js +14 -0
- package/dist/controller/stscontrollerbase.js.map +1 -0
- package/dist/controller/stslatencycontroller.js +28 -0
- package/dist/controller/stslatencycontroller.js.map +1 -0
- package/dist/index.js.map +1 -0
- package/dist/logger/stsTransportLoggerWinston.js +22 -0
- package/dist/logger/stsTransportLoggerWinston.js.map +1 -0
- package/dist/logger/stsTransportWinston.js +43 -0
- package/dist/logger/stsTransportWinston.js.map +1 -0
- package/{src/middleware/serverNetworkMiddleware.ts → dist/middleware/serverNetworkMiddleware.js} +80 -124
- package/dist/middleware/serverNetworkMiddleware.js.map +1 -0
- package/dist/network.js +38 -0
- package/dist/network.js.map +1 -0
- package/dist/process/masterprocessbase.js +558 -0
- package/dist/process/masterprocessbase.js.map +1 -0
- package/{src/process/processbase.ts → dist/process/processbase.js} +150 -222
- package/dist/process/processbase.js.map +1 -0
- package/{src/process/serverprocessbase.ts → dist/process/serverprocessbase.js} +158 -212
- package/dist/process/serverprocessbase.js.map +1 -0
- package/dist/process/singleprocessbase.js +55 -0
- package/dist/process/singleprocessbase.js.map +1 -0
- package/{src/process/workerprocessbase.ts → dist/process/workerprocessbase.js} +93 -107
- package/dist/process/workerprocessbase.js.map +1 -0
- package/{src/publishertransports/publishTransportUtils.ts → dist/publishertransports/publishTransportUtils.js} +18 -24
- package/dist/publishertransports/publishTransportUtils.js.map +1 -0
- package/dist/route/stslatencyroute.js +15 -0
- package/dist/route/stslatencyroute.js.map +1 -0
- package/dist/route/stsrouterbase.js +21 -0
- package/dist/route/stsrouterbase.js.map +1 -0
- package/dist/stsexpressserver.js +115 -0
- package/dist/stsexpressserver.js.map +1 -0
- package/dist/validation/errors.js +10 -0
- package/dist/validation/errors.js.map +1 -0
- package/dist/vitesttesting/appConfig.js +93 -0
- package/dist/vitesttesting/appConfig.js.map +1 -0
- package/dist/vitesttesting/appSingleWSS.js +122 -0
- package/dist/vitesttesting/appSingleWSS.js.map +1 -0
- package/dist/vitesttesting/server.js +15 -0
- package/dist/vitesttesting/server.js.map +1 -0
- package/dist/vitesttesting/singleservertest.test.js +286 -0
- package/dist/vitesttesting/singleservertest.test.js.map +1 -0
- package/dist/vitesttesting/wsevents.js +3 -0
- package/dist/vitesttesting/wsevents.js.map +1 -0
- package/package.json +7 -3
- package/.github/dependabot.yml +0 -13
- package/.github/workflows/npm-publish.yml +0 -47
- package/.gitignore copy +0 -108
- package/build.sh +0 -37
- package/esbuild.config.js +0 -81
- package/eslint.config.mjs +0 -55
- package/jest/setEnvVars.js +0 -19
- package/keys/server.cert +0 -21
- package/keys/server.key +0 -28
- package/local-redis-stack.conf +0 -2
- package/run-grpc-client.sh +0 -2
- package/run-grpc-server.sh +0 -2
- package/run1.sh +0 -20
- package/run2.sh +0 -20
- package/run3.sh +0 -20
- package/runc1.sh +0 -19
- package/runc2.sh +0 -19
- package/runkafka.sh +0 -19
- package/runkafkaconsume01.sh +0 -21
- package/runkafkaconsume02.sh +0 -21
- package/runpromise.sh +0 -5
- package/runredis.sh +0 -5
- package/runredis1.sh +0 -4
- package/runredis2.sh +0 -24
- package/runredis3.sh +0 -4
- package/runtest1.sh +0 -19
- package/runtest2.sh +0 -19
- package/runtest_ipc_legacy.sh +0 -19
- package/runtest_ipcex.sh +0 -19
- package/runtest_redis.sh +0 -19
- package/runtest_ww.sh +0 -19
- package/src/commonTypes.ts +0 -374
- package/src/controller/stscontrollerbase.ts +0 -14
- package/src/controller/stslatencycontroller.ts +0 -26
- package/src/index.ts +0 -13
- package/src/logger/stsTransportLoggerWinston.ts +0 -24
- package/src/logger/stsTransportWinston.ts +0 -48
- package/src/network.ts +0 -36
- package/src/process/masterprocessbase.ts +0 -674
- package/src/process/singleprocessbase.ts +0 -63
- package/src/route/stslatencyroute.ts +0 -15
- package/src/route/stsrouterbase.ts +0 -21
- package/src/stsexpressserver.ts +0 -137
- package/src/validation/errors.ts +0 -6
- package/src/vitesttesting/appConfig.ts +0 -111
- package/src/vitesttesting/appSingleWSS.ts +0 -142
- package/src/vitesttesting/server.ts +0 -17
- package/src/vitesttesting/singleservertest.test.ts +0 -352
- package/src/vitesttesting/wsevents.ts +0 -44
- package/tsconfig.json +0 -42
- package/vite.config.ts +0 -19
|
@@ -1,674 +0,0 @@
|
|
|
1
|
-
/* eslint @typescript-eslint/no-explicit-any: 0, @typescript-eslint/no-unused-vars: 0 */ // --> OFF
|
|
2
|
-
import fs from "node:fs"
|
|
3
|
-
import si from 'systeminformation' // https://systeminformation.io/
|
|
4
|
-
|
|
5
|
-
import axios from 'axios';
|
|
6
|
-
|
|
7
|
-
import cluster, { Worker, Address } from 'node:cluster'
|
|
8
|
-
|
|
9
|
-
import chalk from 'chalk';
|
|
10
|
-
|
|
11
|
-
import express from 'express'
|
|
12
|
-
|
|
13
|
-
import { setupPrimary } from '@socket.io/cluster-adapter'
|
|
14
|
-
import { createServer as createServerHttps } from 'node:https'
|
|
15
|
-
import { createServer } from 'node:http'
|
|
16
|
-
import { AggregatorRegistry } from 'prom-client'
|
|
17
|
-
|
|
18
|
-
import { goptions, STSAxiosConfig, AgentManager } from '@nsshunt/stsconfig'
|
|
19
|
-
|
|
20
|
-
import { Gauge, GaugeTypes, InstrumentGaugeTelemetry, InstrumentGaugeOptions, InstrumentHistogramTelemetry } from '@nsshunt/stsobservability'
|
|
21
|
-
|
|
22
|
-
import { ProcessBase } from './processbase';
|
|
23
|
-
import { IMasterProcessBase, ProcessOptions } from './../commonTypes';
|
|
24
|
-
|
|
25
|
-
import { InstrumentDefinitions } from '@nsshunt/stsobservability'
|
|
26
|
-
import { IPCMessagePayload, IPCMessageCommand } from './../commonTypes'
|
|
27
|
-
|
|
28
|
-
import { STSTransportLoggerWinston } from './../logger/stsTransportLoggerWinston'
|
|
29
|
-
|
|
30
|
-
import { Sleep } from '@nsshunt/stsutils'
|
|
31
|
-
|
|
32
|
-
//import { GetFirstNetworkInterface } from './network';
|
|
33
|
-
|
|
34
|
-
export class MasterProcessBase extends ProcessBase implements IMasterProcessBase
|
|
35
|
-
{
|
|
36
|
-
//static WORKER_MESSAGE_EVENT = 'sts_worker_message_event';
|
|
37
|
-
#masterProcessExitTime = goptions.masterProcessExitTime;
|
|
38
|
-
#childProcessExitTime = goptions.childProcessExitTime;
|
|
39
|
-
#siValueObject = {
|
|
40
|
-
currentLoad: 'currentLoad'
|
|
41
|
-
}
|
|
42
|
-
#httpServer: any = null; // Prometheus cluster server. See https://github.com/siimon/prom-client/blob/master/example/cluster.js
|
|
43
|
-
#metricsServer: any= null;
|
|
44
|
-
#aggregatorRegistry: any = null;
|
|
45
|
-
#checkLatency: NodeJS.Timeout | null = null;
|
|
46
|
-
#killWorkers: Record<string, string> = { };
|
|
47
|
-
#workers = 1; // Start at 1 becuase of main thread
|
|
48
|
-
#shuttingDown = false;
|
|
49
|
-
#agentManager: AgentManager;
|
|
50
|
-
|
|
51
|
-
constructor(options: ProcessOptions)
|
|
52
|
-
{
|
|
53
|
-
super(options);
|
|
54
|
-
|
|
55
|
-
this.#agentManager = new AgentManager({
|
|
56
|
-
agentOptions: {
|
|
57
|
-
keepAlive: false, // Use basic defaults. This agent will not use keep-alive.
|
|
58
|
-
maxFreeSockets: goptions.maxFreeSockets,
|
|
59
|
-
maxSockets: goptions.maxSockets,
|
|
60
|
-
maxTotalSockets: goptions.maxTotalSockets,
|
|
61
|
-
timeout: goptions.timeout,
|
|
62
|
-
rejectUnauthorized: goptions.isProduction // Allows self-signed certificates if non-production
|
|
63
|
-
}
|
|
64
|
-
});
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
#LogErrorMessage(message: any) {
|
|
68
|
-
this.options.logger.error(message);
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
#LogDebugMessage(message: any) {
|
|
72
|
-
this.options.logger.debug(message);
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
override CollectAdditionalTelemetry(): void {
|
|
76
|
-
si.get(this.#siValueObject).then(data => {
|
|
77
|
-
this.UpdateInstrument(Gauge.CPU_SYSTEM_LOAD_GAUGE, {
|
|
78
|
-
val: data.currentLoad.currentLoad
|
|
79
|
-
} as InstrumentGaugeTelemetry);
|
|
80
|
-
});
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
override GetAdditionalInstruments(): InstrumentDefinitions {
|
|
84
|
-
return [
|
|
85
|
-
[ Gauge.CPU_SYSTEM_LOAD_GAUGE, GaugeTypes.INSTRUMENT_GAUGE, {
|
|
86
|
-
interval: goptions.instrumentationObservationInterval,
|
|
87
|
-
sampleSize: goptions.instrumentationTimeWindow
|
|
88
|
-
} as InstrumentGaugeOptions]
|
|
89
|
-
]
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
/**
|
|
93
|
-
* Note: msg removed from signature but will be passed at run-time.
|
|
94
|
-
* @param {*} msg
|
|
95
|
-
*/
|
|
96
|
-
|
|
97
|
-
WorkerMessageEvent(workerId: number, msg: any) {
|
|
98
|
-
return null;
|
|
99
|
-
}
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
#WorkerMessageEvent(workerId: number, msg: any) {
|
|
103
|
-
if (msg.command) {
|
|
104
|
-
this.WorkerMessageEvent(workerId, msg);
|
|
105
|
-
}
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
#InitCluster = () => {
|
|
109
|
-
cluster.on('listening', (worker: Worker, address: Address) => {
|
|
110
|
-
this.emit('clusterListening', worker, address);
|
|
111
|
-
let allListening = true;
|
|
112
|
-
for (const worker of Object.values(cluster.workers as NodeJS.Dict<Worker>)) {
|
|
113
|
-
if (worker?.isConnected) {
|
|
114
|
-
allListening = false;
|
|
115
|
-
break;
|
|
116
|
-
}
|
|
117
|
-
}
|
|
118
|
-
if (allListening) {
|
|
119
|
-
this.LogInfoMessage(`Service instance started.`);
|
|
120
|
-
this.emit('allListening');
|
|
121
|
-
}
|
|
122
|
-
});
|
|
123
|
-
};
|
|
124
|
-
|
|
125
|
-
// https://github.com/siimon/prom-client/blob/master/example/cluster.js
|
|
126
|
-
#SetupPrometheusForMaster = () => {
|
|
127
|
-
this.#metricsServer = express();
|
|
128
|
-
this.#aggregatorRegistry = new AggregatorRegistry();
|
|
129
|
-
|
|
130
|
-
switch (goptions.STSServerType) {
|
|
131
|
-
case 'EXPRESS_TLS' : {
|
|
132
|
-
const options = {
|
|
133
|
-
key: fs.readFileSync(this.options.httpsServerKeyPath),
|
|
134
|
-
cert: fs.readFileSync(this.options.httpsServerCertificatePath)
|
|
135
|
-
};
|
|
136
|
-
this.#httpServer = createServerHttps(options, this.#metricsServer);
|
|
137
|
-
}
|
|
138
|
-
break;
|
|
139
|
-
case 'EXPRESS' : {
|
|
140
|
-
this.#httpServer = createServer(this.#metricsServer);
|
|
141
|
-
}
|
|
142
|
-
}
|
|
143
|
-
//this.#httpServer.maxConnections = 50;
|
|
144
|
-
|
|
145
|
-
this.#metricsServer.get('/cluster_metrics', async (req: any, res: any) => {
|
|
146
|
-
try {
|
|
147
|
-
const metrics = await this.#aggregatorRegistry.clusterMetrics();
|
|
148
|
-
res.set('Content-Type', this.#aggregatorRegistry.contentType);
|
|
149
|
-
res.send(metrics);
|
|
150
|
-
} catch (ex: any) {
|
|
151
|
-
res.statusCode = 500;
|
|
152
|
-
res.send(ex.message);
|
|
153
|
-
}
|
|
154
|
-
});
|
|
155
|
-
|
|
156
|
-
//@@@ options wrong
|
|
157
|
-
|
|
158
|
-
try {
|
|
159
|
-
// https://stackoverflow.com/questions/21342828/node-express-unix-domain-socket-permissions
|
|
160
|
-
//@@httpServer.listen('/tmp/stsrest01.sock').on('listening', () =>
|
|
161
|
-
//@@httpServer.listen('/var/run/sts/stsrest01.sock').on('listening', () =>
|
|
162
|
-
//@@httpServer.listen('/var/lib/sts/stsrest01.sock').on('listening', () =>
|
|
163
|
-
this.#httpServer.listen(this.options.prometheusClusterPort, () => {
|
|
164
|
-
//@@chmodSync(this.options.port, 511);
|
|
165
|
-
}).on('listening', () =>
|
|
166
|
-
{
|
|
167
|
-
this.emit('prometheusScrapesListening', `${this.options.endpoint}:${this.options.prometheusClusterPort}/metrics`,
|
|
168
|
-
this.options.endpoint,
|
|
169
|
-
this.options.prometheusClusterPort);
|
|
170
|
-
this.LogInfoMessage(`Prometheus scrapes ready and live on ${this.options.endpoint}:${this.options.prometheusClusterPort}/metrics`);
|
|
171
|
-
});
|
|
172
|
-
} catch (error)
|
|
173
|
-
{
|
|
174
|
-
this.#LogErrorMessage(error);
|
|
175
|
-
throw error;
|
|
176
|
-
}
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
#CheckLatency = async () => {
|
|
180
|
-
const start = process.hrtime();
|
|
181
|
-
await this.#GetLatency();
|
|
182
|
-
this.#LatencyRequestCompleted(start);
|
|
183
|
-
}
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
#GetLatency = async (): Promise<any> => {
|
|
187
|
-
let retVal: any = null;
|
|
188
|
-
// We use port rather than hostport becuase this test going outside the service and comes back in as a regular client
|
|
189
|
-
const url = `${this.options.endpoint}:${this.options.port}${this.options.apiRoot}/latency`;
|
|
190
|
-
try {
|
|
191
|
-
retVal = await axios(new STSAxiosConfig(url, 'get')
|
|
192
|
-
.withDefaultHeaders()
|
|
193
|
-
.withAgentManager(this.#agentManager).config);
|
|
194
|
-
if (retVal.status !== 200) {
|
|
195
|
-
this.#LogDebugMessage(chalk.magenta(`Error (MasterProcessBase:#GetLatency): Invalid response from server: [${retVal.status}]`));
|
|
196
|
-
return null;
|
|
197
|
-
}
|
|
198
|
-
return retVal.data.detail;
|
|
199
|
-
} catch (error: any) {
|
|
200
|
-
this.#LogDebugMessage(chalk.red(`Error (MasterProcessBase:#GetLatency:catch): [${error}]`));
|
|
201
|
-
this.#LogDebugMessage(chalk.red(` url: [${url}]`));
|
|
202
|
-
if (error.response && error.response.data) {
|
|
203
|
-
this.#LogDebugMessage(chalk.red(` Details: [${JSON.stringify(error.response.data)}]`));
|
|
204
|
-
}
|
|
205
|
-
}
|
|
206
|
-
}
|
|
207
|
-
|
|
208
|
-
#LatencyRequestCompleted = (start: any) => {
|
|
209
|
-
// Update request duration histo data
|
|
210
|
-
let timeInMs = 0;
|
|
211
|
-
const end = process.hrtime(start);
|
|
212
|
-
timeInMs = (end[0]* 1000000000 + end[1]) / 1000000;
|
|
213
|
-
timeInMs = parseFloat(timeInMs.toFixed(4));
|
|
214
|
-
|
|
215
|
-
this.UpdateInstrument(Gauge.LATENCY_HISTOGRAM_GAUGE, {
|
|
216
|
-
val: timeInMs
|
|
217
|
-
} as InstrumentHistogramTelemetry);
|
|
218
|
-
|
|
219
|
-
this.UpdateInstrument(Gauge.LATENCY_GAUGE, {
|
|
220
|
-
val: timeInMs
|
|
221
|
-
} as InstrumentGaugeTelemetry);
|
|
222
|
-
};
|
|
223
|
-
|
|
224
|
-
SetupServer = async () => {
|
|
225
|
-
this.SetupInstrumentation();
|
|
226
|
-
setTimeout(() => {
|
|
227
|
-
this.SetupServerEx();
|
|
228
|
-
}, 100);
|
|
229
|
-
}
|
|
230
|
-
|
|
231
|
-
#_KillWorker = (id: string, signal: NodeJS.Signals, killProcess: boolean): boolean => {
|
|
232
|
-
try {
|
|
233
|
-
if (cluster.workers && cluster.workers[id]) {
|
|
234
|
-
const worker = cluster.workers[id] as Worker;
|
|
235
|
-
if (worker.process) {
|
|
236
|
-
this.LogInfoMessage(chalk.grey(`Sending terminate message `) + chalk.yellow(`(initiated by ${signal})`) + chalk.grey(` for worker PID: ${worker.process.pid}`));
|
|
237
|
-
const command: string = (killProcess ? 'TerminateAndKill' : 'Terminate');
|
|
238
|
-
try {
|
|
239
|
-
worker.send( { command } );
|
|
240
|
-
//worker.process.send( { command } );
|
|
241
|
-
} catch (error) {
|
|
242
|
-
this.LogInfoMessage(chalk.red(`MasterProcessBase:#_KillWorker() (1): id: [${id}], signal: [${signal}], killProcess: [${killProcess}], error: [${error}]`));
|
|
243
|
-
return false;
|
|
244
|
-
}
|
|
245
|
-
return true;
|
|
246
|
-
} else {
|
|
247
|
-
this.LogInfoMessage(chalk.red(`MasterProcessBase:#_KillWorker(): Could not kill worker with id: [${id}]. The process does not exists`));
|
|
248
|
-
return false;
|
|
249
|
-
}
|
|
250
|
-
} else {
|
|
251
|
-
this.LogInfoMessage(chalk.red(`MasterProcessBase:#_KillWorker(): Could not kill worker with id: [${id}]. Worker does not exist within workers collection`));
|
|
252
|
-
return false;
|
|
253
|
-
}
|
|
254
|
-
} catch (error) {
|
|
255
|
-
this.LogInfoMessage(chalk.red(`MasterProcessBase:#_KillWorker() (2): id: [${id}], signal: [${signal}], killProcess: [${killProcess}], error: [${error}]`));
|
|
256
|
-
return false;
|
|
257
|
-
}
|
|
258
|
-
}
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
KillWorker = (id: string, signal: NodeJS.Signals = "SIGTERM", options: any, killProcess: boolean, allowKillAll: boolean): boolean => {
|
|
262
|
-
try {
|
|
263
|
-
if (allowKillAll || Object.keys(cluster.workers as NodeJS.Dict<Worker>).length > 1) {
|
|
264
|
-
if (id.localeCompare('0') === 0) {
|
|
265
|
-
const keys = Object.keys(cluster.workers as NodeJS.Dict<Worker>);
|
|
266
|
-
for (let i=keys.length-1; i > 0; i--) {
|
|
267
|
-
// Kill the last one added (assumed node keeps them in order)
|
|
268
|
-
id = keys[i];
|
|
269
|
-
if (!this.#killWorkers[id]) {
|
|
270
|
-
this.#killWorkers[id] = id;
|
|
271
|
-
// Allow some time for the worker to be terminated before clean-up of the killWorkers array
|
|
272
|
-
setTimeout(() => {
|
|
273
|
-
delete this.#killWorkers[id];
|
|
274
|
-
}, 2000).unref(); //@@
|
|
275
|
-
return this.#_KillWorker(id, signal, killProcess);
|
|
276
|
-
}
|
|
277
|
-
}
|
|
278
|
-
return false;
|
|
279
|
-
} else {
|
|
280
|
-
return this.#_KillWorker(id, signal, killProcess);
|
|
281
|
-
}
|
|
282
|
-
} else {
|
|
283
|
-
this.LogInfoMessage(chalk.yellow(`MasterProcessBase:KillWorker(): Not allowed to kill the last worker process.`));
|
|
284
|
-
return false;
|
|
285
|
-
}
|
|
286
|
-
} catch (error) {
|
|
287
|
-
this.LogInfoMessage(chalk.red(`MasterProcessBase:KillWorker(): id: [${id}], signal: [${signal}], killProcess: [${killProcess}], error: [${error}]`));
|
|
288
|
-
return false;
|
|
289
|
-
}
|
|
290
|
-
}
|
|
291
|
-
|
|
292
|
-
KillWorkers = (signal: NodeJS.Signals, keepOne?: boolean): void => {
|
|
293
|
-
const logPrefix = `MasterProcessBase:KillWorkers():${process.pid}:`;
|
|
294
|
-
try {
|
|
295
|
-
this.LogInfoMessage(`${logPrefix} Killing Workers.`);
|
|
296
|
-
const keepOneAlive = (keepOne ? keepOne : false);
|
|
297
|
-
let skippedFirst = false;
|
|
298
|
-
const sortedIdList: number[] = [ ];
|
|
299
|
-
for (const id in cluster.workers) {
|
|
300
|
-
sortedIdList.push(parseInt(id));
|
|
301
|
-
}
|
|
302
|
-
sortedIdList.sort().forEach((id) => {
|
|
303
|
-
if (keepOneAlive && !skippedFirst) {
|
|
304
|
-
skippedFirst = true;
|
|
305
|
-
} else {
|
|
306
|
-
this.KillWorker(id.toString(), signal, null, false, true);
|
|
307
|
-
}
|
|
308
|
-
});
|
|
309
|
-
} catch (error) {
|
|
310
|
-
this.LogInfoMessage(chalk.red(`${logPrefix} signal: [${signal}], keepOne: [${keepOne}], error: [${error}]`));
|
|
311
|
-
}
|
|
312
|
-
}
|
|
313
|
-
|
|
314
|
-
#UpdateWorkersInstrument = (): void => {
|
|
315
|
-
setTimeout(() => {
|
|
316
|
-
this.UpdateInstrument(Gauge.CORE_COUNT_GAUGE, {
|
|
317
|
-
val: this.#workers
|
|
318
|
-
} as InstrumentGaugeTelemetry);
|
|
319
|
-
}, 2000);
|
|
320
|
-
}
|
|
321
|
-
|
|
322
|
-
IncWorkers = (): void => {
|
|
323
|
-
this.#workers++;
|
|
324
|
-
this.#LogDebugMessage(` Inc Workers. Total thread count: [${this.#workers}]`);
|
|
325
|
-
this.#UpdateWorkersInstrument();
|
|
326
|
-
}
|
|
327
|
-
|
|
328
|
-
DecWorkers = (): void => {
|
|
329
|
-
this.#workers--;
|
|
330
|
-
this.#LogDebugMessage(` Dec Workers. Total thread count: [${this.#workers}]`);
|
|
331
|
-
this.#UpdateWorkersInstrument();
|
|
332
|
-
}
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
AddWorker = (options: any): number => {
|
|
336
|
-
const workerId: number = this.#SpawnWorker(options);
|
|
337
|
-
this.#LogDebugMessage(chalk.yellow(` Spawned worker with id: [${workerId}]`));
|
|
338
|
-
if (options) {
|
|
339
|
-
this.#LogDebugMessage(chalk.yellow(` Options: [${JSON.stringify(options)}]`));
|
|
340
|
-
}
|
|
341
|
-
return workerId;
|
|
342
|
-
}
|
|
343
|
-
|
|
344
|
-
async ProcessIPCCommand(iPCMessagePayload: IPCMessagePayload): Promise<IPCMessagePayload> {
|
|
345
|
-
return this.#processIPCCommand(iPCMessagePayload);
|
|
346
|
-
}
|
|
347
|
-
|
|
348
|
-
#processIPCCommand = async (iPCMessagePayload: IPCMessagePayload): Promise<IPCMessagePayload> => {
|
|
349
|
-
this.#LogDebugMessage(chalk.yellow(` Processing message command: [${iPCMessagePayload.command}]`));
|
|
350
|
-
switch (iPCMessagePayload.command) {
|
|
351
|
-
case IPCMessageCommand.AddWorker : {
|
|
352
|
-
const workerId = this.AddWorker(iPCMessagePayload.requestDetail?.options);
|
|
353
|
-
iPCMessagePayload.responseDetail = {
|
|
354
|
-
workerId
|
|
355
|
-
}
|
|
356
|
-
return iPCMessagePayload;
|
|
357
|
-
}
|
|
358
|
-
case IPCMessageCommand.DeleteWorker : {
|
|
359
|
-
const workerId = iPCMessagePayload.requestDetail?.workerId;
|
|
360
|
-
const workerKilled = this.KillWorker(workerId, "SIGTERM", iPCMessagePayload.requestDetail?.options, true, false);
|
|
361
|
-
this.#LogDebugMessage(chalk.yellow(` Killed worker with id: [${workerId}]`));
|
|
362
|
-
iPCMessagePayload.responseDetail = {
|
|
363
|
-
workerId,
|
|
364
|
-
workerKilled
|
|
365
|
-
}
|
|
366
|
-
return iPCMessagePayload;
|
|
367
|
-
}
|
|
368
|
-
case IPCMessageCommand.GetConfig : {
|
|
369
|
-
const safeOptions = { ...this.options };
|
|
370
|
-
delete (safeOptions as any).logger
|
|
371
|
-
delete (safeOptions as any).publisherLogger
|
|
372
|
-
delete (safeOptions as any).expressServerRouteFactory
|
|
373
|
-
//this.#LogDebugMessage(chalk.yellow(` Safe options: [${JSON.stringify(safeOptions)}]`));
|
|
374
|
-
iPCMessagePayload.responseDetail = {
|
|
375
|
-
safeOptions
|
|
376
|
-
}
|
|
377
|
-
return iPCMessagePayload;
|
|
378
|
-
}
|
|
379
|
-
default : {
|
|
380
|
-
const errorMessage = `Could not process command: [${iPCMessagePayload.command}].`;
|
|
381
|
-
this.#LogDebugMessage(chalk.red(` ${errorMessage}`));
|
|
382
|
-
throw new Error(errorMessage);
|
|
383
|
-
}
|
|
384
|
-
}
|
|
385
|
-
}
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
#SpawnWorker = (spawnWorkerOptions?: any): number => {
|
|
389
|
-
const workerEnv: any = { };
|
|
390
|
-
|
|
391
|
-
const tempOptions = { ...this.options };
|
|
392
|
-
|
|
393
|
-
// Remove the assigned loggers (this will need to be re-created in the worker thread)
|
|
394
|
-
delete (tempOptions as any).logger;
|
|
395
|
-
delete (tempOptions as any).publisherLogger;
|
|
396
|
-
|
|
397
|
-
workerEnv['STS_GSD_SII'] = JSON.stringify(tempOptions);
|
|
398
|
-
if (spawnWorkerOptions) {
|
|
399
|
-
workerEnv['STS_GSD_OPTIONS'] = JSON.stringify(spawnWorkerOptions);
|
|
400
|
-
}
|
|
401
|
-
|
|
402
|
-
// https://nodejs.org/api/cluster.html#clustersetupprimarysettings
|
|
403
|
-
if (this.options.workerExec) {
|
|
404
|
-
cluster.setupPrimary({
|
|
405
|
-
exec: this.options.workerExec,
|
|
406
|
-
silent: true,
|
|
407
|
-
});
|
|
408
|
-
}
|
|
409
|
-
|
|
410
|
-
const worker = cluster.fork(workerEnv);
|
|
411
|
-
this.IncWorkers();
|
|
412
|
-
|
|
413
|
-
worker.on('exit', (code, signal) => {
|
|
414
|
-
this.emit('workerExit', code, signal);
|
|
415
|
-
if (signal) {
|
|
416
|
-
this.LogInfoMessage(`Worker: ${worker.process.pid} was killed by signal: ${signal}`);
|
|
417
|
-
} else if (code !== 0) {
|
|
418
|
-
this.LogInfoMessage(`Worker: ${worker.process.pid} exited with error code: ${code}`);
|
|
419
|
-
} else {
|
|
420
|
-
this.LogInfoMessage(`Worker: ${worker.process.pid} exited successfully, code: ${code}, signal: ${signal}`);
|
|
421
|
-
}
|
|
422
|
-
});
|
|
423
|
-
|
|
424
|
-
worker.on('message', async (payload) => {
|
|
425
|
-
this.emit('workerMessage', worker.id, payload);
|
|
426
|
-
// Only handle request/response message types here ...
|
|
427
|
-
if (payload.requestResponse) {
|
|
428
|
-
const iPCMessagePayload: IPCMessagePayload = payload as IPCMessagePayload;
|
|
429
|
-
//this.#LogDebugMessage(chalk.yellow(`Received message with id: [${iPCMessagePayload.id}] from worker: [${worker.process.pid}]. Details: [${JSON.stringify(iPCMessagePayload)}]`));
|
|
430
|
-
const response: IPCMessagePayload = await this.ProcessIPCCommand(iPCMessagePayload);
|
|
431
|
-
//this.#LogDebugMessage(chalk.green(`Sending response message with id: [${iPCMessagePayload.id}] to worker: [${worker.process.pid}]. Details: [${JSON.stringify(response)}]`));
|
|
432
|
-
worker.send(response);
|
|
433
|
-
} else {
|
|
434
|
-
this.#WorkerMessageEvent(worker.id, payload);
|
|
435
|
-
}
|
|
436
|
-
});
|
|
437
|
-
|
|
438
|
-
worker.on('error', (error) => {
|
|
439
|
-
this.emit('workerError', error);
|
|
440
|
-
const message = chalk.red(`#SpawnWorker():worker.on('error'): Error: [${error}]`);
|
|
441
|
-
this.LogErrorMessage(message);
|
|
442
|
-
});
|
|
443
|
-
|
|
444
|
-
this.emit('workerAdded', worker.id);
|
|
445
|
-
|
|
446
|
-
return worker.id;
|
|
447
|
-
};
|
|
448
|
-
|
|
449
|
-
MasterStarted(): void { // eslint-disable @typescript-eslint/no-empty-function
|
|
450
|
-
const transport = new STSTransportLoggerWinston({
|
|
451
|
-
stsApp: this
|
|
452
|
-
});
|
|
453
|
-
setTimeout(() => {
|
|
454
|
-
(this.options.logger as any).add(transport);
|
|
455
|
-
}, 0);
|
|
456
|
-
}
|
|
457
|
-
|
|
458
|
-
override get shuttingDown(): boolean {
|
|
459
|
-
return this.#shuttingDown;
|
|
460
|
-
}
|
|
461
|
-
|
|
462
|
-
SetupServerEx = async () =>
|
|
463
|
-
{
|
|
464
|
-
this.ProcessStartup();
|
|
465
|
-
|
|
466
|
-
this.LogInfoMessage(`Service instance starting. Instance Id: [${this.options.serviceInstanceId}]`);
|
|
467
|
-
|
|
468
|
-
this.LogSystemTelemetry();
|
|
469
|
-
|
|
470
|
-
// socket.io
|
|
471
|
-
// setup connections between the workers
|
|
472
|
-
if (this.options.wssServer === true) {
|
|
473
|
-
if (this.options.useSocketIoRedisAdaptor) {
|
|
474
|
-
this.LogInfoMessage(`Using Redis for socket.io cluster management (master)`);
|
|
475
|
-
} else {
|
|
476
|
-
this.LogInfoMessage(`Using nodejs cluster mode for socket.io cluster management`);
|
|
477
|
-
setupPrimary();
|
|
478
|
-
}
|
|
479
|
-
}
|
|
480
|
-
|
|
481
|
-
if (this.options.prometheusSupport === true) {
|
|
482
|
-
this.#SetupPrometheusForMaster();
|
|
483
|
-
}
|
|
484
|
-
|
|
485
|
-
const numCPUs = await this.GetNumCPUs();
|
|
486
|
-
for (let i=0; i < numCPUs; i++) {
|
|
487
|
-
this.#SpawnWorker();
|
|
488
|
-
}
|
|
489
|
-
|
|
490
|
-
this.#InitCluster();
|
|
491
|
-
|
|
492
|
-
cluster.on('listening', (worker, address) => {
|
|
493
|
-
this.emit('clusterListening', worker, address);
|
|
494
|
-
this.LogInfoMessage(`Worker process ${worker.process.pid} is listening at address: ${JSON.stringify(address)}`);
|
|
495
|
-
});
|
|
496
|
-
|
|
497
|
-
//Setting up lifecycle event listeners for worker processes
|
|
498
|
-
cluster.on('online', worker => {
|
|
499
|
-
this.emit('clusterOnline', worker);
|
|
500
|
-
this.LogInfoMessage(`Worker process ${worker.process.pid} is online`);
|
|
501
|
-
});
|
|
502
|
-
|
|
503
|
-
cluster.on('exit', (worker, code, signal) => {
|
|
504
|
-
this.emit('clusterExit', worker, code, signal);
|
|
505
|
-
if ((code !== null && code === 0) || (signal === 'SIGINT')) {
|
|
506
|
-
this.LogInfoMessage(chalk.green(`Process ${worker.process.pid} terminated gracefully with code: ${code}, signal: ${signal}`));
|
|
507
|
-
this.DecWorkers();
|
|
508
|
-
} else if ((code !== null && code === 15) || (signal === 'SIGTERM')) {
|
|
509
|
-
this.DecWorkers();
|
|
510
|
-
this.LogInfoMessage(chalk.red(`Process ${worker.process.pid} terminated with code: ${code}, signal: ${signal}`));
|
|
511
|
-
} else {
|
|
512
|
-
this.DecWorkers();
|
|
513
|
-
this.LogInfoMessage(chalk.red(`worker ${worker.process.pid} died`));
|
|
514
|
-
this.LogInfoMessage(chalk.red(`code: ${code}`));
|
|
515
|
-
this.LogInfoMessage(chalk.red(`signal: ${signal}`));
|
|
516
|
-
this.LogInfoMessage(chalk.red('process terminated in an error state'));
|
|
517
|
-
if (goptions.respawnOnFail === true) {
|
|
518
|
-
this.LogInfoMessage(chalk.magenta(`Attemping to respawn worker`));
|
|
519
|
-
this.#SpawnWorker();
|
|
520
|
-
}
|
|
521
|
-
}
|
|
522
|
-
});
|
|
523
|
-
|
|
524
|
-
const TerminateLatency = () => {
|
|
525
|
-
if (this.#checkLatency) {
|
|
526
|
-
clearInterval(this.#checkLatency);
|
|
527
|
-
this.#checkLatency = null;
|
|
528
|
-
}
|
|
529
|
-
}
|
|
530
|
-
|
|
531
|
-
const TerminateHTTPServer = async (): Promise<void> => {
|
|
532
|
-
if (this.#httpServer !== null) {
|
|
533
|
-
const logPrefix = `ServerProcessBase:TerminateHTTPServer():${process.pid}:`;
|
|
534
|
-
this.LogInfoMessage(`${logPrefix} Closing httpServer.`);
|
|
535
|
-
await this.#httpServer.close();
|
|
536
|
-
this.#httpServer = null;
|
|
537
|
-
}
|
|
538
|
-
}
|
|
539
|
-
|
|
540
|
-
// Terminate in order;
|
|
541
|
-
// forked worker threads (send signal)
|
|
542
|
-
// De-Register service
|
|
543
|
-
// systeminformation observers
|
|
544
|
-
// instrument timers (gauge etc.)
|
|
545
|
-
// publisher
|
|
546
|
-
// terminate UI (if loaded)
|
|
547
|
-
const Terminate = async (signal: any): Promise<void> => {
|
|
548
|
-
const logPrefix = `MasterProcessBase:Terminate():${process.pid}:`;
|
|
549
|
-
if (this.#shuttingDown === false) {
|
|
550
|
-
this.#shuttingDown = true;
|
|
551
|
-
|
|
552
|
-
if (signal) {
|
|
553
|
-
this.LogInfoMessage(this.GetSignalColour(signal)(`${logPrefix} Received signal: ${signal}`));
|
|
554
|
-
} else {
|
|
555
|
-
this.LogInfoMessage(this.GetSignalColour(null)(`${logPrefix} Received Terminate without signal.`));
|
|
556
|
-
}
|
|
557
|
-
|
|
558
|
-
TerminateLatency();
|
|
559
|
-
|
|
560
|
-
await this.ProcessTerminate();
|
|
561
|
-
|
|
562
|
-
this.TerminateUIController();
|
|
563
|
-
|
|
564
|
-
this.LogInfoMessage(`${logPrefix} De-Registering service.`);
|
|
565
|
-
//@@ De-register here ...
|
|
566
|
-
|
|
567
|
-
await TerminateHTTPServer();
|
|
568
|
-
|
|
569
|
-
this.KillWorkers(signal);
|
|
570
|
-
|
|
571
|
-
await this.TerminateDatabase();
|
|
572
|
-
|
|
573
|
-
this.TerminateInstrumentController();
|
|
574
|
-
|
|
575
|
-
await Sleep(1000);
|
|
576
|
-
|
|
577
|
-
this.ProcessExit(this.#childProcessExitTime + this.#masterProcessExitTime);
|
|
578
|
-
|
|
579
|
-
} else {
|
|
580
|
-
this.LogInfoMessage(`${logPrefix} Process already terminating.`);
|
|
581
|
-
}
|
|
582
|
-
}
|
|
583
|
-
|
|
584
|
-
process.on('SIGINT', async () => {
|
|
585
|
-
this.emit('processSigint');
|
|
586
|
-
await Terminate('SIGINT');
|
|
587
|
-
});
|
|
588
|
-
|
|
589
|
-
process.on('SIGTERM', async () => {
|
|
590
|
-
this.emit('processSigterm');
|
|
591
|
-
await Terminate('SIGTERM');
|
|
592
|
-
});
|
|
593
|
-
|
|
594
|
-
process.on('exit', (code) => {
|
|
595
|
-
this.emit('processExit', code);
|
|
596
|
-
if (code === 0) {
|
|
597
|
-
this.LogInfoMessage(chalk.green(`Main Process: ${process.pid} terminated gracefully with code: ${code}`));
|
|
598
|
-
} else {
|
|
599
|
-
this.LogInfoMessage(chalk.red(`Main Process: ${process.pid} terminated with code: ${code}`));
|
|
600
|
-
}
|
|
601
|
-
});
|
|
602
|
-
|
|
603
|
-
if (this.options.useLatency) {
|
|
604
|
-
this.#checkLatency = setInterval(() => {
|
|
605
|
-
this.#CheckLatency();
|
|
606
|
-
}, 1000).unref();
|
|
607
|
-
}
|
|
608
|
-
|
|
609
|
-
this.MasterStarted();
|
|
610
|
-
|
|
611
|
-
this.LogInfoMessage(chalk.green(`Master process:${process.pid} started`));
|
|
612
|
-
}
|
|
613
|
-
|
|
614
|
-
BroadcastDataToWorkers = (command: any, data: any) => {
|
|
615
|
-
try {
|
|
616
|
-
for (const id in cluster.workers) {
|
|
617
|
-
try {
|
|
618
|
-
//@@this.LogInfoMessage(chalk.gray(`Sending message to worker PID: ${cluster.workers[id].process.pid}`));
|
|
619
|
-
(cluster.workers[id] as Worker).process.send( { command: command, data: data } );
|
|
620
|
-
} catch (error) {
|
|
621
|
-
//@@this.LogInfoMessage(error);
|
|
622
|
-
}
|
|
623
|
-
}
|
|
624
|
-
} catch (error) {
|
|
625
|
-
//@@this.LogInfoMessage(error);
|
|
626
|
-
}
|
|
627
|
-
}
|
|
628
|
-
|
|
629
|
-
/*
|
|
630
|
-
#GetSystemInformation = async (): Promise<JSONObject> => {
|
|
631
|
-
// https://systeminformation.io/
|
|
632
|
-
const valueObject = {
|
|
633
|
-
system: '*',
|
|
634
|
-
osInfo: '*',
|
|
635
|
-
cpu: '*',
|
|
636
|
-
mem: '*',
|
|
637
|
-
dockerInfo: '*',
|
|
638
|
-
//dockerImages: '*',
|
|
639
|
-
dockerContainers: '*',
|
|
640
|
-
}
|
|
641
|
-
|
|
642
|
-
const sysinfo = await si.get(valueObject);
|
|
643
|
-
const numCPUs = await this.GetNumCPUs();
|
|
644
|
-
const hostname = sysinfo.osInfo.hostname;
|
|
645
|
-
|
|
646
|
-
const hostaddr = GetFirstNetworkInterface();
|
|
647
|
-
|
|
648
|
-
const promArray: Promise<any>[] = [ ];
|
|
649
|
-
|
|
650
|
-
sysinfo.dockerContainers.forEach((dc: { id: string; }) => {
|
|
651
|
-
const dcs = promArray.push(si.dockerContainerStats(dc.id));
|
|
652
|
-
console.log(dcs);
|
|
653
|
-
});
|
|
654
|
-
const dockerContainerStats = await Promise.all(promArray);
|
|
655
|
-
|
|
656
|
-
const sysInfo = {
|
|
657
|
-
serviceProcessContext: this.options.serviceProcessContext,
|
|
658
|
-
hostname,
|
|
659
|
-
numCPUs,
|
|
660
|
-
hostaddr,
|
|
661
|
-
system: sysinfo.system,
|
|
662
|
-
osInfo: sysinfo.osInfo,
|
|
663
|
-
cpu: sysinfo.cpu,
|
|
664
|
-
mem: sysinfo.mem,
|
|
665
|
-
dockerInfo: sysinfo.dockerInfo,
|
|
666
|
-
//dockerImages: sysinfo.dockerImages,
|
|
667
|
-
dockerContainers: sysinfo.dockerContainers,
|
|
668
|
-
dockerContainerStats
|
|
669
|
-
}
|
|
670
|
-
|
|
671
|
-
return sysInfo;
|
|
672
|
-
}
|
|
673
|
-
*/
|
|
674
|
-
}
|