@nsshunt/stsappframework 3.0.93 → 3.0.95
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/authutilsnode.js +6 -7
- package/dist/authutilsnode.js.map +1 -1
- package/dist/influxdb/influxDBManager.js +16 -17
- package/dist/influxdb/influxDBManager.js.map +1 -1
- package/dist/influxdb/influxDBManagerAgent.js +9 -13
- package/dist/influxdb/influxDBManagerAgent.js.map +1 -1
- package/dist/influxdb/influxDBManagerBase.js +2 -6
- package/dist/influxdb/influxDBManagerBase.js.map +1 -1
- package/dist/influxdb/influxDBManagerService.js +10 -14
- package/dist/influxdb/influxDBManagerService.js.map +1 -1
- package/dist/instrumentationsubscriber.js +11 -15
- package/dist/instrumentationsubscriber.js.map +1 -1
- package/dist/kafka/IMKafkaManager.js +2 -6
- package/dist/kafka/IMKafkaManager.js.map +1 -1
- package/dist/kafkatesting/produce.js +1 -5
- package/dist/kafkatesting/produce.js.map +1 -1
- package/dist/masterprocessbase.js +19 -20
- package/dist/masterprocessbase.js.map +1 -1
- package/dist/processbase.js +7 -8
- package/dist/processbase.js.map +1 -1
- package/dist/server.js +1 -1
- package/dist/server.js.map +1 -1
- package/dist/singleprocessbase.js +3 -4
- package/dist/singleprocessbase.js.map +1 -1
- package/dist/tcpclient/app2.js +2 -2
- package/dist/tcpserver/appmaster.js +39 -16
- package/dist/tcpserver/appmaster.js.map +1 -1
- package/dist/testing/appWorkerWSS.js +6 -7
- package/dist/testing/appWorkerWSS.js.map +1 -1
- package/dist/testing/singleservertest.test.js +8 -9
- package/dist/testing/singleservertest.test.js.map +1 -1
- package/dist/workerprocessbase.js +3 -4
- package/dist/workerprocessbase.js.map +1 -1
- package/package.json +1 -1
- package/src/authutilsnode.ts +8 -10
- package/src/influxdb/influxDBManager.ts +16 -18
- package/src/influxdb/influxDBManagerAgent.ts +9 -11
- package/src/influxdb/influxDBManagerBase.ts +2 -4
- package/src/influxdb/influxDBManagerService.ts +10 -12
- package/src/instrumentationsubscriber.ts +11 -14
- package/src/kafka/IMKafkaManager.ts +2 -4
- package/src/kafkatesting/produce.ts +1 -3
- package/src/masterprocessbase.ts +32 -24
- package/src/processbase.ts +7 -9
- package/src/server.ts +1 -1
- package/src/singleprocessbase.ts +3 -5
- package/src/tcpclient/app2.ts +2 -2
- package/src/tcpserver/appmaster.ts +39 -17
- package/src/testing/appWorkerWSS.ts +6 -8
- package/src/testing/singleservertest.test.ts +8 -10
- package/src/workerprocessbase.ts +3 -5
- package/src_new/authDefs.ts +37 -0
- package/src_new/authutilsnode.ts +375 -0
- package/src_new/commonTypes.ts +239 -0
- package/src_new/index.ts +22 -0
- package/src_new/influxdb/influxDBManager.ts +972 -0
- package/src_new/influxdb/influxDBManagerAgent.ts +316 -0
- package/src_new/influxdb/influxDBManagerBase.ts +111 -0
- package/src_new/influxdb/influxDBManagerService.ts +375 -0
- package/src_new/instrumentationsubscriber.ts +286 -0
- package/src_new/kafka/IMKafkaManager.ts +154 -0
- package/src_new/kafka/kafkaconsumer.ts +82 -0
- package/src_new/kafka/kafkamanager.ts +186 -0
- package/src_new/kafka/kafkaproducer.ts +58 -0
- package/src_new/kafkatesting/config.ts +10 -0
- package/src_new/kafkatesting/consume.ts +116 -0
- package/src_new/kafkatesting/produce.ts +155 -0
- package/src_new/masterprocessbase.ts +590 -0
- package/src_new/middleware/serverNetworkMiddleware.ts +240 -0
- package/src_new/network.ts +36 -0
- package/src_new/processbase.ts +413 -0
- package/src_new/processoptions.ts +164 -0
- package/src_new/publishertransports/publishTransportDirect.ts +45 -0
- package/src_new/publishertransports/publishTransportUtils.ts +53 -0
- package/src_new/server.ts +141 -0
- package/src_new/serverprocessbase.ts +393 -0
- package/src_new/singleprocessbase.ts +123 -0
- package/src_new/socketIoServerHelper.ts +177 -0
- package/src_new/stscontrollerbase.ts +15 -0
- package/src_new/stslatencycontroller.ts +27 -0
- package/src_new/stslatencyroute.ts +16 -0
- package/src_new/stsrouterbase.ts +22 -0
- package/src_new/tcpclient/app.ts +19 -0
- package/src_new/tcpclient/app2.ts +56 -0
- package/src_new/tcpserver/app.ts +11 -0
- package/src_new/tcpserver/appConfig.ts +65 -0
- package/src_new/tcpserver/appmaster.ts +522 -0
- package/src_new/validation/errors.ts +6 -0
- package/src_new/webworkertesting/app.ts +49 -0
- package/src_new/webworkertesting/worker.ts +24 -0
- package/src_new/workerprocessbase.test.ts +47 -0
- package/src_new/workerprocessbase.ts +187 -0
- package/types/authutilsnode.d.ts.map +1 -1
- package/types/influxdb/influxDBManager.d.ts.map +1 -1
- package/types/influxdb/influxDBManagerAgent.d.ts.map +1 -1
- package/types/influxdb/influxDBManagerBase.d.ts.map +1 -1
- package/types/influxdb/influxDBManagerService.d.ts.map +1 -1
- package/types/instrumentationsubscriber.d.ts.map +1 -1
- package/types/kafka/IMKafkaManager.d.ts.map +1 -1
- package/types/masterprocessbase.d.ts.map +1 -1
- package/types/processbase.d.ts +2 -2
- package/types/processbase.d.ts.map +1 -1
- package/types/singleprocessbase.d.ts.map +1 -1
- package/types/tcpserver/appmaster.d.ts.map +1 -1
- package/types/testing/appWorkerWSS.d.ts.map +1 -1
- package/types/workerprocessbase.d.ts.map +1 -1
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
/* eslint @typescript-eslint/no-explicit-any: 0, @typescript-eslint/no-unused-vars: 0 */ // --> OFF
|
|
2
|
+
import chalk from 'chalk'; // Note: Do NOT upgrade beyond 4.1.2 as the ESM version (5.0.0+) breaks this entire codebase
|
|
3
|
+
import { Request, Response, NextFunction } from 'express'
|
|
4
|
+
import { Socket } from 'node:net'
|
|
5
|
+
import { TinyEmitter } from 'tiny-emitter';
|
|
6
|
+
|
|
7
|
+
import debugModule from 'debug'
|
|
8
|
+
import { STSOptionsBase } from '@nsshunt/stsutils';
|
|
9
|
+
const debug = debugModule(`proc:${process.pid}`);
|
|
10
|
+
|
|
11
|
+
import { v4 as uuidv4 } from 'uuid';
|
|
12
|
+
import onHeaders from 'on-headers';
|
|
13
|
+
|
|
14
|
+
export enum ServerNetworkMiddlewareEventName {
|
|
15
|
+
UpdateInstrument_SERVER_NET_VAL = 'UpdateInstrument_SERVER_NET_VAL' // request net stats
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export interface ISocketRecord {
|
|
19
|
+
id: string
|
|
20
|
+
socket: Socket,
|
|
21
|
+
currentBytesRead: number
|
|
22
|
+
currentBytesWritten: number
|
|
23
|
+
lastBytesRead: number
|
|
24
|
+
lastBytesWritten: number
|
|
25
|
+
requestBytesRead: number
|
|
26
|
+
requestBytesWritten: number
|
|
27
|
+
originalUrl: string
|
|
28
|
+
eventName: string
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export type ServerNetworkMiddlewareEventFunc = (data: ISocketRecord) => void;
|
|
32
|
+
|
|
33
|
+
export interface IServerNetworkMiddleware {
|
|
34
|
+
name: string
|
|
35
|
+
outputDebug: boolean
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export class ServerNetworkMiddleware extends STSOptionsBase
|
|
39
|
+
{
|
|
40
|
+
#tinyEmitter: TinyEmitter = new TinyEmitter();
|
|
41
|
+
#socketCollection: Record<string, ISocketRecord> = { };
|
|
42
|
+
#id: string = '';
|
|
43
|
+
#debug: any;
|
|
44
|
+
|
|
45
|
+
constructor(options: IServerNetworkMiddleware) {
|
|
46
|
+
super(options);
|
|
47
|
+
if (options.outputDebug) {
|
|
48
|
+
this.#debug = debug;
|
|
49
|
+
} else {
|
|
50
|
+
this.#debug = () => { };
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
on(eventName: ServerNetworkMiddlewareEventName, callBackFn: ServerNetworkMiddlewareEventFunc) {
|
|
55
|
+
this.#tinyEmitter.on(eventName, callBackFn);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
GetSocketRecord = (socket: Socket): ISocketRecord | null => {
|
|
59
|
+
for (const [, socketRecord] of Object.entries(this.#socketCollection)) {
|
|
60
|
+
if (socketRecord.socket === socket) {
|
|
61
|
+
return socketRecord;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
return null;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
UpdateNetworkStats = (workingSocketRecord: ISocketRecord, eventName: string, req: Request) => {
|
|
68
|
+
workingSocketRecord.originalUrl = req.originalUrl;
|
|
69
|
+
workingSocketRecord.eventName = eventName;
|
|
70
|
+
|
|
71
|
+
workingSocketRecord.currentBytesRead = req.socket.bytesRead;
|
|
72
|
+
workingSocketRecord.currentBytesWritten = req.socket.bytesWritten;
|
|
73
|
+
|
|
74
|
+
workingSocketRecord.requestBytesRead = workingSocketRecord.currentBytesRead - workingSocketRecord.lastBytesRead;
|
|
75
|
+
workingSocketRecord.requestBytesWritten = workingSocketRecord.currentBytesWritten - workingSocketRecord.lastBytesWritten;
|
|
76
|
+
|
|
77
|
+
//this.#debug(chalk.gray(`totalBytesRead: [${workingSocketRecord.id}] [${workingSocketRecord.originalUrl}] [${eventName}] [${workingSocketRecord.requestBytesRead}]`));
|
|
78
|
+
//this.#debug(chalk.gray(`totalBytesWritten: [${workingSocketRecord.id}] [${workingSocketRecord.originalUrl}] [${eventName}] [${workingSocketRecord.requestBytesWritten}]`));
|
|
79
|
+
|
|
80
|
+
const workingSocketEventRecord = { ...workingSocketRecord };
|
|
81
|
+
delete (workingSocketEventRecord as any).socket;
|
|
82
|
+
this.#debug(chalk.gray(`Sending event: [${JSON.stringify(workingSocketEventRecord)}]`));
|
|
83
|
+
this.#tinyEmitter.emit(ServerNetworkMiddlewareEventName.UpdateInstrument_SERVER_NET_VAL, workingSocketEventRecord);
|
|
84
|
+
|
|
85
|
+
workingSocketRecord.lastBytesRead = workingSocketRecord.currentBytesRead;
|
|
86
|
+
workingSocketRecord.lastBytesWritten = workingSocketRecord.currentBytesWritten;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
#GetRequestContentSize = (req: Request) => {
|
|
90
|
+
try {
|
|
91
|
+
let contentLength = 0;
|
|
92
|
+
if ((req as any)['_contentLength']) {
|
|
93
|
+
contentLength = (req as any)['_contentLength'];
|
|
94
|
+
} else {
|
|
95
|
+
if (req.headers['content-length']) {
|
|
96
|
+
contentLength = parseInt(req.headers['content-length'])
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
return contentLength;
|
|
100
|
+
} catch (error) {
|
|
101
|
+
this.#debug(chalk.red(`ServerNetworkMiddleware:#GetRequestContentSize(): Error: [${error}]`));
|
|
102
|
+
return 0;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
Middleware = (req: Request, res: Response, next: NextFunction) => {
|
|
107
|
+
let workingSocketRecord: ISocketRecord | null = null;
|
|
108
|
+
|
|
109
|
+
workingSocketRecord = this.GetSocketRecord(req.socket);
|
|
110
|
+
|
|
111
|
+
if (!workingSocketRecord) {
|
|
112
|
+
this.#id = uuidv4();
|
|
113
|
+
workingSocketRecord = {
|
|
114
|
+
id: `${(this.options as IServerNetworkMiddleware).name}_${this.#id.toString()}`,
|
|
115
|
+
socket: req.socket,
|
|
116
|
+
currentBytesRead: 0,
|
|
117
|
+
currentBytesWritten: 0,
|
|
118
|
+
lastBytesRead: 0,
|
|
119
|
+
lastBytesWritten: 0,
|
|
120
|
+
requestBytesRead: 0,
|
|
121
|
+
requestBytesWritten: 0,
|
|
122
|
+
originalUrl: req.originalUrl,
|
|
123
|
+
eventName: '',
|
|
124
|
+
};
|
|
125
|
+
|
|
126
|
+
this.#socketCollection[workingSocketRecord.id] = workingSocketRecord;
|
|
127
|
+
|
|
128
|
+
this.#debug(chalk.gray(`Adding new socket to recordset: ID: [${workingSocketRecord.id}], originalUrl: [${workingSocketRecord.originalUrl}]`));
|
|
129
|
+
|
|
130
|
+
workingSocketRecord.socket.on('data', () => {
|
|
131
|
+
const socketRecord = this.GetSocketRecord((workingSocketRecord as ISocketRecord).socket);
|
|
132
|
+
if (socketRecord) {
|
|
133
|
+
this.#debug(chalk.gray(`Socket data event: ID: [${socketRecord.id}], originalUrl: [${socketRecord.originalUrl}]`));
|
|
134
|
+
this.UpdateNetworkStats(socketRecord, 'socket_data', req);
|
|
135
|
+
} else {
|
|
136
|
+
this.#debug(chalk.magenta(`Socket data event: Could not find socket within recordset`));
|
|
137
|
+
}
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
workingSocketRecord.socket.on('close', () => {
|
|
141
|
+
const socketRecord = this.GetSocketRecord((workingSocketRecord as ISocketRecord).socket);
|
|
142
|
+
if (socketRecord) {
|
|
143
|
+
this.#debug(chalk.gray(`Socket close event: ID: [${socketRecord.id}], originalUrl: [${socketRecord.originalUrl}]`));
|
|
144
|
+
this.UpdateNetworkStats(socketRecord, 'socket_close', req);
|
|
145
|
+
delete this.#socketCollection[socketRecord.id];
|
|
146
|
+
this.#debug(chalk.gray(`Socket removed from recordset: ID: [${socketRecord.id}], originalUrl: [${socketRecord.originalUrl}]`));
|
|
147
|
+
} else {
|
|
148
|
+
this.#debug(chalk.magenta(`Socket close event: Could not find socket within recordset`));
|
|
149
|
+
}
|
|
150
|
+
});
|
|
151
|
+
|
|
152
|
+
workingSocketRecord.socket.on('end', () => {
|
|
153
|
+
const socketRecord = this.GetSocketRecord((workingSocketRecord as ISocketRecord).socket);
|
|
154
|
+
if (socketRecord) {
|
|
155
|
+
this.#debug(chalk.gray(`Socket end event: ID: [${socketRecord.id}], originalUrl: [${socketRecord.originalUrl}]`));
|
|
156
|
+
this.UpdateNetworkStats(socketRecord, 'socket_end', req);
|
|
157
|
+
} else {
|
|
158
|
+
this.#debug(chalk.magenta(`Socket end event: Could not find socket within recordset`));
|
|
159
|
+
}
|
|
160
|
+
});
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
// The following gets the payload size from adding _contentLength or content-length to the header size.
|
|
164
|
+
// From brief observations, this appears to be about 80 bytes short from what is reported by the socket writtenBytes.
|
|
165
|
+
onHeaders(res, () => {
|
|
166
|
+
let contentLength = 0;
|
|
167
|
+
if ((res as any)['_contentLength']) {
|
|
168
|
+
contentLength = (res as any)['_contentLength'];
|
|
169
|
+
} else {
|
|
170
|
+
if (res.hasHeader('content-length')) {
|
|
171
|
+
contentLength = parseInt(res.getHeader('content-length') as string);
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
// Ensure that the client does NOT cache the response
|
|
176
|
+
res.setHeader('Cache-Control', 'no-cache, no-store, must-revalidate'); // HTTP 1.1
|
|
177
|
+
res.setHeader('Pragma', 'no-cache'); // HTTP 1.0
|
|
178
|
+
res.setHeader('Expires', '0'); // Proxies
|
|
179
|
+
|
|
180
|
+
const headerLength = JSON.stringify(res.getHeaders()).length;
|
|
181
|
+
const totalSize = contentLength + headerLength;
|
|
182
|
+
|
|
183
|
+
this.#debug(chalk.white.bgBlue(`contentLength: [${contentLength}], headerLength: [${headerLength}], total Size: [${totalSize}]`));
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
// This event is the one where bytesWritten is generally recorded
|
|
187
|
+
req.on('end', () => {
|
|
188
|
+
const socketRecord = this.GetSocketRecord(req.socket);
|
|
189
|
+
if (socketRecord) {
|
|
190
|
+
this.#debug(chalk.gray(`Request end event: ID: [${socketRecord.id}], originalUrl: [${socketRecord.originalUrl}]`))
|
|
191
|
+
this.UpdateNetworkStats(socketRecord, 'req_end', req);
|
|
192
|
+
} else {
|
|
193
|
+
this.#debug(chalk.magenta(`Request end event: Could not find socket within recordset`));
|
|
194
|
+
}
|
|
195
|
+
});
|
|
196
|
+
|
|
197
|
+
req.on('close', () => {
|
|
198
|
+
const socketRecord = this.GetSocketRecord(req.socket);
|
|
199
|
+
if (socketRecord) {
|
|
200
|
+
this.#debug(chalk.gray(`Request close event: ID: [${socketRecord.id}], originalUrl: [${socketRecord.originalUrl}]`));
|
|
201
|
+
this.UpdateNetworkStats(socketRecord, 'req_close', req);
|
|
202
|
+
} else {
|
|
203
|
+
this.#debug(chalk.magenta(`Request close event: Could not find socket within recordset`));
|
|
204
|
+
}
|
|
205
|
+
});
|
|
206
|
+
|
|
207
|
+
/* These methods are also available but from testing they provide no specific value add, i.e. the calculated numbers are always 0.
|
|
208
|
+
req.on('data', () => {
|
|
209
|
+
workingSocketRecord = this.GetSocketRecord(req.socket);
|
|
210
|
+
if (workingSocketRecord) {
|
|
211
|
+
this.#debug(chalk.gray(`Request close event: ID: [${workingSocketRecord.id}], originalUrl: [${workingSocketRecord.originalUrl}]`));
|
|
212
|
+
this.UpdateNetworkStats(workingSocketRecord, 'data', req);
|
|
213
|
+
} else {
|
|
214
|
+
this.#debug(chalk.magenta(`Request close event: Could not find socket within recordset`));
|
|
215
|
+
}
|
|
216
|
+
});
|
|
217
|
+
|
|
218
|
+
req.on('readable', () => {
|
|
219
|
+
workingSocketRecord = this.GetSocketRecord(req.socket);
|
|
220
|
+
if (workingSocketRecord) {
|
|
221
|
+
this.#debug(chalk.gray(`Request close event: ID: [${workingSocketRecord.id}], originalUrl: [${workingSocketRecord.originalUrl}]`));
|
|
222
|
+
this.UpdateNetworkStats(workingSocketRecord, 'readable', req);
|
|
223
|
+
} else {
|
|
224
|
+
this.#debug(chalk.magenta(`Request close event: Could not find socket within recordset`));
|
|
225
|
+
}
|
|
226
|
+
});
|
|
227
|
+
*/
|
|
228
|
+
|
|
229
|
+
const inHeadersLength = JSON.stringify(req.headers).length;
|
|
230
|
+
const inContentLength = this.#GetRequestContentSize(req);
|
|
231
|
+
const inTotal = inHeadersLength + inContentLength;
|
|
232
|
+
|
|
233
|
+
this.#debug(chalk.white.bgGray(`inHeadersLength: [${inHeadersLength}], inContentLength: [${inContentLength}], inTotal: [${inTotal}]`));
|
|
234
|
+
|
|
235
|
+
// This event is the one where bytesRead is generally recorded
|
|
236
|
+
this.UpdateNetworkStats(workingSocketRecord, 'middleware', req);
|
|
237
|
+
|
|
238
|
+
next();
|
|
239
|
+
}
|
|
240
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import os from 'os'
|
|
2
|
+
|
|
3
|
+
export type NetworkInterfaces = Record<string, string[]>
|
|
4
|
+
|
|
5
|
+
export function GetNetworkInterfaces(): NetworkInterfaces {
|
|
6
|
+
const nets = os.networkInterfaces();
|
|
7
|
+
const results: NetworkInterfaces = { };
|
|
8
|
+
for (const name of Object.keys(nets)) {
|
|
9
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
10
|
+
for (const net of nets[name] as any) {
|
|
11
|
+
// Skip over non-IPv4 and internal (i.e. 127.0.0.1) addresses
|
|
12
|
+
if (net.family === 'IPv4' && !net.internal) {
|
|
13
|
+
if (!results[name]) {
|
|
14
|
+
results[name] = [];
|
|
15
|
+
}
|
|
16
|
+
results[name].push(net.address);
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
return results;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export function GetFirstNetworkInterface(): string | null {
|
|
24
|
+
const nics: NetworkInterfaces = GetNetworkInterfaces();
|
|
25
|
+
let hostaddr = null;
|
|
26
|
+
for (const nic in nics)
|
|
27
|
+
{
|
|
28
|
+
const val: string[] = nics[nic];
|
|
29
|
+
if (val.length > 0)
|
|
30
|
+
{
|
|
31
|
+
hostaddr = val[0];
|
|
32
|
+
break;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
return hostaddr;
|
|
36
|
+
}
|
|
@@ -0,0 +1,413 @@
|
|
|
1
|
+
/* eslint @typescript-eslint/no-explicit-any: 0, @typescript-eslint/no-unused-vars: 0 */ // --> OFF
|
|
2
|
+
import pidusage from 'pidusage'
|
|
3
|
+
import { memoryUsage, exit } from 'process'
|
|
4
|
+
import path from 'node:path';
|
|
5
|
+
import cluster from 'node:cluster'
|
|
6
|
+
import colors from 'colors'
|
|
7
|
+
import si from 'systeminformation' // https://systeminformation.io/
|
|
8
|
+
import { GetFirstNetworkInterface } from './network'
|
|
9
|
+
|
|
10
|
+
import chalk from 'chalk';
|
|
11
|
+
|
|
12
|
+
import debugModule from 'debug'
|
|
13
|
+
const debug = debugModule(`proc:${process.pid}:processBase`);
|
|
14
|
+
|
|
15
|
+
import { Gauge, InstrumentBaseTelemetry, InstrumentGaugeTelemetry, InstrumentObjectTelemetry, InstrumentLogTelemetry } from '@nsshunt/stsinstrumentation'
|
|
16
|
+
|
|
17
|
+
import { PGAccessLayer, PGPoolManager, PGPoolManagerEventName } from '@nsshunt/stsdatamanagement'
|
|
18
|
+
|
|
19
|
+
import { $Options } from '@nsshunt/stsconfig'
|
|
20
|
+
const goptions = $Options()
|
|
21
|
+
|
|
22
|
+
import { ProcessOptions, STSServerType } from './processoptions'
|
|
23
|
+
import { STSOptionsBase, JSONObject } from '@nsshunt/stsutils'
|
|
24
|
+
import { StatusCodes } from 'http-status-codes'
|
|
25
|
+
|
|
26
|
+
import { PublishInstrumentControllerV2, IPublishInstrumentControllerOptionsV2, InstrumentDefinitions,
|
|
27
|
+
IWorkerFactory, TransportType, IPublishTransportRESTServerOptions,
|
|
28
|
+
CreateServiceProcessContext, PublishTransportRESTServerV2, IPublisherTransport } from '@nsshunt/stspublisherserver'
|
|
29
|
+
|
|
30
|
+
//import { PublishCollectorInProcessWorkerFactory } from './processcollectorinprocessworker'
|
|
31
|
+
|
|
32
|
+
import { Worker } from 'worker_threads';
|
|
33
|
+
|
|
34
|
+
import os from 'os';
|
|
35
|
+
|
|
36
|
+
import { IProcessBase, ISocketIoServerHelper, logFunction } from './commonTypes'
|
|
37
|
+
import { TimeSeriesDuplicatePolicies } from 'redis';
|
|
38
|
+
|
|
39
|
+
import { STSDefaultClientToServerEvents, STSDefaultServerToClientEvents } from '@nsshunt/stssocketio-client'
|
|
40
|
+
|
|
41
|
+
//import ProcessBrokerWorker from './processbrokerworker.js?worker'
|
|
42
|
+
|
|
43
|
+
import { PublishTransportDirect } from './publishertransports/publishTransportDirect'
|
|
44
|
+
|
|
45
|
+
export abstract class ProcessBase extends STSOptionsBase implements IProcessBase
|
|
46
|
+
{
|
|
47
|
+
#instrumentController: PublishInstrumentControllerV2 | null = null;
|
|
48
|
+
#systemInformationInterval: NodeJS.Timer | null = null;
|
|
49
|
+
#socketIoServerHelper: ISocketIoServerHelper<STSDefaultClientToServerEvents, STSDefaultServerToClientEvents> | null = null;
|
|
50
|
+
#accessLayer: PGAccessLayer | null = null;
|
|
51
|
+
#poolManager: PGPoolManager | null = null;
|
|
52
|
+
#logEx: logFunction | null = null;
|
|
53
|
+
|
|
54
|
+
constructor(options: ProcessOptions)
|
|
55
|
+
{
|
|
56
|
+
super(options);
|
|
57
|
+
|
|
58
|
+
if (options.serviceName) {
|
|
59
|
+
this.options.serviceProcessContext = CreateServiceProcessContext(options.serviceName, options.serviceVersion, options.serviceInstanceId,
|
|
60
|
+
os.hostname(), process.pid, options.isMaster ? process.pid : process.ppid)
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
override get options(): ProcessOptions {
|
|
65
|
+
return super.options as ProcessOptions;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
get shuttingDown(): boolean {
|
|
69
|
+
return false;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
#log(appName: any): logFunction {
|
|
73
|
+
return (msg) =>
|
|
74
|
+
{
|
|
75
|
+
let prefix = '';
|
|
76
|
+
let col = null;
|
|
77
|
+
|
|
78
|
+
if (cluster.isPrimary)
|
|
79
|
+
{
|
|
80
|
+
prefix = 'M';
|
|
81
|
+
col = colors.bold.cyan; // chalk.bold.cyan;
|
|
82
|
+
} else {
|
|
83
|
+
prefix = 'W';
|
|
84
|
+
col = colors.green; // chalk.green;
|
|
85
|
+
}
|
|
86
|
+
const msgEx = col(`${prefix}(${process.pid}) [${appName}]: ${msg}`);
|
|
87
|
+
debug(msgEx);
|
|
88
|
+
|
|
89
|
+
this.UpdateInstrument(Gauge.LOGGER, {
|
|
90
|
+
LogMessage: msgEx
|
|
91
|
+
} as InstrumentLogTelemetry);
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
#GetRootFolder = (): string => {
|
|
96
|
+
const mainpath = require.resolve('@nsshunt/stsutils');
|
|
97
|
+
const anotherDir = path.dirname(mainpath);
|
|
98
|
+
const sSplit = anotherDir.split('/')
|
|
99
|
+
const result = sSplit.slice(0, -5) // Remove the last 4 path elements, this will be 'node_modules/@nsshunt/stsutils/dist'
|
|
100
|
+
return result.join('/');
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
#ServiceCollectorWorkerFactory: IWorkerFactory = {
|
|
104
|
+
createWorker: (): Worker => {
|
|
105
|
+
const fileName = this.#GetRootFolder() + '/stsappframework/dist/processcollectorworker.js';
|
|
106
|
+
|
|
107
|
+
//@@ check for copy env like
|
|
108
|
+
// const envCopy = { ...process.env };
|
|
109
|
+
const publishCollectorWebWorker = new Worker(fileName, {
|
|
110
|
+
workerData: 500 // ??
|
|
111
|
+
// ,env: envCopy
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
publishCollectorWebWorker.unref();
|
|
115
|
+
|
|
116
|
+
return publishCollectorWebWorker;
|
|
117
|
+
},
|
|
118
|
+
/*
|
|
119
|
+
createInProcessWorker: (): PublishCollectorWorker => {
|
|
120
|
+
return PublishCollectorInProcessWorkerFactory();
|
|
121
|
+
},
|
|
122
|
+
*/
|
|
123
|
+
options: {
|
|
124
|
+
transportType: TransportType.RESTAPI,
|
|
125
|
+
url: `${goptions.imendpoint}:${goptions.import}${goptions.imapiroot}/publishmessage`,
|
|
126
|
+
publishdebug: goptions.publishdebug,
|
|
127
|
+
// socketPath: '/var/run/sts/stsrest01.sock'
|
|
128
|
+
agentOptions: {
|
|
129
|
+
keepAlive: goptions.keepAlive,
|
|
130
|
+
maxSockets: goptions.maxSockets,
|
|
131
|
+
maxTotalSockets: goptions.maxTotalSockets,
|
|
132
|
+
maxFreeSockets: goptions.maxFreeSockets,
|
|
133
|
+
timeout: goptions.timeout,
|
|
134
|
+
rejectUnauthorized: goptions.isProduction // Allows self signed certs in non production mode(s)
|
|
135
|
+
}
|
|
136
|
+
} as IPublishTransportRESTServerOptions
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
#ServiceBrokerWorkerFactory: IWorkerFactory = {
|
|
140
|
+
createWorker: (): Worker => {
|
|
141
|
+
const fileName = this.#GetRootFolder() + '/stsappframework/dist/processbrokerworker.js';
|
|
142
|
+
|
|
143
|
+
//@@ check for copy env like
|
|
144
|
+
// const envCopy = { ...process.env };
|
|
145
|
+
const publishBrokerWebWorker = new Worker(fileName, {
|
|
146
|
+
//workerData: this.options.serviceProcessContext
|
|
147
|
+
// ,env: envCopy
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
publishBrokerWebWorker.unref();
|
|
151
|
+
|
|
152
|
+
return publishBrokerWebWorker;
|
|
153
|
+
}
|
|
154
|
+
/*
|
|
155
|
+
createInProcessWorker: (): PublishBrokerWorker => {
|
|
156
|
+
return PublishBrokerInProcessWorkerFactory();
|
|
157
|
+
},
|
|
158
|
+
*/
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
SetupInstrumentation()
|
|
162
|
+
{
|
|
163
|
+
if (!this.options.serviceProcessContext) {
|
|
164
|
+
return;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
let publisherTransport: IPublisherTransport
|
|
168
|
+
|
|
169
|
+
if (goptions.observabilityPublishMode.localeCompare('PROXY') === 0) {
|
|
170
|
+
const publishTransportOptions: IPublishTransportRESTServerOptions = {
|
|
171
|
+
transportType: TransportType.RESTAPI,
|
|
172
|
+
url: `${goptions.imendpoint}:${goptions.import}${goptions.imapiroot}/publishmessage`,
|
|
173
|
+
publishdebug: goptions.publishdebug,
|
|
174
|
+
// socketPath: '/var/run/sts/stsrest01.sock'
|
|
175
|
+
agentOptions: {
|
|
176
|
+
keepAlive: goptions.keepAlive,
|
|
177
|
+
maxSockets: goptions.maxSockets,
|
|
178
|
+
maxTotalSockets: goptions.maxTotalSockets,
|
|
179
|
+
maxFreeSockets: goptions.maxFreeSockets,
|
|
180
|
+
timeout: goptions.timeout,
|
|
181
|
+
rejectUnauthorized: goptions.isProduction // Allows self signed certs in non production mode(s)
|
|
182
|
+
}
|
|
183
|
+
};
|
|
184
|
+
publisherTransport = new PublishTransportRESTServerV2(publishTransportOptions);
|
|
185
|
+
} else {
|
|
186
|
+
publisherTransport = new PublishTransportDirect();
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
const instrumentControllerOptions: IPublishInstrumentControllerOptionsV2 = {
|
|
190
|
+
processContext: this.options.serviceProcessContext,
|
|
191
|
+
//payloadType: InstrumentPayloadType.service,
|
|
192
|
+
consoleLogging: this.options.consoleLogging,
|
|
193
|
+
instrumentLogging: this.options.instrumentLogging,
|
|
194
|
+
httpServer: (this.options.serverType === STSServerType.EXPRESS || this.options.serverType === STSServerType.EXPRESS_TLS),
|
|
195
|
+
instrumentationObservationInterval: this.options.instrumentationObservationInterval,
|
|
196
|
+
instrumentationTimeWindow: this.options.instrumentationTimeWindow,
|
|
197
|
+
instrumentDefinitions: this.GetAdditionalInstruments(),
|
|
198
|
+
publisherTransport
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
this.#instrumentController = new PublishInstrumentControllerV2(instrumentControllerOptions);
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
GetAdditionalInstruments(): InstrumentDefinitions {
|
|
205
|
+
return [ ];
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
CollectAdditionalTelemetry(): void { // eslint-disable @typescript-eslint/no-empty-function
|
|
209
|
+
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
ProcessStartup = () =>
|
|
213
|
+
{
|
|
214
|
+
// use this as well
|
|
215
|
+
// performance.eventLoopUtilization([utilization1[, utilization2]])
|
|
216
|
+
// https://nodejs.org/api/perf_hooks.html#perf_hooksmonitoreventloopdelayoptions
|
|
217
|
+
this.#systemInformationInterval = setInterval(() => {
|
|
218
|
+
pidusage(process.pid, (err, stats) => {
|
|
219
|
+
|
|
220
|
+
if (this.#instrumentController) {
|
|
221
|
+
this.#instrumentController.UpdateInstrument(Gauge.CPU_LOAD_GAUGE, {
|
|
222
|
+
val: stats.cpu
|
|
223
|
+
} as InstrumentGaugeTelemetry);
|
|
224
|
+
}
|
|
225
|
+
});
|
|
226
|
+
|
|
227
|
+
// https://nodejs.org/api/process.html#processmemoryusage
|
|
228
|
+
const usage = memoryUsage();
|
|
229
|
+
const telemetry: JSONObject = {
|
|
230
|
+
r: usage.rss,
|
|
231
|
+
t: usage.heapTotal,
|
|
232
|
+
u: usage.heapUsed,
|
|
233
|
+
x: usage.external,
|
|
234
|
+
a: usage.arrayBuffers,
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
this.UpdateInstrument(Gauge.OBJECT_GAUGE, {
|
|
238
|
+
val: telemetry
|
|
239
|
+
} as InstrumentObjectTelemetry);
|
|
240
|
+
|
|
241
|
+
this.CollectAdditionalTelemetry();
|
|
242
|
+
|
|
243
|
+
}, goptions.systemInformationInterval).unref();
|
|
244
|
+
|
|
245
|
+
this.#logEx = this.#log(this.options.serviceName);
|
|
246
|
+
|
|
247
|
+
if (this.options.useDatabase) {
|
|
248
|
+
// Get the accessLayer to force DB connection test
|
|
249
|
+
this.accessLayer;
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
UpdateInstrument = (instrumentName: Gauge, telemetry: InstrumentBaseTelemetry) => {
|
|
254
|
+
if (this.#instrumentController) {
|
|
255
|
+
this.#instrumentController.UpdateInstrument(instrumentName, telemetry);
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
get InstrumentController(): PublishInstrumentControllerV2 | null {
|
|
260
|
+
return this.#instrumentController;
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
LogEx = (message: any) => {
|
|
264
|
+
if (this.#logEx) {
|
|
265
|
+
this.#logEx(message);
|
|
266
|
+
} else {
|
|
267
|
+
console.log('')
|
|
268
|
+
console.log('************* Attempt to log before _logex set ***************')
|
|
269
|
+
console.log(message)
|
|
270
|
+
console.log('')
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
InstrumentExists(instrumentName: Gauge): boolean
|
|
275
|
+
{
|
|
276
|
+
if (this.#instrumentController) {
|
|
277
|
+
return this.#instrumentController.InstrumentExists(instrumentName);
|
|
278
|
+
}
|
|
279
|
+
return false;
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
ProcessTerminate = async () =>
|
|
283
|
+
{
|
|
284
|
+
if (this.#systemInformationInterval) {
|
|
285
|
+
clearInterval(this.#systemInformationInterval);
|
|
286
|
+
}
|
|
287
|
+
this.#systemInformationInterval = null;
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
/**
|
|
291
|
+
* UIController (instance of UIController) to manage a console based user interface associated for this node application.
|
|
292
|
+
* @returns UIController instance to manage a console based user interface associated for this node application.
|
|
293
|
+
*/
|
|
294
|
+
GetUIController(): any {
|
|
295
|
+
return null;
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
get socketIoServerHelper(): ISocketIoServerHelper<STSDefaultClientToServerEvents, STSDefaultServerToClientEvents> | null
|
|
299
|
+
{
|
|
300
|
+
return this.#socketIoServerHelper;
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
set socketIoServerHelper(value: ISocketIoServerHelper<STSDefaultClientToServerEvents, STSDefaultServerToClientEvents> | null)
|
|
304
|
+
{
|
|
305
|
+
this.#socketIoServerHelper = value;
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
#UpdatePGPoolManagerInstrument = (data: any) => {
|
|
309
|
+
this.UpdateInstrument(Gauge.CONNECTION_POOL_TOTAL_GAUGE, {
|
|
310
|
+
val: data.totalCount
|
|
311
|
+
} as InstrumentGaugeTelemetry);
|
|
312
|
+
|
|
313
|
+
this.UpdateInstrument(Gauge.CONNECTION_POOL_IDLE_GAUGE, {
|
|
314
|
+
val: data.idleCount
|
|
315
|
+
} as InstrumentGaugeTelemetry);
|
|
316
|
+
|
|
317
|
+
this.UpdateInstrument(Gauge.CONNECTION_POOL_WAITING_GAUGE, {
|
|
318
|
+
val: data.waitingCount
|
|
319
|
+
} as InstrumentGaugeTelemetry);
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
async TerminateDatabase() {
|
|
323
|
+
if (this.#accessLayer && this.#poolManager) {
|
|
324
|
+
this.#poolManager.off(PGPoolManagerEventName.UpdateInstruments, this.#UpdatePGPoolManagerInstrument)
|
|
325
|
+
await this.#accessLayer.enddatabase();
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
get accessLayer(): PGAccessLayer | null
|
|
330
|
+
{
|
|
331
|
+
if (this.options.useDatabase) {
|
|
332
|
+
if (this.#accessLayer === null) {
|
|
333
|
+
this.#poolManager = new PGPoolManager();
|
|
334
|
+
this.#accessLayer = new PGAccessLayer(this.#poolManager);
|
|
335
|
+
(async () => {
|
|
336
|
+
try {
|
|
337
|
+
const retVal = await (this.accessLayer as PGAccessLayer).getResourceCount();
|
|
338
|
+
if (retVal.status !== StatusCodes.OK) {
|
|
339
|
+
this.LogEx(chalk.red(`Unable to get resources from the database. Is the database running? [${JSON.stringify(retVal)}]`));
|
|
340
|
+
exit(1);
|
|
341
|
+
} else {
|
|
342
|
+
this.LogEx(chalk.green(`Database connection successful. Resources: [${retVal.detail.count}]`));
|
|
343
|
+
}
|
|
344
|
+
} catch (error) {
|
|
345
|
+
this.LogEx(chalk.red(`Unable to get resources from the database. Is the database running? Error: [${error}]`));
|
|
346
|
+
exit(1);
|
|
347
|
+
}
|
|
348
|
+
})();
|
|
349
|
+
this.#poolManager.on(PGPoolManagerEventName.UpdateInstruments, this.#UpdatePGPoolManagerInstrument)
|
|
350
|
+
}
|
|
351
|
+
return this.#accessLayer;
|
|
352
|
+
}
|
|
353
|
+
return null;
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
GetNumCPUs = async (): Promise<number> => {
|
|
357
|
+
// https://systeminformation.io/
|
|
358
|
+
const valueObject = {
|
|
359
|
+
cpu: '*'
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
const sysinfo = await si.get(valueObject);
|
|
363
|
+
let numCPUs = 2;
|
|
364
|
+
if (goptions.useCPUs > 0) {
|
|
365
|
+
if (goptions.useCPUs >= 1) {
|
|
366
|
+
numCPUs = goptions.useCPUs;
|
|
367
|
+
} else {
|
|
368
|
+
numCPUs = Math.round(sysinfo.cpu.cores * goptions.useCPUs);
|
|
369
|
+
}
|
|
370
|
+
} else {
|
|
371
|
+
numCPUs = sysinfo.cpu.physicalCores;
|
|
372
|
+
}
|
|
373
|
+
return numCPUs;
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
LogSystemTelemetry = async () => {
|
|
377
|
+
// https://systeminformation.io/
|
|
378
|
+
const valueObject = {
|
|
379
|
+
system: '*',
|
|
380
|
+
osInfo: '*',
|
|
381
|
+
cpu: '*',
|
|
382
|
+
mem: '*'
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
const sysinfo = await si.get(valueObject);
|
|
386
|
+
const numCPUs = await this.GetNumCPUs();
|
|
387
|
+
const hostname = sysinfo.osInfo.hostname;
|
|
388
|
+
|
|
389
|
+
const hostaddr = GetFirstNetworkInterface();
|
|
390
|
+
if (hostaddr !== null) {
|
|
391
|
+
this.LogEx(`Host Address: ${hostaddr}`);
|
|
392
|
+
} else {
|
|
393
|
+
this.LogEx(`Unknown Host Address.`);
|
|
394
|
+
}
|
|
395
|
+
this.LogEx(`Server starting with ${numCPUs} Cores/Threads`);
|
|
396
|
+
|
|
397
|
+
this.LogEx(`Hostname: ${hostname}`);
|
|
398
|
+
this.LogEx(`System: ${JSON.stringify(sysinfo.system)}`);
|
|
399
|
+
this.LogEx(`OS Info: ${JSON.stringify(sysinfo.osInfo)}`);
|
|
400
|
+
this.LogEx(`CPU: ${JSON.stringify(sysinfo.cpu)}`);
|
|
401
|
+
this.LogEx(`Memory: ${JSON.stringify(sysinfo.mem)}`);
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
GetSignalColour = (signal: any) => {
|
|
405
|
+
let msgcolor = null;
|
|
406
|
+
if (signal === 'SIGINT') {
|
|
407
|
+
msgcolor = chalk.yellow;
|
|
408
|
+
} else {
|
|
409
|
+
msgcolor = chalk.red;
|
|
410
|
+
}
|
|
411
|
+
return msgcolor;
|
|
412
|
+
};
|
|
413
|
+
}
|