@nsshunt/stsappframework 3.0.93 → 3.0.95
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/authutilsnode.js +6 -7
- package/dist/authutilsnode.js.map +1 -1
- package/dist/influxdb/influxDBManager.js +16 -17
- package/dist/influxdb/influxDBManager.js.map +1 -1
- package/dist/influxdb/influxDBManagerAgent.js +9 -13
- package/dist/influxdb/influxDBManagerAgent.js.map +1 -1
- package/dist/influxdb/influxDBManagerBase.js +2 -6
- package/dist/influxdb/influxDBManagerBase.js.map +1 -1
- package/dist/influxdb/influxDBManagerService.js +10 -14
- package/dist/influxdb/influxDBManagerService.js.map +1 -1
- package/dist/instrumentationsubscriber.js +11 -15
- package/dist/instrumentationsubscriber.js.map +1 -1
- package/dist/kafka/IMKafkaManager.js +2 -6
- package/dist/kafka/IMKafkaManager.js.map +1 -1
- package/dist/kafkatesting/produce.js +1 -5
- package/dist/kafkatesting/produce.js.map +1 -1
- package/dist/masterprocessbase.js +19 -20
- package/dist/masterprocessbase.js.map +1 -1
- package/dist/processbase.js +7 -8
- package/dist/processbase.js.map +1 -1
- package/dist/server.js +1 -1
- package/dist/server.js.map +1 -1
- package/dist/singleprocessbase.js +3 -4
- package/dist/singleprocessbase.js.map +1 -1
- package/dist/tcpclient/app2.js +2 -2
- package/dist/tcpserver/appmaster.js +39 -16
- package/dist/tcpserver/appmaster.js.map +1 -1
- package/dist/testing/appWorkerWSS.js +6 -7
- package/dist/testing/appWorkerWSS.js.map +1 -1
- package/dist/testing/singleservertest.test.js +8 -9
- package/dist/testing/singleservertest.test.js.map +1 -1
- package/dist/workerprocessbase.js +3 -4
- package/dist/workerprocessbase.js.map +1 -1
- package/package.json +1 -1
- package/src/authutilsnode.ts +8 -10
- package/src/influxdb/influxDBManager.ts +16 -18
- package/src/influxdb/influxDBManagerAgent.ts +9 -11
- package/src/influxdb/influxDBManagerBase.ts +2 -4
- package/src/influxdb/influxDBManagerService.ts +10 -12
- package/src/instrumentationsubscriber.ts +11 -14
- package/src/kafka/IMKafkaManager.ts +2 -4
- package/src/kafkatesting/produce.ts +1 -3
- package/src/masterprocessbase.ts +32 -24
- package/src/processbase.ts +7 -9
- package/src/server.ts +1 -1
- package/src/singleprocessbase.ts +3 -5
- package/src/tcpclient/app2.ts +2 -2
- package/src/tcpserver/appmaster.ts +39 -17
- package/src/testing/appWorkerWSS.ts +6 -8
- package/src/testing/singleservertest.test.ts +8 -10
- package/src/workerprocessbase.ts +3 -5
- package/src_new/authDefs.ts +37 -0
- package/src_new/authutilsnode.ts +375 -0
- package/src_new/commonTypes.ts +239 -0
- package/src_new/index.ts +22 -0
- package/src_new/influxdb/influxDBManager.ts +972 -0
- package/src_new/influxdb/influxDBManagerAgent.ts +316 -0
- package/src_new/influxdb/influxDBManagerBase.ts +111 -0
- package/src_new/influxdb/influxDBManagerService.ts +375 -0
- package/src_new/instrumentationsubscriber.ts +286 -0
- package/src_new/kafka/IMKafkaManager.ts +154 -0
- package/src_new/kafka/kafkaconsumer.ts +82 -0
- package/src_new/kafka/kafkamanager.ts +186 -0
- package/src_new/kafka/kafkaproducer.ts +58 -0
- package/src_new/kafkatesting/config.ts +10 -0
- package/src_new/kafkatesting/consume.ts +116 -0
- package/src_new/kafkatesting/produce.ts +155 -0
- package/src_new/masterprocessbase.ts +590 -0
- package/src_new/middleware/serverNetworkMiddleware.ts +240 -0
- package/src_new/network.ts +36 -0
- package/src_new/processbase.ts +413 -0
- package/src_new/processoptions.ts +164 -0
- package/src_new/publishertransports/publishTransportDirect.ts +45 -0
- package/src_new/publishertransports/publishTransportUtils.ts +53 -0
- package/src_new/server.ts +141 -0
- package/src_new/serverprocessbase.ts +393 -0
- package/src_new/singleprocessbase.ts +123 -0
- package/src_new/socketIoServerHelper.ts +177 -0
- package/src_new/stscontrollerbase.ts +15 -0
- package/src_new/stslatencycontroller.ts +27 -0
- package/src_new/stslatencyroute.ts +16 -0
- package/src_new/stsrouterbase.ts +22 -0
- package/src_new/tcpclient/app.ts +19 -0
- package/src_new/tcpclient/app2.ts +56 -0
- package/src_new/tcpserver/app.ts +11 -0
- package/src_new/tcpserver/appConfig.ts +65 -0
- package/src_new/tcpserver/appmaster.ts +522 -0
- package/src_new/validation/errors.ts +6 -0
- package/src_new/webworkertesting/app.ts +49 -0
- package/src_new/webworkertesting/worker.ts +24 -0
- package/src_new/workerprocessbase.test.ts +47 -0
- package/src_new/workerprocessbase.ts +187 -0
- package/types/authutilsnode.d.ts.map +1 -1
- package/types/influxdb/influxDBManager.d.ts.map +1 -1
- package/types/influxdb/influxDBManagerAgent.d.ts.map +1 -1
- package/types/influxdb/influxDBManagerBase.d.ts.map +1 -1
- package/types/influxdb/influxDBManagerService.d.ts.map +1 -1
- package/types/instrumentationsubscriber.d.ts.map +1 -1
- package/types/kafka/IMKafkaManager.d.ts.map +1 -1
- package/types/masterprocessbase.d.ts.map +1 -1
- package/types/processbase.d.ts +2 -2
- package/types/processbase.d.ts.map +1 -1
- package/types/singleprocessbase.d.ts.map +1 -1
- package/types/tcpserver/appmaster.d.ts.map +1 -1
- package/types/testing/appWorkerWSS.d.ts.map +1 -1
- package/types/workerprocessbase.d.ts.map +1 -1
|
@@ -4,8 +4,6 @@ import { InfluxDB, WriteApi, QueryApi, DEFAULT_WriteOptions, WriteOptions } from
|
|
|
4
4
|
//import { Agent } from 'https'
|
|
5
5
|
import { Agent } from 'http'
|
|
6
6
|
|
|
7
|
-
import chalk from 'chalk';
|
|
8
|
-
|
|
9
7
|
import { $Options } from '@nsshunt/stsconfig'
|
|
10
8
|
import { JSONObject } from '@nsshunt/stsutils'
|
|
11
9
|
const goptions = $Options()
|
|
@@ -276,15 +274,15 @@ export class InfluxDBManager
|
|
|
276
274
|
org: 'my-org'
|
|
277
275
|
});
|
|
278
276
|
if (tasks.tasks) {
|
|
279
|
-
console.log(
|
|
277
|
+
console.log(`[InfluxDB Tasks] - Count: [${tasks.tasks.length}]`.cyan);
|
|
280
278
|
tasks.tasks.forEach((task: Task) => {
|
|
281
|
-
console.log(
|
|
282
|
-
console.log(
|
|
283
|
-
console.log(
|
|
284
|
-
console.log(
|
|
279
|
+
console.log(`-- [ Start Task Definition ]--`.cyan);
|
|
280
|
+
console.log(`${JSON.stringify(task)}`.green);
|
|
281
|
+
console.log(`authorizationID: [${task.authorizationID}]`.yellow);
|
|
282
|
+
console.log(`-- [ End Task Definition ]--`.cyan);
|
|
285
283
|
});
|
|
286
284
|
} else {
|
|
287
|
-
console.log(
|
|
285
|
+
console.log(`[InfluxDB Tasks] - Could not get tasks from the InfluxDB API`.magenta);
|
|
288
286
|
}
|
|
289
287
|
}
|
|
290
288
|
|
|
@@ -334,9 +332,9 @@ export class InfluxDBManager
|
|
|
334
332
|
|
|
335
333
|
this.#writeClient = null;
|
|
336
334
|
|
|
337
|
-
console.log(
|
|
335
|
+
console.log(`ProducerDisconnect`.yellow);
|
|
338
336
|
} catch (error) {
|
|
339
|
-
console.error(
|
|
337
|
+
console.error(`${_logPrefix}#Terminate: Error: [${error}]`.red);
|
|
340
338
|
}
|
|
341
339
|
}
|
|
342
340
|
}
|
|
@@ -356,7 +354,7 @@ export class InfluxDBManager
|
|
|
356
354
|
});
|
|
357
355
|
return results;
|
|
358
356
|
} catch (error) {
|
|
359
|
-
console.error(
|
|
357
|
+
console.error(`${_logPrefix}#CreateGlobalCountModel: Error: [${error}]`.red);
|
|
360
358
|
}
|
|
361
359
|
}
|
|
362
360
|
|
|
@@ -430,13 +428,13 @@ export class InfluxDBManager
|
|
|
430
428
|
// https://influxdata.github.io/influxdb-client-js/influxdb-client.writeoptions.html and
|
|
431
429
|
// https://influxdata.github.io/influxdb-client-js/influxdb-client.writeretryoptions.html
|
|
432
430
|
writeFailed(error, lines, attempt, expires) {
|
|
433
|
-
debug(
|
|
431
|
+
debug(`writeOptions:writeSuccess(): Error: [${error}], Attempt: [${attempt}], Expires: [${expires}], Lines: [${lines}]`.red);
|
|
434
432
|
},
|
|
435
433
|
writeSuccess(lines) {
|
|
436
|
-
debug(
|
|
434
|
+
debug(`writeOptions:writeSuccess(): Lines: [${lines}]`.green);
|
|
437
435
|
},
|
|
438
436
|
writeRetrySkipped(entry) {
|
|
439
|
-
debug(
|
|
437
|
+
debug(`writeOptions:writeSuccess(): Expires: [${entry.expires}], Lines: [${entry.lines}]`.magenta);
|
|
440
438
|
}
|
|
441
439
|
}
|
|
442
440
|
|
|
@@ -459,10 +457,10 @@ export class InfluxDBManager
|
|
|
459
457
|
attemptCount = 0;
|
|
460
458
|
StartWriteClient(normalFlushDelay);
|
|
461
459
|
} catch (error) {
|
|
462
|
-
console.error(
|
|
460
|
+
console.error(`InfluxDBManager:StartWriteClient(): Attempt: [${attemptCount}], Error: [${error}]`.red)
|
|
463
461
|
if (attemptCount === maxAttemptCount) {
|
|
464
|
-
console.error(
|
|
465
|
-
console.error(
|
|
462
|
+
console.error(`InfluxDBManager:StartWriteClient(): Max Attempts reached. Will no longer retry.`.red);
|
|
463
|
+
console.error(`InfluxDBManager:StartWriteClient(): InfluxDB data capture ended in permanent faiilure.`.red);
|
|
466
464
|
} else {
|
|
467
465
|
StartWriteClient(retryFlushDelay);
|
|
468
466
|
}
|
|
@@ -489,7 +487,7 @@ export class InfluxDBManager
|
|
|
489
487
|
return this.#influxDBManagerClients['service'].OutputInfluxDB(this.writeClient, instrumentPayload);
|
|
490
488
|
}
|
|
491
489
|
} catch (error) {
|
|
492
|
-
console.error(
|
|
490
|
+
console.error(`${_logPrefix}OutputInfluxDB: Could not write data point: [${error}]`.red);
|
|
493
491
|
return false;
|
|
494
492
|
}
|
|
495
493
|
}
|
|
@@ -105,8 +105,6 @@ import { InfluxDBManagerBase } from './influxDBManagerBase'
|
|
|
105
105
|
import { IInfluxDBManagerOptions, InstrumentPayload } from './../commonTypes'
|
|
106
106
|
import { ISubscriptionPayload, ISubscriptionKey } from '@nsshunt/stssocketio-client'
|
|
107
107
|
|
|
108
|
-
import chalk from 'chalk';
|
|
109
|
-
|
|
110
108
|
const AGENT_STATS_BY_AGENT_THREAD_ASYNCRUNNER = "sts-stats-by-agentthreadasyncunner";
|
|
111
109
|
const AGENT_STATS_BY_AGENT_THREAD = "sts-stats-by-agentthread";
|
|
112
110
|
const AGENT_STATS_BY_AGENT = "sts-stats-by-agent";
|
|
@@ -140,7 +138,7 @@ export class InfluxDBManagerAgent extends InfluxDBManagerBase
|
|
|
140
138
|
|
|
141
139
|
return this.queryApi.collectRows(query)
|
|
142
140
|
} catch (error) {
|
|
143
|
-
console.error(
|
|
141
|
+
console.error(`${_logPrefix}#GetSTSCountGeneric: Error: [${error}]`.red);
|
|
144
142
|
}
|
|
145
143
|
}
|
|
146
144
|
|
|
@@ -156,7 +154,7 @@ export class InfluxDBManagerAgent extends InfluxDBManagerBase
|
|
|
156
154
|
|> difference()`;
|
|
157
155
|
return this.queryApi.collectRows(query)
|
|
158
156
|
} catch (error) {
|
|
159
|
-
console.error(
|
|
157
|
+
console.error(`${_logPrefix}#GetSTSHistoGeneric: Error: [${error}]`.red);
|
|
160
158
|
}
|
|
161
159
|
}
|
|
162
160
|
|
|
@@ -185,7 +183,7 @@ export class InfluxDBManagerAgent extends InfluxDBManagerBase
|
|
|
185
183
|
|
|
186
184
|
return this.queryApi.collectRows(query)
|
|
187
185
|
} catch (error) {
|
|
188
|
-
console.error(
|
|
186
|
+
console.error(`${_logPrefix}#GetSTSQuantileGeneric: Error: [${error}]`.red);
|
|
189
187
|
}
|
|
190
188
|
}
|
|
191
189
|
|
|
@@ -200,7 +198,7 @@ export class InfluxDBManagerAgent extends InfluxDBManagerBase
|
|
|
200
198
|
this.#GetSTSHistoGenericService(AGENT_STATS_GLOBALAGENT, '')],
|
|
201
199
|
[ ])
|
|
202
200
|
} catch (error) {
|
|
203
|
-
console.error(
|
|
201
|
+
console.error(`${_logPrefix}GetInfluxDBResultsRootAgent: Error: [${error}]`.red);
|
|
204
202
|
}
|
|
205
203
|
return {
|
|
206
204
|
subscriptionKey,
|
|
@@ -218,7 +216,7 @@ export class InfluxDBManagerAgent extends InfluxDBManagerBase
|
|
|
218
216
|
this.#GetSTSHistoGenericService(AGENT_STATS_BY_AGENT, '')],
|
|
219
217
|
['agentName'])
|
|
220
218
|
} catch (error) {
|
|
221
|
-
console.error(
|
|
219
|
+
console.error(`${_logPrefix}GetInfluxDBResultsAgent: Error: [${error}]`.red);
|
|
222
220
|
}
|
|
223
221
|
return {
|
|
224
222
|
subscriptionKey,
|
|
@@ -237,7 +235,7 @@ export class InfluxDBManagerAgent extends InfluxDBManagerBase
|
|
|
237
235
|
this.#GetSTSHistoGenericService(AGENT_STATS_BY_AGENT_THREAD, `and r["agentName"] == "${agentName}"`)],
|
|
238
236
|
['agentName', 'threadId'])
|
|
239
237
|
} catch (error) {
|
|
240
|
-
console.error(
|
|
238
|
+
console.error(`${_logPrefix}GetInfluxDBResultsAgentThreads: Error: [${error}]`.red);
|
|
241
239
|
}
|
|
242
240
|
return {
|
|
243
241
|
subscriptionKey,
|
|
@@ -257,7 +255,7 @@ export class InfluxDBManagerAgent extends InfluxDBManagerBase
|
|
|
257
255
|
this.#GetSTSHistoGenericService(AGENT_STATS_BY_AGENT_THREAD_ASYNCRUNNER, `and r["agentName"] == "${agentName}" and r["threadId"] == "${threadId}"`)],
|
|
258
256
|
['agentName', 'threadId', 'asyncRunnerId'])
|
|
259
257
|
} catch (error) {
|
|
260
|
-
console.error(
|
|
258
|
+
console.error(`${_logPrefix}GetInfluxDBResultsAgentThread: Error: [${error}]`.red);
|
|
261
259
|
}
|
|
262
260
|
return {
|
|
263
261
|
subscriptionKey,
|
|
@@ -305,11 +303,11 @@ export class InfluxDBManagerAgent extends InfluxDBManagerBase
|
|
|
305
303
|
writeApi.writePoint(point);
|
|
306
304
|
return true;
|
|
307
305
|
} catch (error: any) {
|
|
308
|
-
console.error(
|
|
306
|
+
console.error(`${_logPrefix}OutputInfluxDBAgent: Could not write data point: [${error}]`.red);
|
|
309
307
|
return false;
|
|
310
308
|
}
|
|
311
309
|
} else {
|
|
312
|
-
console.error(
|
|
310
|
+
console.error(`${_logPrefix}OutputInfluxDBAgent: Could not write data point as writeClient is null`.red);
|
|
313
311
|
return false;
|
|
314
312
|
}
|
|
315
313
|
}
|
|
@@ -7,8 +7,6 @@ import { IInfluxDBManagerOptions, InstrumentPayload } from './../commonTypes'
|
|
|
7
7
|
|
|
8
8
|
import { JSONObject } from '@nsshunt/stsutils'
|
|
9
9
|
|
|
10
|
-
import chalk from 'chalk';
|
|
11
|
-
|
|
12
10
|
const _logPrefix = 'InfluxDBManager:'
|
|
13
11
|
|
|
14
12
|
export class InfluxDBManagerBase
|
|
@@ -76,7 +74,7 @@ export class InfluxDBManagerBase
|
|
|
76
74
|
}
|
|
77
75
|
return combinedResults;
|
|
78
76
|
} catch (error) {
|
|
79
|
-
console.error(
|
|
77
|
+
console.error(`${_logPrefix}ProcessInfluxDBResults: Error: [${error}]`.red);
|
|
80
78
|
}
|
|
81
79
|
}
|
|
82
80
|
|
|
@@ -88,7 +86,7 @@ export class InfluxDBManagerBase
|
|
|
88
86
|
});
|
|
89
87
|
return results;
|
|
90
88
|
} catch (error) {
|
|
91
|
-
console.error(
|
|
89
|
+
console.error(`${_logPrefix}#CreateGlobalCountModel: Error: [${error}]`.red);
|
|
92
90
|
}
|
|
93
91
|
}
|
|
94
92
|
|
|
@@ -117,8 +117,6 @@ import { InfluxDBManagerBase } from './influxDBManagerBase'
|
|
|
117
117
|
import { IInfluxDBManagerOptions, InstrumentPayload } from './../commonTypes'
|
|
118
118
|
import { ISubscriptionPayload, ISubscriptionKey } from '@nsshunt/stssocketio-client'
|
|
119
119
|
|
|
120
|
-
import chalk from 'chalk';
|
|
121
|
-
|
|
122
120
|
const _logPrefix = 'InfluxDBManagerService:'
|
|
123
121
|
|
|
124
122
|
/*
|
|
@@ -181,7 +179,7 @@ data
|
|
|
181
179
|
|
|
182
180
|
return this.queryApi.collectRows(query)
|
|
183
181
|
} catch (error) {
|
|
184
|
-
console.error(
|
|
182
|
+
console.error(`${_logPrefix}#GetSTSCountGeneric: Error: [${error}]`.red);
|
|
185
183
|
}
|
|
186
184
|
}
|
|
187
185
|
|
|
@@ -197,7 +195,7 @@ data
|
|
|
197
195
|
|> difference()`;
|
|
198
196
|
return this.queryApi.collectRows(query)
|
|
199
197
|
} catch (error) {
|
|
200
|
-
console.error(
|
|
198
|
+
console.error(`${_logPrefix}#GetSTSHistoGeneric: Error: [${error}]`.red);
|
|
201
199
|
}
|
|
202
200
|
}
|
|
203
201
|
|
|
@@ -228,7 +226,7 @@ data
|
|
|
228
226
|
|
|
229
227
|
return this.queryApi.collectRows(query)
|
|
230
228
|
} catch (error) {
|
|
231
|
-
console.error(
|
|
229
|
+
console.error(`${_logPrefix}#GetSTSQuantileGeneric: Error: [${error}]`.red);
|
|
232
230
|
}
|
|
233
231
|
}
|
|
234
232
|
|
|
@@ -243,7 +241,7 @@ data
|
|
|
243
241
|
this.#GetSTSHistoGenericService(SERVICE_STATS_GLOBAL, '')],
|
|
244
242
|
[ ])
|
|
245
243
|
} catch (error) {
|
|
246
|
-
console.error(
|
|
244
|
+
console.error(`${_logPrefix}GetInfluxDBResultsRoot: Error: [${error}]`.red);
|
|
247
245
|
}
|
|
248
246
|
return {
|
|
249
247
|
subscriptionKey,
|
|
@@ -261,7 +259,7 @@ data
|
|
|
261
259
|
this.#GetSTSHistoGenericService(SERVICE_STATS_BY_SERVICE, '')],
|
|
262
260
|
['serviceId'])
|
|
263
261
|
} catch (error) {
|
|
264
|
-
console.error(
|
|
262
|
+
console.error(`${_logPrefix}GetInfluxDBResultsService: Error: [${error}]`.red);
|
|
265
263
|
}
|
|
266
264
|
return {
|
|
267
265
|
subscriptionKey,
|
|
@@ -281,7 +279,7 @@ data
|
|
|
281
279
|
this.#GetSTSHistoGenericService(SERVICE_STATS_BY_SERVICE_INSTANCE, `and r["serviceId"] == "${serviceId}"`)],
|
|
282
280
|
['serviceId', 'serviceInstanceId'])
|
|
283
281
|
} catch (error) {
|
|
284
|
-
console.error(
|
|
282
|
+
console.error(`${_logPrefix}GetInfluxDBResultsServiceInstances: Error: [${error}]`.red);
|
|
285
283
|
}
|
|
286
284
|
return {
|
|
287
285
|
subscriptionKey,
|
|
@@ -300,7 +298,7 @@ data
|
|
|
300
298
|
this.#GetSTSHistoGenericService(SERVICE_STATS_BY_SERVICE_INSTANCE_PROCESS, `and r["serviceInstanceId"] == "${serviceInstanceId}"`)],
|
|
301
299
|
['serviceId', 'serviceInstanceId', 'serviceInstanceProcessId'])
|
|
302
300
|
} catch (error) {
|
|
303
|
-
console.error(
|
|
301
|
+
console.error(`${_logPrefix}GetInfluxDBResultsServiceInstance: Error: [${error}]`.red);
|
|
304
302
|
}
|
|
305
303
|
return {
|
|
306
304
|
subscriptionKey,
|
|
@@ -363,12 +361,12 @@ data
|
|
|
363
361
|
writeApi.writePoint(point);
|
|
364
362
|
return true;
|
|
365
363
|
} catch (error: any) {
|
|
366
|
-
console.error(
|
|
367
|
-
console.error(
|
|
364
|
+
console.error(`${_logPrefix}OutputInfluxDB: Could not write data point: [${error}]`.red);
|
|
365
|
+
console.error(`${JSON.stringify(instrumentPayload.context)}`.red);
|
|
368
366
|
return false;
|
|
369
367
|
}
|
|
370
368
|
} else {
|
|
371
|
-
console.error(
|
|
369
|
+
console.error(`${_logPrefix}OutputInfluxDBService: Could not write data point as writeClient is null`.red);
|
|
372
370
|
return false;
|
|
373
371
|
}
|
|
374
372
|
}
|
|
@@ -7,9 +7,6 @@ import { v4 as uuidv4 } from 'uuid';
|
|
|
7
7
|
|
|
8
8
|
import { $Options } from '@nsshunt/stsconfig'
|
|
9
9
|
import { STSOptionsBase } from '@nsshunt/stsutils';
|
|
10
|
-
|
|
11
|
-
import chalk from 'chalk';
|
|
12
|
-
|
|
13
10
|
const goptions = $Options()
|
|
14
11
|
|
|
15
12
|
declare interface IKafkaSubscription {
|
|
@@ -83,12 +80,12 @@ export class InstrumentationSubscriber extends STSOptionsBase {
|
|
|
83
80
|
const kafka = subscription.kafka;
|
|
84
81
|
if (kafka) {
|
|
85
82
|
// Un-Subscribe from this kafka topic
|
|
86
|
-
this.#Log(
|
|
83
|
+
this.#Log(`InstrumentationSubscriber:#UnSubscribeKafka(): Unsubscribe from Kafka topc: [${kafka.kafkaTopic}] Starting`.yellow)
|
|
87
84
|
await kafka.kafkaConsumer?.Stop();
|
|
88
85
|
await kafka.kafkaConsumer?.Disconnect();
|
|
89
|
-
this.#Log(
|
|
86
|
+
this.#Log(`InstrumentationSubscriber:#UnSubscribeKafka(): Unsubscribe from Kafka topc: [${kafka.kafkaTopic}] Completed.`.yellow)
|
|
90
87
|
} else {
|
|
91
|
-
this.#Log(
|
|
88
|
+
this.#Log(`InstrumentationSubscriber:#UnSubscribeKafka(): Kafka details do not exist for this topc: [${JSON.stringify(subscription.subscriptionKey)}].`.magenta)
|
|
92
89
|
}
|
|
93
90
|
};
|
|
94
91
|
return unsubscribeKafka();
|
|
@@ -129,18 +126,18 @@ export class InstrumentationSubscriber extends STSOptionsBase {
|
|
|
129
126
|
} as IKafkaData
|
|
130
127
|
});
|
|
131
128
|
} else {
|
|
132
|
-
this.#Log(
|
|
129
|
+
this.#Log(`InstrumentationSubscriber:#SubscribeKafka(): Invalid message format from Kafka data, topic: [${topic}] and key: [${key}] do not match`.red);
|
|
133
130
|
}
|
|
134
131
|
} else {
|
|
135
|
-
this.#Log(
|
|
132
|
+
this.#Log(`InstrumentationSubscriber:#SubscribeKafka(): Invalid message format from Kafka data, topic: [${topic}], key: [${key}], value: [${value}]`.red);
|
|
136
133
|
}
|
|
137
134
|
} catch (error) {
|
|
138
|
-
this.#Log(
|
|
135
|
+
this.#Log(`InstrumentationSubscriber:#SubscribeKafka(): Error: [${error}]`.red);
|
|
139
136
|
}
|
|
140
137
|
});
|
|
141
|
-
this.#Log(
|
|
138
|
+
this.#Log(`InstrumentationSubscriber:#SubscribeKafka(): Subscribe to Kafka topc: [${kafka.kafkaTopic}] Completed`.yellow)
|
|
142
139
|
} else {
|
|
143
|
-
this.#Log(
|
|
140
|
+
this.#Log(`InstrumentationSubscriber:#SubscribeKafka(): Could not subscribe to Kafka topc, missing combinedKey for topic: [${JSON.stringify(subscription.subscriptionKey)}]`.magenta);
|
|
144
141
|
}
|
|
145
142
|
}
|
|
146
143
|
return kafkaSubscribe();
|
|
@@ -215,7 +212,7 @@ export class InstrumentationSubscriber extends STSOptionsBase {
|
|
|
215
212
|
}
|
|
216
213
|
}
|
|
217
214
|
} catch (error) {
|
|
218
|
-
this.#Log(
|
|
215
|
+
this.#Log(`InstrumentationSubscriber:#ExecuteSubscriptions(): Error: [${error}], [${JSON.stringify(retVal)}]`.red);
|
|
219
216
|
}
|
|
220
217
|
}
|
|
221
218
|
|
|
@@ -226,7 +223,7 @@ export class InstrumentationSubscriber extends STSOptionsBase {
|
|
|
226
223
|
}
|
|
227
224
|
if (this.#subscriptions[subscriptionKey.id]) {
|
|
228
225
|
// Subscription already exists - ignoring
|
|
229
|
-
this.#Log(
|
|
226
|
+
this.#Log(`InstrumentationSubscriber.Subscribe(): Subscription already exists: [${subscriptionKey.id}] - ignoring.`.magenta);
|
|
230
227
|
} else {
|
|
231
228
|
this.#subscriptions[subscriptionKey.id] = {
|
|
232
229
|
subscriptionKey,
|
|
@@ -253,7 +250,7 @@ export class InstrumentationSubscriber extends STSOptionsBase {
|
|
|
253
250
|
|
|
254
251
|
UnSubscribe(subscriptionKey: ISubscriptionKey) {
|
|
255
252
|
if (!this.#subscriptions[subscriptionKey.id]) {
|
|
256
|
-
this.#Log(
|
|
253
|
+
this.#Log(`UnSubscribe(): Warning, subscriptionKey.id: [${subscriptionKey.id}] does not exist within subscription records`.magenta);
|
|
257
254
|
} else {
|
|
258
255
|
if (subscriptionKey.topic.localeCompare(SubscriptionTopic.LogProcessing) === 0) {
|
|
259
256
|
this.#UnSubscribeKafka(this.#subscriptions[subscriptionKey.id]);
|
|
@@ -9,8 +9,6 @@ import { KafkaProducer } from './kafkaproducer'
|
|
|
9
9
|
import { JSONObject } from '@nsshunt/stsutils'
|
|
10
10
|
import { v4 as uuidv4 } from 'uuid';
|
|
11
11
|
|
|
12
|
-
import chalk from 'chalk';
|
|
13
|
-
|
|
14
12
|
import { $Options } from '@nsshunt/stsconfig'
|
|
15
13
|
const goptions = $Options()
|
|
16
14
|
|
|
@@ -71,7 +69,7 @@ export class IMKafkaManager {
|
|
|
71
69
|
}
|
|
72
70
|
}
|
|
73
71
|
} catch (error) {
|
|
74
|
-
console.error(
|
|
72
|
+
console.error(`${_logPrefix}#Terminate: Error: [${error}]`.red);
|
|
75
73
|
}
|
|
76
74
|
}
|
|
77
75
|
|
|
@@ -147,7 +145,7 @@ export class IMKafkaManager {
|
|
|
147
145
|
}
|
|
148
146
|
return true;
|
|
149
147
|
} catch (error: any) {
|
|
150
|
-
console.error(
|
|
148
|
+
console.error(`${_logPrefix}OutputLogs: Could not output log data to kafka: [${error}]`.red);
|
|
151
149
|
return false;
|
|
152
150
|
}
|
|
153
151
|
}
|
|
@@ -56,8 +56,6 @@ import { KafkaManager } from './../kafka/kafkamanager'
|
|
|
56
56
|
|
|
57
57
|
import 'colors'
|
|
58
58
|
|
|
59
|
-
import chalk from 'chalk';
|
|
60
|
-
|
|
61
59
|
async function Sleep(milliseconds = 1000) {
|
|
62
60
|
return new Promise(resolve => setTimeout(resolve, milliseconds))
|
|
63
61
|
}
|
|
@@ -94,7 +92,7 @@ const runme = async () => {
|
|
|
94
92
|
for (let i=0; i < count; i++) {
|
|
95
93
|
if (i % 100 === 0) console.log(i);
|
|
96
94
|
const retVal = await producer.SendMessages(TOPIC, [
|
|
97
|
-
{ key: 'key1', value:
|
|
95
|
+
{ key: 'key1', value: `hello world - ${i}`.green },
|
|
98
96
|
{ key: 'key2', value: 'hey hey! -2' },
|
|
99
97
|
{ key: 'key3', value: 'hey hey! -3' },
|
|
100
98
|
{ key: 'key4', value: 'hey hey! -4' },
|
package/src/masterprocessbase.ts
CHANGED
|
@@ -7,7 +7,9 @@ import axios from 'axios';
|
|
|
7
7
|
|
|
8
8
|
import cluster, { Worker } from 'node:cluster'
|
|
9
9
|
|
|
10
|
-
import
|
|
10
|
+
import os from 'os';
|
|
11
|
+
|
|
12
|
+
import colors from 'colors'
|
|
11
13
|
|
|
12
14
|
import express from 'express'
|
|
13
15
|
|
|
@@ -22,6 +24,7 @@ const goptions = $Options()
|
|
|
22
24
|
import debugModule from 'debug'
|
|
23
25
|
|
|
24
26
|
import { Gauge, GaugeTypes, InstrumentGaugeTelemetry, InstrumentGaugeOptions, InstrumentHistogramTelemetry } from '@nsshunt/stsinstrumentation'
|
|
27
|
+
import { GetFirstNetworkInterface } from './network'
|
|
25
28
|
|
|
26
29
|
import { ProcessOptions, STSServerType } from './processoptions'
|
|
27
30
|
import { ProcessBase } from './processbase';
|
|
@@ -29,6 +32,7 @@ import { IMasterProcessBase } from './commonTypes';
|
|
|
29
32
|
|
|
30
33
|
import { InstrumentDefinitions } from '@nsshunt/stspublisherserver'
|
|
31
34
|
import { IPCMessagePayload, IPCMessageCommand } from './commonTypes'
|
|
35
|
+
import { REQUEST_HEADER_FIELDS_TOO_LARGE } from "http-status-codes";
|
|
32
36
|
|
|
33
37
|
|
|
34
38
|
export class MasterProcessBase extends ProcessBase implements IMasterProcessBase
|
|
@@ -186,15 +190,15 @@ export class MasterProcessBase extends ProcessBase implements IMasterProcessBase
|
|
|
186
190
|
,httpsAgent: this.#httpsAgent
|
|
187
191
|
});
|
|
188
192
|
if (retVal.status !== 200) {
|
|
189
|
-
this.#debug(
|
|
193
|
+
this.#debug(`Error (MasterProcessBase:#GetLatency): Invalid response from server: [${retVal.status}]`.magenta);
|
|
190
194
|
return null;
|
|
191
195
|
}
|
|
192
196
|
return retVal.data.detail;
|
|
193
197
|
} catch (error: any) {
|
|
194
|
-
this.#debug(
|
|
195
|
-
this.#debug(
|
|
198
|
+
this.#debug(`Error (MasterProcessBase:#GetLatency:catch): [${error}]`.red);
|
|
199
|
+
this.#debug(` url: [${url}]`.red);
|
|
196
200
|
if (error.response && error.response.data) {
|
|
197
|
-
this.#debug(
|
|
201
|
+
this.#debug(` Details: [${JSON.stringify(error.response.data)}]`.red);
|
|
198
202
|
}
|
|
199
203
|
}
|
|
200
204
|
}
|
|
@@ -232,11 +236,11 @@ export class MasterProcessBase extends ProcessBase implements IMasterProcessBase
|
|
|
232
236
|
worker.process.send( { command } );
|
|
233
237
|
return true;
|
|
234
238
|
} else {
|
|
235
|
-
this.LogEx(
|
|
239
|
+
this.LogEx(`Could not kill worker with id: [${id}]. The process does not exists`.red);
|
|
236
240
|
return false;
|
|
237
241
|
}
|
|
238
242
|
} else {
|
|
239
|
-
this.LogEx(
|
|
243
|
+
this.LogEx(`Could not kill worker with id: [${id}]. Worker does not exist within workers collection`.red);
|
|
240
244
|
return false;
|
|
241
245
|
}
|
|
242
246
|
}
|
|
@@ -338,7 +342,7 @@ export class MasterProcessBase extends ProcessBase implements IMasterProcessBase
|
|
|
338
342
|
}
|
|
339
343
|
default : {
|
|
340
344
|
const errorMessage = `Could not process command: [${iPCMessagePayload.command}].`;
|
|
341
|
-
this.#debug(
|
|
345
|
+
this.#debug(` ${errorMessage}`.red);
|
|
342
346
|
throw new Error(errorMessage);
|
|
343
347
|
}
|
|
344
348
|
}
|
|
@@ -377,9 +381,9 @@ export class MasterProcessBase extends ProcessBase implements IMasterProcessBase
|
|
|
377
381
|
// Only handle request/response message types here ...
|
|
378
382
|
if (payload.requestResponse) {
|
|
379
383
|
const iPCMessagePayload: IPCMessagePayload = payload as IPCMessagePayload;
|
|
380
|
-
this.#debug(
|
|
384
|
+
this.#debug(`Received message with id: [${iPCMessagePayload.id}] from worker: [${worker.process.pid}]. Details: [${JSON.stringify(iPCMessagePayload)}]`.yellow);
|
|
381
385
|
const response: IPCMessagePayload = await this.#processIPCCommand(iPCMessagePayload);
|
|
382
|
-
this.#debug(
|
|
386
|
+
this.#debug(`Sending response message with id: [${iPCMessagePayload.id}] to worker: [${worker.process.pid}]. Details: [${JSON.stringify(response)}]`.green);
|
|
383
387
|
worker.send(response);
|
|
384
388
|
} else {
|
|
385
389
|
this.#WorkerMessageEvent(payload);
|
|
@@ -443,21 +447,25 @@ export class MasterProcessBase extends ProcessBase implements IMasterProcessBase
|
|
|
443
447
|
LogEx(`Worker process ${worker.process.pid} is online`);
|
|
444
448
|
});
|
|
445
449
|
|
|
446
|
-
cluster.on('exit', (worker, code, signal) =>
|
|
447
|
-
|
|
448
|
-
|
|
450
|
+
cluster.on('exit', (worker, code, signal) =>
|
|
451
|
+
{
|
|
452
|
+
if ((code !== null && code === 0) || (signal === 'SIGINT'))
|
|
453
|
+
{
|
|
454
|
+
LogEx(`Process ${worker.process.pid} terminated gracefully with code: ${code}, signal: ${signal}`.green);
|
|
449
455
|
this.DecWorkers();
|
|
450
|
-
} else if ((code !== null && code === 15) || (signal === 'SIGTERM'))
|
|
456
|
+
} else if ((code !== null && code === 15) || (signal === 'SIGTERM'))
|
|
457
|
+
{
|
|
451
458
|
this.DecWorkers();
|
|
452
|
-
LogEx(
|
|
459
|
+
LogEx(`Process ${worker.process.pid} terminated with code: ${code}, signal: ${signal}`.red);
|
|
453
460
|
} else {
|
|
454
461
|
this.DecWorkers();
|
|
455
|
-
LogEx(
|
|
456
|
-
LogEx(
|
|
457
|
-
LogEx(
|
|
458
|
-
LogEx(
|
|
459
|
-
if (goptions.respawnOnFail === true)
|
|
460
|
-
|
|
462
|
+
LogEx(`worker ${worker.process.pid} died`.red);
|
|
463
|
+
LogEx(`code: ${code}`.red);
|
|
464
|
+
LogEx(`signal: ${signal}`.red);
|
|
465
|
+
LogEx('process terminated in an error state'.red);
|
|
466
|
+
if (goptions.respawnOnFail === true)
|
|
467
|
+
{
|
|
468
|
+
LogEx(`Attemping to respawn worker`);
|
|
461
469
|
this.#SpawnWorker();
|
|
462
470
|
}
|
|
463
471
|
}
|
|
@@ -556,9 +564,9 @@ export class MasterProcessBase extends ProcessBase implements IMasterProcessBase
|
|
|
556
564
|
{
|
|
557
565
|
if (code === 0)
|
|
558
566
|
{
|
|
559
|
-
LogEx(
|
|
567
|
+
LogEx(`Main Process: ${process.pid} terminated gracefully with code: ${code}`.green);
|
|
560
568
|
} else {
|
|
561
|
-
LogEx(
|
|
569
|
+
LogEx(`Main Process: ${process.pid} terminated with code: ${code}`.red);
|
|
562
570
|
}
|
|
563
571
|
});
|
|
564
572
|
|
|
@@ -570,7 +578,7 @@ export class MasterProcessBase extends ProcessBase implements IMasterProcessBase
|
|
|
570
578
|
|
|
571
579
|
this.MasterStarted();
|
|
572
580
|
|
|
573
|
-
LogEx(
|
|
581
|
+
LogEx(`Master process:${process.pid} started`.green);
|
|
574
582
|
}
|
|
575
583
|
|
|
576
584
|
BroadcastDataToWorkers = (command: any, data: any) => {
|
package/src/processbase.ts
CHANGED
|
@@ -7,8 +7,6 @@ import colors from 'colors'
|
|
|
7
7
|
import si from 'systeminformation' // https://systeminformation.io/
|
|
8
8
|
import { GetFirstNetworkInterface } from './network'
|
|
9
9
|
|
|
10
|
-
import chalk from 'chalk';
|
|
11
|
-
|
|
12
10
|
import debugModule from 'debug'
|
|
13
11
|
const debug = debugModule(`proc:${process.pid}:processBase`);
|
|
14
12
|
|
|
@@ -78,10 +76,10 @@ export abstract class ProcessBase extends STSOptionsBase implements IProcessBase
|
|
|
78
76
|
if (cluster.isPrimary)
|
|
79
77
|
{
|
|
80
78
|
prefix = 'M';
|
|
81
|
-
col = colors.bold.cyan;
|
|
79
|
+
col = colors.bold.cyan;
|
|
82
80
|
} else {
|
|
83
81
|
prefix = 'W';
|
|
84
|
-
col = colors.green;
|
|
82
|
+
col = colors.green;
|
|
85
83
|
}
|
|
86
84
|
const msgEx = col(`${prefix}(${process.pid}) [${appName}]: ${msg}`);
|
|
87
85
|
debug(msgEx);
|
|
@@ -336,13 +334,13 @@ export abstract class ProcessBase extends STSOptionsBase implements IProcessBase
|
|
|
336
334
|
try {
|
|
337
335
|
const retVal = await (this.accessLayer as PGAccessLayer).getResourceCount();
|
|
338
336
|
if (retVal.status !== StatusCodes.OK) {
|
|
339
|
-
this.LogEx(
|
|
337
|
+
this.LogEx(`Unable to get resources from the database. Is the database running? [${JSON.stringify(retVal)}]`.red);
|
|
340
338
|
exit(1);
|
|
341
339
|
} else {
|
|
342
|
-
this.LogEx(
|
|
340
|
+
this.LogEx(`Database connection successful. Resources: [${retVal.detail.count}]`.green);
|
|
343
341
|
}
|
|
344
342
|
} catch (error) {
|
|
345
|
-
this.LogEx(
|
|
343
|
+
this.LogEx(`Unable to get resources from the database. Is the database running? Error: [${error}]`.red);
|
|
346
344
|
exit(1);
|
|
347
345
|
}
|
|
348
346
|
})();
|
|
@@ -404,9 +402,9 @@ export abstract class ProcessBase extends STSOptionsBase implements IProcessBase
|
|
|
404
402
|
GetSignalColour = (signal: any) => {
|
|
405
403
|
let msgcolor = null;
|
|
406
404
|
if (signal === 'SIGINT') {
|
|
407
|
-
msgcolor =
|
|
405
|
+
msgcolor = colors.yellow;
|
|
408
406
|
} else {
|
|
409
|
-
msgcolor =
|
|
407
|
+
msgcolor = colors.red;
|
|
410
408
|
}
|
|
411
409
|
return msgcolor;
|
|
412
410
|
};
|
package/src/server.ts
CHANGED
|
@@ -33,7 +33,7 @@ export class STSExpressServer
|
|
|
33
33
|
});
|
|
34
34
|
|
|
35
35
|
serverNetworkMiddleware.on(ServerNetworkMiddlewareEventName.UpdateInstrument_SERVER_NET_VAL, (data: ISocketRecord) => {
|
|
36
|
-
//console.log(
|
|
36
|
+
//console.log(`serverNetworkMiddleware: [${JSON.stringify(data)}]`.magenta)
|
|
37
37
|
stsApp.UpdateInstrument(Gauge.NETWORK_RX_GAUGE, {
|
|
38
38
|
Inc: data.requestBytesRead
|
|
39
39
|
} as InstrumentGaugeTelemetry);
|
package/src/singleprocessbase.ts
CHANGED
|
@@ -2,8 +2,6 @@
|
|
|
2
2
|
import { $Options } from '@nsshunt/stsconfig'
|
|
3
3
|
const goptions = $Options()
|
|
4
4
|
|
|
5
|
-
import chalk from 'chalk';
|
|
6
|
-
|
|
7
5
|
import { Gauge, GaugeTypes, InstrumentGaugeOptions, InstrumentGaugeTelemetry } from '@nsshunt/stsinstrumentation'
|
|
8
6
|
|
|
9
7
|
import { ProcessOptions } from './processoptions'
|
|
@@ -99,9 +97,9 @@ export class SingleProcessBase extends ServerProcessBase implements ISingleProce
|
|
|
99
97
|
|
|
100
98
|
process.on('exit', (code) => {
|
|
101
99
|
if (code === 0) {
|
|
102
|
-
this.LogEx(
|
|
100
|
+
this.LogEx(`Main Process: ${process.pid} terminated gracefully with code: ${code}`.green);
|
|
103
101
|
} else {
|
|
104
|
-
this.LogEx(
|
|
102
|
+
this.LogEx(`Main Process: ${process.pid} terminated with code: ${code}`.red);
|
|
105
103
|
}
|
|
106
104
|
});
|
|
107
105
|
|
|
@@ -109,7 +107,7 @@ export class SingleProcessBase extends ServerProcessBase implements ISingleProce
|
|
|
109
107
|
|
|
110
108
|
this.ProcessStarted();
|
|
111
109
|
|
|
112
|
-
this.LogEx(
|
|
110
|
+
this.LogEx(`Main process:${process.pid} started`.green);
|
|
113
111
|
}
|
|
114
112
|
|
|
115
113
|
async TerminateApplication()
|