@nsshunt/stsappframework 3.0.176 → 3.0.178
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commonTypes.js.map +1 -1
- package/dist/influxdb/influxDBManager.js +12 -8
- package/dist/influxdb/influxDBManager.js.map +1 -1
- package/dist/influxdb/influxDBManagerAgent.js +12 -9
- package/dist/influxdb/influxDBManagerAgent.js.map +1 -1
- package/dist/influxdb/influxDBManagerBase.js +6 -3
- package/dist/influxdb/influxDBManagerBase.js.map +1 -1
- package/dist/influxdb/influxDBManagerService.js +13 -10
- package/dist/influxdb/influxDBManagerService.js.map +1 -1
- package/dist/instrumentationsubscriber.js +26 -36
- package/dist/instrumentationsubscriber.js.map +1 -1
- package/dist/kafka/IMKafkaManager.js +10 -5
- package/dist/kafka/IMKafkaManager.js.map +1 -1
- package/dist/kafka/kafkaconsumer.js +11 -6
- package/dist/kafka/kafkaconsumer.js.map +1 -1
- package/dist/kafka/kafkamanager.js +15 -3
- package/dist/kafka/kafkamanager.js.map +1 -1
- package/dist/kafka/kafkaproducer.js +11 -6
- package/dist/kafka/kafkaproducer.js.map +1 -1
- package/dist/kafkatesting/consume.js +45 -24
- package/dist/kafkatesting/consume.js.map +1 -1
- package/dist/kafkatesting/produce.js +35 -17
- package/dist/kafkatesting/produce.js.map +1 -1
- package/dist/masterprocessbase.js +11 -2
- package/dist/masterprocessbase.js.map +1 -1
- package/dist/processbase.js +46 -39
- package/dist/processbase.js.map +1 -1
- package/dist/processoptions.js.map +1 -1
- package/dist/publishertransports/publishTransportDirect.js +12 -5
- package/dist/publishertransports/publishTransportDirect.js.map +1 -1
- package/dist/server.js +2 -2
- package/dist/server.js.map +1 -1
- package/dist/serverprocessbase.js +11 -11
- package/dist/serverprocessbase.js.map +1 -1
- package/dist/socketIoServerHelper.js +5 -5
- package/dist/socketIoServerHelper.js.map +1 -1
- package/dist/stslatencycontroller.js +1 -1
- package/dist/stslatencycontroller.js.map +1 -1
- package/dist/tcpclient/app.js +2 -1
- package/dist/tcpclient/app.js.map +1 -1
- package/dist/tcpclient/app2.js +7 -6
- package/dist/tcpclient/app2.js.map +1 -1
- package/dist/tcpserver/appConfig.js +24 -1
- package/dist/tcpserver/appConfig.js.map +1 -1
- package/dist/tcpserver/appmaster.js +33 -27
- package/dist/tcpserver/appmaster.js.map +1 -1
- package/dist/testing/app.js +14 -12
- package/dist/testing/app.js.map +1 -1
- package/dist/testing/appConfig.js +24 -1
- package/dist/testing/appConfig.js.map +1 -1
- package/dist/testing/appWorkerWSS.js +18 -12
- package/dist/testing/appWorkerWSS.js.map +1 -1
- package/dist/testing/appsolo.js +15 -13
- package/dist/testing/appsolo.js.map +1 -1
- package/dist/testing/singleservertest.test.js +11 -7
- package/dist/testing/singleservertest.test.js.map +1 -1
- package/dist/webworkertesting/app.js +8 -6
- package/dist/webworkertesting/app.js.map +1 -1
- package/dist/webworkertesting/worker.js +4 -2
- package/dist/webworkertesting/worker.js.map +1 -1
- package/package.json +8 -7
- package/src/commonTypes.ts +4 -4
- package/src/influxdb/influxDBManager.ts +14 -9
- package/src/influxdb/influxDBManagerAgent.ts +13 -9
- package/src/influxdb/influxDBManagerBase.ts +7 -3
- package/src/influxdb/influxDBManagerService.ts +14 -10
- package/src/instrumentationsubscriber.ts +29 -40
- package/src/kafka/IMKafkaManager.ts +16 -6
- package/src/kafka/kafkaconsumer.ts +20 -6
- package/src/kafka/kafkamanager.ts +16 -4
- package/src/kafka/kafkaproducer.ts +19 -6
- package/src/kafkatesting/consume.ts +55 -24
- package/src/kafkatesting/produce.ts +45 -17
- package/src/masterprocessbase.ts +15 -2
- package/src/processbase.ts +52 -45
- package/src/processoptions.ts +6 -0
- package/src/publishertransports/publishTransportDirect.ts +18 -5
- package/src/server.ts +2 -2
- package/src/serverprocessbase.ts +11 -11
- package/src/socketIoServerHelper.ts +5 -5
- package/src/stslatencycontroller.ts +1 -1
- package/src/tcpclient/app.ts +3 -1
- package/src/tcpclient/app2.ts +8 -6
- package/src/tcpserver/appConfig.ts +35 -1
- package/src/tcpserver/appmaster.ts +35 -27
- package/src/testing/app.ts +14 -12
- package/src/testing/appConfig.ts +35 -1
- package/src/testing/appWorkerWSS.ts +22 -14
- package/src/testing/appsolo.ts +16 -13
- package/src/testing/singleservertest.test.ts +13 -7
- package/src/webworkertesting/app.ts +9 -6
- package/src/webworkertesting/worker.ts +5 -2
- package/types/commonTypes.d.ts +4 -2
- package/types/commonTypes.d.ts.map +1 -1
- package/types/influxdb/influxDBManager.d.ts +2 -2
- package/types/influxdb/influxDBManager.d.ts.map +1 -1
- package/types/influxdb/influxDBManagerAgent.d.ts.map +1 -1
- package/types/influxdb/influxDBManagerBase.d.ts.map +1 -1
- package/types/influxdb/influxDBManagerService.d.ts.map +1 -1
- package/types/instrumentationsubscriber.d.ts +2 -3
- package/types/instrumentationsubscriber.d.ts.map +1 -1
- package/types/kafka/IMKafkaManager.d.ts +5 -1
- package/types/kafka/IMKafkaManager.d.ts.map +1 -1
- package/types/kafka/kafkaconsumer.d.ts +8 -1
- package/types/kafka/kafkaconsumer.d.ts.map +1 -1
- package/types/kafka/kafkamanager.d.ts +2 -1
- package/types/kafka/kafkamanager.d.ts.map +1 -1
- package/types/kafka/kafkaproducer.d.ts +7 -1
- package/types/kafka/kafkaproducer.d.ts.map +1 -1
- package/types/masterprocessbase.d.ts.map +1 -1
- package/types/processbase.d.ts +2 -0
- package/types/processbase.d.ts.map +1 -1
- package/types/processoptions.d.ts +3 -0
- package/types/processoptions.d.ts.map +1 -1
- package/types/publishertransports/publishTransportDirect.d.ts +5 -1
- package/types/publishertransports/publishTransportDirect.d.ts.map +1 -1
- package/types/tcpserver/appConfig.d.ts.map +1 -1
- package/types/tcpserver/appmaster.d.ts.map +1 -1
- package/types/testing/appConfig.d.ts.map +1 -1
- package/types/testing/appWorkerWSS.d.ts +2 -2
- package/types/testing/appWorkerWSS.d.ts.map +1 -1
- package/types/testing/appsolo.d.ts.map +1 -1
|
@@ -142,6 +142,10 @@ export class InfluxDBManagerService extends InfluxDBManagerBase
|
|
|
142
142
|
super(options, queryApi);
|
|
143
143
|
}
|
|
144
144
|
|
|
145
|
+
#LogErrorMessage(message: any) {
|
|
146
|
+
this.options.logger.error(message);
|
|
147
|
+
}
|
|
148
|
+
|
|
145
149
|
override get modelType(): string {
|
|
146
150
|
return 'service'
|
|
147
151
|
}
|
|
@@ -184,7 +188,7 @@ data
|
|
|
184
188
|
|
|
185
189
|
return this.queryApi.collectRows(query)
|
|
186
190
|
} catch (error) {
|
|
187
|
-
|
|
191
|
+
this.#LogErrorMessage(chalk.red(`${_logPrefix}#GetSTSCountGeneric: Error: [${error}]`));
|
|
188
192
|
}
|
|
189
193
|
}
|
|
190
194
|
|
|
@@ -200,7 +204,7 @@ data
|
|
|
200
204
|
|> difference()`;
|
|
201
205
|
return this.queryApi.collectRows(query)
|
|
202
206
|
} catch (error) {
|
|
203
|
-
|
|
207
|
+
this.#LogErrorMessage(chalk.red(`${_logPrefix}#GetSTSHistoGeneric: Error: [${error}]`));
|
|
204
208
|
}
|
|
205
209
|
}
|
|
206
210
|
|
|
@@ -231,7 +235,7 @@ data
|
|
|
231
235
|
|
|
232
236
|
return this.queryApi.collectRows(query)
|
|
233
237
|
} catch (error) {
|
|
234
|
-
|
|
238
|
+
this.#LogErrorMessage(chalk.red(`${_logPrefix}#GetSTSQuantileGeneric: Error: [${error}]`));
|
|
235
239
|
}
|
|
236
240
|
}
|
|
237
241
|
|
|
@@ -246,7 +250,7 @@ data
|
|
|
246
250
|
this.#GetSTSHistoGenericService(SERVICE_STATS_GLOBAL, '')],
|
|
247
251
|
[ ])
|
|
248
252
|
} catch (error) {
|
|
249
|
-
|
|
253
|
+
this.#LogErrorMessage(chalk.red(`${_logPrefix}GetInfluxDBResultsRoot: Error: [${error}]`));
|
|
250
254
|
}
|
|
251
255
|
return {
|
|
252
256
|
subscriptionKey,
|
|
@@ -264,7 +268,7 @@ data
|
|
|
264
268
|
this.#GetSTSHistoGenericService(SERVICE_STATS_BY_SERVICE, '')],
|
|
265
269
|
['serviceId'])
|
|
266
270
|
} catch (error) {
|
|
267
|
-
|
|
271
|
+
this.#LogErrorMessage(chalk.red(`${_logPrefix}GetInfluxDBResultsService: Error: [${error}]`));
|
|
268
272
|
}
|
|
269
273
|
return {
|
|
270
274
|
subscriptionKey,
|
|
@@ -284,7 +288,7 @@ data
|
|
|
284
288
|
this.#GetSTSHistoGenericService(SERVICE_STATS_BY_SERVICE_INSTANCE, `and r["serviceId"] == "${serviceId}"`)],
|
|
285
289
|
['serviceId', 'serviceInstanceId'])
|
|
286
290
|
} catch (error) {
|
|
287
|
-
|
|
291
|
+
this.#LogErrorMessage(chalk.red(`${_logPrefix}GetInfluxDBResultsServiceInstances: Error: [${error}]`));
|
|
288
292
|
}
|
|
289
293
|
return {
|
|
290
294
|
subscriptionKey,
|
|
@@ -304,7 +308,7 @@ data
|
|
|
304
308
|
this.#GetSTSHistoGenericService(SERVICE_STATS_BY_SERVICE_INSTANCE_PROCESS, `and r["serviceInstanceId"] == "${serviceInstanceId}"`)],
|
|
305
309
|
['serviceId', 'serviceInstanceId', 'serviceInstanceProcessId'])
|
|
306
310
|
} catch (error) {
|
|
307
|
-
|
|
311
|
+
this.#LogErrorMessage(chalk.red(`${_logPrefix}GetInfluxDBResultsServiceInstance: Error: [${error}]`));
|
|
308
312
|
}
|
|
309
313
|
return {
|
|
310
314
|
subscriptionKey,
|
|
@@ -370,12 +374,12 @@ data
|
|
|
370
374
|
|
|
371
375
|
return true;
|
|
372
376
|
} catch (error: any) {
|
|
373
|
-
|
|
374
|
-
|
|
377
|
+
this.#LogErrorMessage(chalk.red(`${_logPrefix}OutputInfluxDB: Could not write data point: [${error}]`));
|
|
378
|
+
this.#LogErrorMessage(chalk.red(`${JSON.stringify(instrumentPayload.context)}`));
|
|
375
379
|
return false;
|
|
376
380
|
}
|
|
377
381
|
} else {
|
|
378
|
-
|
|
382
|
+
this.#LogErrorMessage(chalk.red(`${_logPrefix}OutputInfluxDBService: Could not write data point as writeClient is null`));
|
|
379
383
|
return false;
|
|
380
384
|
}
|
|
381
385
|
}
|
|
@@ -1,7 +1,4 @@
|
|
|
1
|
-
import
|
|
2
|
-
const debug = debugModule(`proc:${process.pid}`);
|
|
3
|
-
|
|
4
|
-
import { IKafkaConsumer, logFunction } from './commonTypes'
|
|
1
|
+
import { IKafkaConsumer } from './commonTypes'
|
|
5
2
|
import { ISubscriptionPayload, ISubscriptionKey, SubscriptionTopic, IKafkaData } from '@nsshunt/stssocketio-client'
|
|
6
3
|
|
|
7
4
|
import { KafkaManager, IKafkaManagerConfig } from './kafka/kafkamanager'
|
|
@@ -9,7 +6,7 @@ import { InfluxDBManager } from './influxdb/influxDBManager'
|
|
|
9
6
|
import { v4 as uuidv4 } from 'uuid';
|
|
10
7
|
|
|
11
8
|
import { $Options } from '@nsshunt/stsconfig'
|
|
12
|
-
import { STSOptionsBase } from '@nsshunt/stsutils';
|
|
9
|
+
import { STSOptionsBase, ISTSLogger } from '@nsshunt/stsutils';
|
|
13
10
|
|
|
14
11
|
import chalk from 'chalk';
|
|
15
12
|
|
|
@@ -32,7 +29,7 @@ declare type ISubscriptions = Record<string, ISubscription>;
|
|
|
32
29
|
|
|
33
30
|
export interface IInstrumentationSubscriberOptions {
|
|
34
31
|
SubscriptionInterval: number
|
|
35
|
-
logger
|
|
32
|
+
logger: ISTSLogger
|
|
36
33
|
}
|
|
37
34
|
|
|
38
35
|
/**
|
|
@@ -43,17 +40,12 @@ export class InstrumentationSubscriber extends STSOptionsBase {
|
|
|
43
40
|
#influxDBManager: InfluxDBManager;
|
|
44
41
|
#timeout: NodeJS.Timer | null = null;
|
|
45
42
|
#km: KafkaManager;
|
|
46
|
-
#logger: logFunction | null = null;
|
|
47
43
|
#stop: boolean = true;
|
|
48
44
|
|
|
49
45
|
constructor(options: IInstrumentationSubscriberOptions) {
|
|
50
46
|
super(options);
|
|
51
47
|
|
|
52
|
-
|
|
53
|
-
this.#logger = options.logger;
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
this.#influxDBManager = new InfluxDBManager();
|
|
48
|
+
this.#influxDBManager = new InfluxDBManager(this.options?.logger);
|
|
57
49
|
|
|
58
50
|
const kmc: IKafkaManagerConfig = {
|
|
59
51
|
clientId: `${goptions.kafka_clientId}_${uuidv4()}`,
|
|
@@ -63,7 +55,8 @@ export class InstrumentationSubscriber extends STSOptionsBase {
|
|
|
63
55
|
connectionTimeout: goptions.kafka_connection_timeout,
|
|
64
56
|
logLevel: goptions.kafka_log_level,
|
|
65
57
|
useSSL: goptions.kafka_use_ssl,
|
|
66
|
-
requestTimeout: goptions.kafka_request_timeout
|
|
58
|
+
requestTimeout: goptions.kafka_request_timeout,
|
|
59
|
+
logger: this.options?.logger
|
|
67
60
|
}
|
|
68
61
|
if (goptions.kafka_use_ssl) {
|
|
69
62
|
kmc.ssl = {
|
|
@@ -77,12 +70,8 @@ export class InstrumentationSubscriber extends STSOptionsBase {
|
|
|
77
70
|
}
|
|
78
71
|
|
|
79
72
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
80
|
-
#
|
|
81
|
-
|
|
82
|
-
this.#logger(msg);
|
|
83
|
-
} else {
|
|
84
|
-
debug(msg);
|
|
85
|
-
}
|
|
73
|
+
#LogDebugMessage(message: any) {
|
|
74
|
+
this.options?.logger.debug(message);
|
|
86
75
|
}
|
|
87
76
|
|
|
88
77
|
async #UnSubscribeKafka(subscription: ISubscription): Promise<void> {
|
|
@@ -90,19 +79,19 @@ export class InstrumentationSubscriber extends STSOptionsBase {
|
|
|
90
79
|
const kafka = subscription.kafka;
|
|
91
80
|
if (kafka) {
|
|
92
81
|
// Un-Subscribe from this kafka topic
|
|
93
|
-
this.#
|
|
82
|
+
this.#LogDebugMessage(chalk.yellow(`InstrumentationSubscriber:#UnSubscribeKafka(): Unsubscribe from Kafka topc: [${kafka.kafkaTopic}] Starting`));
|
|
94
83
|
|
|
95
84
|
await kafka.kafkaConsumer?.Stop((error) => {
|
|
96
|
-
this.#
|
|
85
|
+
this.#LogDebugMessage(chalk.red(`InstrumentationSubscriber:#UnSubscribeKafka(): Kafka topic: [${kafka.kafkaTopic}], kafkaConsumer.Stop Error: [${error}]`));
|
|
97
86
|
});
|
|
98
87
|
|
|
99
88
|
await kafka.kafkaConsumer?.Disconnect((error) => {
|
|
100
|
-
this.#
|
|
89
|
+
this.#LogDebugMessage(chalk.red(`InstrumentationSubscriber:#UnSubscribeKafka(): Kafka topic: [${kafka.kafkaTopic}], kafkaConsumer.Disconnect Error: [${error}]`));
|
|
101
90
|
});
|
|
102
91
|
|
|
103
|
-
this.#
|
|
92
|
+
this.#LogDebugMessage(chalk.yellow(`InstrumentationSubscriber:#UnSubscribeKafka(): Unsubscribe from Kafka topc: [${kafka.kafkaTopic}] Completed.`));
|
|
104
93
|
} else {
|
|
105
|
-
this.#
|
|
94
|
+
this.#LogDebugMessage(chalk.magenta(`InstrumentationSubscriber:#UnSubscribeKafka(): Kafka details do not exist for this topc: [${JSON.stringify(subscription.subscriptionKey)}].`));
|
|
106
95
|
}
|
|
107
96
|
};
|
|
108
97
|
return unsubscribeKafka();
|
|
@@ -121,20 +110,20 @@ export class InstrumentationSubscriber extends STSOptionsBase {
|
|
|
121
110
|
kafkaConsumer: this.#km.CreateConsumer(kafkaGroupId),
|
|
122
111
|
kafkaFromBeginning: goptions.kafka_consume_from_beginning
|
|
123
112
|
};
|
|
124
|
-
this.#
|
|
113
|
+
this.#LogDebugMessage(chalk.yellow(`InstrumentationSubscriber:#SubscribeKafka(): Subscribe to Kafka topc: [${subscription.kafka.kafkaTopic}] Starting`));
|
|
125
114
|
|
|
126
115
|
const kafka = subscription.kafka;
|
|
127
116
|
|
|
128
117
|
let connectError = false;
|
|
129
118
|
await kafka.kafkaConsumer.Connect((error) => {
|
|
130
|
-
this.#
|
|
119
|
+
this.#LogDebugMessage(chalk.red(`InstrumentationSubscriber:#SubscribeKafka(): Kafka topic: [${kafka.kafkaTopic}], kafkaConsumer.Connect: Error: [${error}]`));
|
|
131
120
|
connectError = true;
|
|
132
121
|
});
|
|
133
122
|
|
|
134
123
|
if (!connectError) {
|
|
135
124
|
let subscribeError = false;
|
|
136
125
|
await kafka.kafkaConsumer.Subscribe([kafka.kafkaTopic], subscription.kafka.kafkaFromBeginning, (error) => {
|
|
137
|
-
this.#
|
|
126
|
+
this.#LogDebugMessage(chalk.red(`InstrumentationSubscriber:#SubscribeKafka(): Kafka topic: [${kafka.kafkaTopic}], kafkaConsumer.Subscribe: Error: [${error}]`));
|
|
138
127
|
subscribeError = true;
|
|
139
128
|
});
|
|
140
129
|
|
|
@@ -156,26 +145,26 @@ export class InstrumentationSubscriber extends STSOptionsBase {
|
|
|
156
145
|
} as IKafkaData
|
|
157
146
|
});
|
|
158
147
|
} else {
|
|
159
|
-
this.#
|
|
148
|
+
this.#LogDebugMessage(chalk.red(`InstrumentationSubscriber:#SubscribeKafka(): Invalid message format from Kafka data, topic: [${topic}] and key: [${key}] do not match`));
|
|
160
149
|
}
|
|
161
150
|
} else {
|
|
162
|
-
this.#
|
|
151
|
+
this.#LogDebugMessage(chalk.red(`InstrumentationSubscriber:#SubscribeKafka(): Invalid message format from Kafka data, topic: [${topic}], key: [${key}], value: [${value}]`));
|
|
163
152
|
}
|
|
164
153
|
} catch (error) {
|
|
165
|
-
this.#
|
|
154
|
+
this.#LogDebugMessage(chalk.red(`InstrumentationSubscriber:#SubscribeKafka(): Error: [${error}]`));
|
|
166
155
|
}
|
|
167
156
|
}, (error) => {
|
|
168
|
-
this.#
|
|
157
|
+
this.#LogDebugMessage(chalk.red(`InstrumentationSubscriber:#SubscribeKafka(): Kafka topic: [${kafka.kafkaTopic}], kafkaConsumer.StartConsumingMessages: Error: [${error}]`));
|
|
169
158
|
});
|
|
170
|
-
this.#
|
|
159
|
+
this.#LogDebugMessage(chalk.yellow(`InstrumentationSubscriber:#SubscribeKafka(): Subscribe to Kafka topc: [${kafka.kafkaTopic}] Completed`));
|
|
171
160
|
} else {
|
|
172
|
-
this.#
|
|
161
|
+
this.#LogDebugMessage(chalk.red(`InstrumentationSubscriber:#SubscribeKafka(): Kafka topic: [${kafka.kafkaTopic}], Could not subscribe to topic, StartConsumingMessages aborted.`));
|
|
173
162
|
}
|
|
174
163
|
} else {
|
|
175
|
-
this.#
|
|
164
|
+
this.#LogDebugMessage(chalk.red(`InstrumentationSubscriber:#SubscribeKafka(): Kafka topic: [${kafka.kafkaTopic}], Could not connect to topic, subscribe aborted.`));
|
|
176
165
|
}
|
|
177
166
|
} else {
|
|
178
|
-
this.#
|
|
167
|
+
this.#LogDebugMessage(chalk.magenta(`InstrumentationSubscriber:#SubscribeKafka(): Could not subscribe to Kafka topc, missing combinedKey for topic: [${JSON.stringify(subscription.subscriptionKey)}]`));
|
|
179
168
|
}
|
|
180
169
|
}
|
|
181
170
|
return kafkaSubscribe();
|
|
@@ -250,7 +239,7 @@ export class InstrumentationSubscriber extends STSOptionsBase {
|
|
|
250
239
|
}
|
|
251
240
|
}
|
|
252
241
|
} catch (error) {
|
|
253
|
-
this.#
|
|
242
|
+
this.#LogDebugMessage(chalk.red(`InstrumentationSubscriber:#ExecuteSubscriptions(): Error: [${error}], [${JSON.stringify(retVal)}]`));
|
|
254
243
|
}
|
|
255
244
|
}
|
|
256
245
|
|
|
@@ -271,7 +260,7 @@ export class InstrumentationSubscriber extends STSOptionsBase {
|
|
|
271
260
|
if (newTimeout < 500) {
|
|
272
261
|
newTimeout = 500;
|
|
273
262
|
}
|
|
274
|
-
this.#
|
|
263
|
+
this.#LogDebugMessage(chalk.grey(`InstrumentationSubscriber.#SetupTimeout(): ExecuteSubscriptions time: [${diff}], next timeout: [${newTimeout}]`));
|
|
275
264
|
this.#SetupTimeout(newTimeout);
|
|
276
265
|
}
|
|
277
266
|
}, timeout);
|
|
@@ -285,7 +274,7 @@ export class InstrumentationSubscriber extends STSOptionsBase {
|
|
|
285
274
|
}
|
|
286
275
|
if (this.#subscriptions[subscriptionKey.id]) {
|
|
287
276
|
// Subscription already exists - ignoring
|
|
288
|
-
this.#
|
|
277
|
+
this.#LogDebugMessage(chalk.magenta(`InstrumentationSubscriber.Subscribe(): Subscription already exists: [${subscriptionKey.id}] - ignoring.`));
|
|
289
278
|
} else {
|
|
290
279
|
this.#subscriptions[subscriptionKey.id] = {
|
|
291
280
|
subscriptionKey,
|
|
@@ -294,7 +283,7 @@ export class InstrumentationSubscriber extends STSOptionsBase {
|
|
|
294
283
|
if (subscriptionKey.topic.localeCompare(SubscriptionTopic.LogProcessing) === 0) {
|
|
295
284
|
this.#SubscribeKafka(this.#subscriptions[subscriptionKey.id]);
|
|
296
285
|
}
|
|
297
|
-
this.#
|
|
286
|
+
this.#LogDebugMessage(chalk.grey(`InstrumentationSubscriber.Subscribe(): subscriptionKey.id: [${subscriptionKey.id}] successfully subscribed.`));
|
|
298
287
|
|
|
299
288
|
if (!this.#timeout) {
|
|
300
289
|
(async () => {
|
|
@@ -312,7 +301,7 @@ export class InstrumentationSubscriber extends STSOptionsBase {
|
|
|
312
301
|
|
|
313
302
|
UnSubscribe(subscriptionKey: ISubscriptionKey) {
|
|
314
303
|
if (!this.#subscriptions[subscriptionKey.id]) {
|
|
315
|
-
this.#
|
|
304
|
+
this.#LogDebugMessage(chalk.magenta(`UnSubscribe(): Warning, subscriptionKey.id: [${subscriptionKey.id}] does not exist within subscription records`));
|
|
316
305
|
} else {
|
|
317
306
|
if (subscriptionKey.topic.localeCompare(SubscriptionTopic.LogProcessing) === 0) {
|
|
318
307
|
this.#UnSubscribeKafka(this.#subscriptions[subscriptionKey.id]);
|
|
@@ -6,7 +6,7 @@ import { KAFKA_PREFIX } from '@nsshunt/stssocketio-client'
|
|
|
6
6
|
import { KafkaManager, IKafkaManagerConfig } from './kafkamanager'
|
|
7
7
|
import { KafkaProducer } from './kafkaproducer'
|
|
8
8
|
|
|
9
|
-
import { JSONObject } from '@nsshunt/stsutils'
|
|
9
|
+
import { ISTSLogger, JSONObject } from '@nsshunt/stsutils'
|
|
10
10
|
import { v4 as uuidv4 } from 'uuid';
|
|
11
11
|
|
|
12
12
|
import chalk from 'chalk';
|
|
@@ -17,17 +17,22 @@ const goptions = $Options()
|
|
|
17
17
|
import debugModule from 'debug'
|
|
18
18
|
const debug = debugModule(`proc:${process.pid}`);
|
|
19
19
|
|
|
20
|
+
export interface IMKafkaManagerOptions {
|
|
21
|
+
logger: ISTSLogger
|
|
22
|
+
}
|
|
23
|
+
|
|
20
24
|
// Manage the publication of InstrumentPayload to Kafka
|
|
21
25
|
export class IMKafkaManager {
|
|
26
|
+
#options: IMKafkaManagerOptions;
|
|
22
27
|
#shuttingDown: boolean = false;
|
|
23
28
|
#producer: KafkaProducer | null = null;
|
|
24
29
|
|
|
25
30
|
#km: KafkaManager;
|
|
26
31
|
#topics: JSONObject = { }
|
|
27
32
|
|
|
28
|
-
constructor()
|
|
33
|
+
constructor(options: IMKafkaManagerOptions)
|
|
29
34
|
{
|
|
30
|
-
|
|
35
|
+
this.#options = options;
|
|
31
36
|
|
|
32
37
|
const kmc: IKafkaManagerConfig = {
|
|
33
38
|
clientId: `${goptions.kafka_clientId}_${uuidv4()}`,
|
|
@@ -37,7 +42,8 @@ export class IMKafkaManager {
|
|
|
37
42
|
connectionTimeout: goptions.kafka_connection_timeout,
|
|
38
43
|
logLevel: goptions.kafka_log_level,
|
|
39
44
|
useSSL: goptions.kafka_use_ssl,
|
|
40
|
-
requestTimeout: goptions.kafka_request_timeout
|
|
45
|
+
requestTimeout: goptions.kafka_request_timeout,
|
|
46
|
+
logger: options.logger
|
|
41
47
|
}
|
|
42
48
|
if (goptions.kafka_use_ssl) {
|
|
43
49
|
kmc.ssl = {
|
|
@@ -51,8 +57,12 @@ export class IMKafkaManager {
|
|
|
51
57
|
this.#km = new KafkaManager(kmc);
|
|
52
58
|
}
|
|
53
59
|
|
|
60
|
+
#LogErrorMessage(message: any) {
|
|
61
|
+
this.#options.logger.error(message);
|
|
62
|
+
}
|
|
63
|
+
|
|
54
64
|
#RaiseError = (msg: string, errorCb: (error: any) => void) => {
|
|
55
|
-
|
|
65
|
+
this.#LogErrorMessage(chalk.red(msg));
|
|
56
66
|
errorCb(msg);
|
|
57
67
|
}
|
|
58
68
|
|
|
@@ -119,7 +129,7 @@ export class IMKafkaManager {
|
|
|
119
129
|
// Don't wait for these to return ...
|
|
120
130
|
if (this.#producer) {
|
|
121
131
|
this.#producer.SendMessages(topic, messages as any, (error) => {
|
|
122
|
-
|
|
132
|
+
this.#LogErrorMessage(chalk.red(`${process.pid}:IMKafkaManager:OutputLogsToKafkaTopic(): Kafka topic: [${topic}], Error during producer.SendMessages: [${error}]`));
|
|
123
133
|
});
|
|
124
134
|
}
|
|
125
135
|
} catch (error) {
|
|
@@ -3,6 +3,14 @@ import { Kafka, Consumer, IHeaders } from 'kafkajs'
|
|
|
3
3
|
import { IKafkaConsumer } from './../commonTypes'
|
|
4
4
|
|
|
5
5
|
import chalk from 'chalk';
|
|
6
|
+
import { ISTSLogger } from '@nsshunt/stsutils';
|
|
7
|
+
|
|
8
|
+
export interface IKafkaConsumerOptions {
|
|
9
|
+
kafka: Kafka
|
|
10
|
+
id: string
|
|
11
|
+
groupId: string
|
|
12
|
+
logger: ISTSLogger
|
|
13
|
+
}
|
|
6
14
|
|
|
7
15
|
export class KafkaConsumer implements IKafkaConsumer {
|
|
8
16
|
#id: string
|
|
@@ -10,22 +18,28 @@ export class KafkaConsumer implements IKafkaConsumer {
|
|
|
10
18
|
#consumer: Consumer;
|
|
11
19
|
#kafka: Kafka;
|
|
12
20
|
#connected: boolean = false;
|
|
21
|
+
#options: IKafkaConsumerOptions;
|
|
13
22
|
|
|
14
|
-
constructor(
|
|
15
|
-
this.#
|
|
16
|
-
this.#
|
|
17
|
-
this.#
|
|
23
|
+
constructor(options: IKafkaConsumerOptions) {
|
|
24
|
+
this.#options = options;
|
|
25
|
+
this.#id = this.#options.id;
|
|
26
|
+
this.#groupId = this.#options.groupId;
|
|
27
|
+
this.#kafka = this.#options.kafka;
|
|
18
28
|
this.#consumer = this.#kafka.consumer({
|
|
19
29
|
groupId: this.#groupId,
|
|
20
30
|
retry: {
|
|
21
31
|
restartOnFailure: async (error: Error): Promise<boolean> => {
|
|
22
|
-
|
|
32
|
+
this.#LogErrorMessage(chalk.magenta(`${process.pid}:KafkaConsumer:constructor:restartOnFailure(): Error: [${error}]`))
|
|
23
33
|
return true;
|
|
24
34
|
}
|
|
25
35
|
}
|
|
26
36
|
})
|
|
27
37
|
}
|
|
28
38
|
|
|
39
|
+
#LogErrorMessage(message: any) {
|
|
40
|
+
this.#options.logger.error(message);
|
|
41
|
+
}
|
|
42
|
+
|
|
29
43
|
get consumer() {
|
|
30
44
|
return this.#consumer;
|
|
31
45
|
}
|
|
@@ -36,7 +50,7 @@ export class KafkaConsumer implements IKafkaConsumer {
|
|
|
36
50
|
|
|
37
51
|
#RaiseError = (msg: string, errorCb: (error: any) => void) => {
|
|
38
52
|
const errorMessage = `${process.pid}:KafkaConsumer:${msg}`;
|
|
39
|
-
|
|
53
|
+
this.#LogErrorMessage(chalk.red(errorMessage));
|
|
40
54
|
errorCb(errorMessage);
|
|
41
55
|
}
|
|
42
56
|
|
|
@@ -51,7 +51,7 @@ services:
|
|
|
51
51
|
- KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://192.168.14.92:9092
|
|
52
52
|
|
|
53
53
|
*/
|
|
54
|
-
import { STSOptionsBase } from '@nsshunt/stsutils'
|
|
54
|
+
import { ISTSLogger, STSOptionsBase } from '@nsshunt/stsutils'
|
|
55
55
|
|
|
56
56
|
import { Kafka, KafkaConfig, logLevel } from 'kafkajs'
|
|
57
57
|
|
|
@@ -84,6 +84,7 @@ export interface IKafkaManagerConfig {
|
|
|
84
84
|
keyfile: string
|
|
85
85
|
certfileFile: string
|
|
86
86
|
}
|
|
87
|
+
logger: ISTSLogger
|
|
87
88
|
}
|
|
88
89
|
|
|
89
90
|
// https://kafka.js.org/docs/configuration
|
|
@@ -153,8 +154,12 @@ export class KafkaManager extends STSOptionsBase {
|
|
|
153
154
|
this.#kafka = new Kafka(kc);
|
|
154
155
|
}
|
|
155
156
|
|
|
157
|
+
#LogErrorMessage(message: any) {
|
|
158
|
+
this.options?.logger.error(message);
|
|
159
|
+
}
|
|
160
|
+
|
|
156
161
|
#RaiseError = (msg: string, errorCb: (error: any) => void) => {
|
|
157
|
-
|
|
162
|
+
this.#LogErrorMessage(chalk.red(msg));
|
|
158
163
|
errorCb(msg);
|
|
159
164
|
}
|
|
160
165
|
|
|
@@ -163,11 +168,18 @@ export class KafkaManager extends STSOptionsBase {
|
|
|
163
168
|
}
|
|
164
169
|
|
|
165
170
|
CreateProducer(): KafkaProducer {
|
|
166
|
-
return new KafkaProducer(
|
|
171
|
+
return new KafkaProducer({
|
|
172
|
+
kafka: this.#kafka,
|
|
173
|
+
id: uuidv4(),
|
|
174
|
+
logger: this.options?.logger});
|
|
167
175
|
}
|
|
168
176
|
|
|
169
177
|
CreateConsumer(groupId: string) {
|
|
170
|
-
return new KafkaConsumer(
|
|
178
|
+
return new KafkaConsumer({
|
|
179
|
+
kafka: this.#kafka,
|
|
180
|
+
id: uuidv4(),
|
|
181
|
+
groupId,
|
|
182
|
+
logger: this.options?.logger});
|
|
171
183
|
}
|
|
172
184
|
|
|
173
185
|
CreateTopic = async (topic: string, partitions: number, errorCb: (error: any) => void): Promise<boolean> => {
|
|
@@ -2,21 +2,34 @@
|
|
|
2
2
|
import { Kafka, Producer, RecordMetadata } from 'kafkajs'
|
|
3
3
|
|
|
4
4
|
import chalk from 'chalk';
|
|
5
|
+
import { ISTSLogger } from '@nsshunt/stsutils';
|
|
6
|
+
|
|
7
|
+
export interface IKafkaProducerOptions {
|
|
8
|
+
kafka: Kafka
|
|
9
|
+
id: string
|
|
10
|
+
logger: ISTSLogger
|
|
11
|
+
}
|
|
5
12
|
|
|
6
13
|
export class KafkaProducer {
|
|
14
|
+
#options: IKafkaProducerOptions;
|
|
7
15
|
#id: string
|
|
8
16
|
#producer: Producer;
|
|
9
17
|
#kafka: Kafka;
|
|
10
18
|
#connected: boolean = false;
|
|
11
19
|
|
|
12
|
-
constructor(
|
|
13
|
-
this.#
|
|
14
|
-
this.#
|
|
20
|
+
constructor(options: IKafkaProducerOptions) {
|
|
21
|
+
this.#options = options;
|
|
22
|
+
this.#id = this.#options.id;
|
|
23
|
+
this.#kafka = this.#options.kafka;
|
|
15
24
|
this.#producer = this.#kafka.producer()
|
|
16
25
|
}
|
|
17
26
|
|
|
27
|
+
#LogErrorMessage(message: any) {
|
|
28
|
+
this.#options.logger.error(message);
|
|
29
|
+
}
|
|
30
|
+
|
|
18
31
|
#RaiseError = (msg: string, errorCb: (error: any) => void) => {
|
|
19
|
-
|
|
32
|
+
this.#LogErrorMessage(chalk.red(msg));
|
|
20
33
|
errorCb(msg);
|
|
21
34
|
}
|
|
22
35
|
|
|
@@ -58,7 +71,7 @@ export class KafkaProducer {
|
|
|
58
71
|
messages: [ message ]
|
|
59
72
|
})
|
|
60
73
|
} catch (error) {
|
|
61
|
-
|
|
74
|
+
this.#LogErrorMessage(chalk.red(`${process.pid}:KafkaProducer:SendMessage(): Error: [${error}]`));
|
|
62
75
|
return [ ];
|
|
63
76
|
}
|
|
64
77
|
} else {
|
|
@@ -77,7 +90,7 @@ export class KafkaProducer {
|
|
|
77
90
|
messages
|
|
78
91
|
})
|
|
79
92
|
} catch (error) {
|
|
80
|
-
|
|
93
|
+
this.#LogErrorMessage(chalk.red(`${process.pid}:KafkaProducer:SendMessages(): Error: [${error}]`));
|
|
81
94
|
return [ ];
|
|
82
95
|
}
|
|
83
96
|
} else {
|