@nsshunt/stsappframework 3.0.20 → 3.0.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,239 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.InstrumentationSubscriber = void 0;
4
+ const commonTypes_1 = require("./commonTypes");
5
+ const kafkamanager_1 = require("./kafka/kafkamanager");
6
+ const influxDBManager_1 = require("./influxdb/influxDBManager");
7
+ const uuid_1 = require("uuid");
8
+ const stsconfig_1 = require("@nsshunt/stsconfig");
9
+ const stsutils_1 = require("@nsshunt/stsutils");
10
+ const goptions = (0, stsconfig_1.$Options)();
11
+ class InstrumentationSubscriber extends stsutils_1.STSOptionsBase {
12
+ #subscriptions = {};
13
+ #influxDBManager;
14
+ #interval = null;
15
+ #km;
16
+ #KAFKA_PREFIX = '__STS__';
17
+ constructor(options) {
18
+ super(options);
19
+ this.#influxDBManager = new influxDBManager_1.InfluxDBManager();
20
+ const kmc = {
21
+ clientId: goptions.kafka_clientId,
22
+ brokers: goptions.kafka_brokers.split(','),
23
+ keepAlive: goptions.kafka_keep_alive,
24
+ adminTimeout: goptions.kafka_admin_timeout,
25
+ connectionTimeout: goptions.kafka_connection_timeout,
26
+ logLevel: goptions.kafka_log_level,
27
+ useSSL: goptions.kafka_use_ssl,
28
+ requestTimeout: goptions.kafka_request_timeout
29
+ };
30
+ if (goptions.kafka_use_ssl) {
31
+ kmc.ssl = {
32
+ rejectUnauthorized: goptions.kafka_ssl_rejectUnauthorized,
33
+ cafile: goptions.kafka_ssl_cafile,
34
+ certfileFile: goptions.kafka_ssl_certfile,
35
+ keyfile: goptions.kafka_ssl_keyfile
36
+ };
37
+ }
38
+ this.#km = new kafkamanager_1.KafkaManager(kmc);
39
+ }
40
+ async #UnSubscribeKafka(subscription) {
41
+ const unsubscribeKafka = async () => {
42
+ const kafka = subscription.kafka;
43
+ if (kafka) {
44
+ // Un-Subscribe from this kafka topic
45
+ console.log(`InstrumentationSubscriber:#UnSubscribeKafka(): Unsubscribe from Kafka topc: [${kafka.kafkaTopic}] Starting`.yellow);
46
+ await kafka.kafkaConsumer?.Stop();
47
+ await kafka.kafkaConsumer?.Disconnect();
48
+ console.log(`InstrumentationSubscriber:#UnSubscribeKafka(): Unsubscribe from Kafka topc: [${kafka.kafkaTopic}] Completed.`.yellow);
49
+ }
50
+ else {
51
+ console.log(`InstrumentationSubscriber:#UnSubscribeKafka(): Kafka details do not eixsting for this topc: [${subscription.topic}].`.magenta);
52
+ }
53
+ };
54
+ return unsubscribeKafka();
55
+ }
56
+ async #SubscribeKafka(subscription) {
57
+ const kafkaSubscribe = async () => {
58
+ if (subscription.key) {
59
+ const kafkaSafeKey = `${this.#KAFKA_PREFIX}_${subscription.key}`.replace(/@/g, '_');
60
+ const kafkaGroupId = (0, uuid_1.v4)();
61
+ subscription.kafka = {
62
+ kafkaTopic: kafkaSafeKey,
63
+ kafkaGroupId: kafkaGroupId,
64
+ kafkaConsumer: this.#km.CreateConsumer(kafkaGroupId)
65
+ };
66
+ console.log(`InstrumentationSubscriber:#SubscribeKafka(): Subscribe to Kafka topc: [${subscription.kafka.kafkaTopic}] Starting`.yellow);
67
+ const kafka = subscription.kafka;
68
+ await kafka.kafkaConsumer.Connect();
69
+ await kafka.kafkaConsumer.Subscribe([kafka.kafkaTopic], false);
70
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
71
+ await kafka.kafkaConsumer.StartConsumingMessages((topic, key, partition, value, headers) => {
72
+ try {
73
+ if (key != "" && value != "") {
74
+ if (topic.localeCompare(key) === 0) {
75
+ subscription.cb({
76
+ topic: commonTypes_1.SubscriptionTopic.LogProcessing,
77
+ key: subscription.key,
78
+ data: {
79
+ kafkaTopic: kafka.kafkaTopic,
80
+ kafkaGroupId: kafka.kafkaGroupId,
81
+ partition,
82
+ topic,
83
+ key,
84
+ value
85
+ }
86
+ });
87
+ }
88
+ else {
89
+ console.log(`InstrumentationSubscriber:#SubscribeKafka(): Invalid message format from Kafka data, topic: [${topic}] and key: [${key}] do not match`.red);
90
+ }
91
+ }
92
+ else {
93
+ console.log(`InstrumentationSubscriber:#SubscribeKafka(): Invalid message format from Kafka data, topic: [${topic}], key: [${key}], value: [${value}]`.red);
94
+ }
95
+ }
96
+ catch (error) {
97
+ console.log(`InstrumentationSubscriber:#SubscribeKafka(): Error: [${error}]`.red);
98
+ }
99
+ });
100
+ console.log(`InstrumentationSubscriber:#SubscribeKafka(): Subscribe to Kafka topc: [${kafka.kafkaTopic}] Completed`.yellow);
101
+ }
102
+ else {
103
+ console.log(`InstrumentationSubscriber:#SubscribeKafka(): Could not subscribe to Kafka topc, missing key for topic: [${subscription.topic}]`.magenta);
104
+ }
105
+ };
106
+ return kafkaSubscribe();
107
+ }
108
+ async #OutputSubscription(subscription) {
109
+ // Service subscriptions
110
+ const { topic, key, subkey } = subscription;
111
+ switch (topic) {
112
+ // Services -------------------------------------------------------------------------------------------
113
+ case commonTypes_1.SubscriptionTopic.AllServicesCombined:
114
+ return this.#influxDBManager.serviceManager.GetInfluxDBResultsRootService();
115
+ case commonTypes_1.SubscriptionTopic.Services:
116
+ return this.#influxDBManager.serviceManager.GetInfluxDBResultsService();
117
+ case commonTypes_1.SubscriptionTopic.ServiceInstances:
118
+ if (key) { // key = service_id
119
+ return this.#influxDBManager.serviceManager.GetInfluxDBResultsServiceInstances(key);
120
+ }
121
+ else {
122
+ throw new Error(`#OutputSubscription: key not provided for subscription: [${topic}]`);
123
+ }
124
+ case commonTypes_1.SubscriptionTopic.ServiceInstance:
125
+ if (key) { // key = service instance id
126
+ return this.#influxDBManager.serviceManager.GetInfluxDBResultsServiceInstance(key);
127
+ }
128
+ else {
129
+ throw new Error(`#OutputSubscription: key not provided for subscription: [${topic}]`);
130
+ }
131
+ // User Agents -------------------------------------------------------------------------------------------
132
+ case commonTypes_1.SubscriptionTopic.AllAgentsCombined:
133
+ return this.#influxDBManager.agentManager.GetInfluxDBResultsRootAgent();
134
+ case commonTypes_1.SubscriptionTopic.Agents:
135
+ return this.#influxDBManager.agentManager.GetInfluxDBResultsAgent();
136
+ case commonTypes_1.SubscriptionTopic.AgentWorkers:
137
+ if (key) { // key = agent instance id
138
+ return this.#influxDBManager.agentManager.GetInfluxDBResultsAgentThreads(key);
139
+ }
140
+ else {
141
+ throw new Error(`#OutputSubscription: key not provided for subscription: [${topic}]`);
142
+ }
143
+ case commonTypes_1.SubscriptionTopic.AgentWorker:
144
+ if (key && subkey) { // key = agent instance id, subkey = agent instance worker id
145
+ return this.#influxDBManager.agentManager.GetInfluxDBResultsAgentThread(key, subkey);
146
+ }
147
+ else {
148
+ throw new Error(`#OutputSubscription: key and/or subkey not provided for subscription: [${topic}]`);
149
+ }
150
+ // Logging (both service and agent) -----------------------------------------------------------------------
151
+ case commonTypes_1.SubscriptionTopic.LogProcessing:
152
+ // Kafka processing will be handled by a dedicated kafka run consumer
153
+ return { topic, data: null };
154
+ default:
155
+ return { topic, data: null };
156
+ }
157
+ }
158
+ #ExecuteSubscriptions = async () => {
159
+ const promArray = [];
160
+ const indexArray = [];
161
+ for (const [, subscription] of Object.entries(this.#subscriptions)) {
162
+ promArray.push(this.#OutputSubscription(subscription));
163
+ indexArray.push(subscription);
164
+ }
165
+ const retVal = await Promise.all(promArray);
166
+ try {
167
+ for (let i = 0; i < retVal.length; i++) {
168
+ indexArray[i].cb(retVal[i]);
169
+ }
170
+ }
171
+ catch (error) {
172
+ console.log(`${error} -- ${JSON.stringify(retVal)}`.red);
173
+ }
174
+ };
175
+ Subscribe(subscriptions) {
176
+ if (this.#interval) {
177
+ clearInterval(this.#interval);
178
+ }
179
+ for (let i = 0; i < subscriptions.length; i++) {
180
+ if (this.#subscriptions[subscriptions[i].topic]) {
181
+ // Subscription already exists - ignoring
182
+ console.log(`InstrumentationSubscriber.Subscribe(): Subscription already exists - ignoring.`.magenta);
183
+ }
184
+ else {
185
+ this.#subscriptions[subscriptions[i].topic] = subscriptions[i];
186
+ console.log(`InstrumentationSubscriber.Subscribe(): topicKey: [${subscriptions[i].topic}] successfully subscribed.`.grey);
187
+ if (subscriptions[i].topic.localeCompare(commonTypes_1.SubscriptionTopic.LogProcessing) === 0) {
188
+ this.#SubscribeKafka(subscriptions[i]);
189
+ }
190
+ }
191
+ }
192
+ if (!this.#interval) {
193
+ // Output subscription immediately and then iterate ...
194
+ this.#ExecuteSubscriptions();
195
+ let interval = 1000; // default
196
+ if (this.options) {
197
+ interval = this.options.SubscriptionInterval;
198
+ }
199
+ this.#interval = setInterval(() => {
200
+ this.#ExecuteSubscriptions();
201
+ }, interval);
202
+ }
203
+ }
204
+ UnSubscribe(subscriptions) {
205
+ for (let i = 0; i < subscriptions.length; i++) {
206
+ if (subscriptions[i].topic.localeCompare(commonTypes_1.SubscriptionTopic.LogProcessing) === 0) {
207
+ this.#UnSubscribeKafka(subscriptions[i]);
208
+ }
209
+ delete this.#subscriptions[subscriptions[i].topic];
210
+ }
211
+ if (Object.keys(this.#subscriptions).length === 0) {
212
+ if (this.#interval) {
213
+ clearInterval(this.#interval);
214
+ this.#interval = null;
215
+ }
216
+ }
217
+ }
218
+ #UnSubscribeAll() {
219
+ if (this.#interval) {
220
+ clearInterval(this.#interval);
221
+ this.#interval = null;
222
+ }
223
+ for (const [, subscription] of Object.entries(this.#subscriptions)) {
224
+ if (subscription.topic.localeCompare(commonTypes_1.SubscriptionTopic.LogProcessing) === 0) {
225
+ this.#UnSubscribeKafka(subscription);
226
+ }
227
+ }
228
+ this.#subscriptions = {};
229
+ }
230
+ async Start() {
231
+ return this.#influxDBManager.Start();
232
+ }
233
+ async Stop() {
234
+ this.#UnSubscribeAll();
235
+ return this.#influxDBManager.Terminate();
236
+ }
237
+ }
238
+ exports.InstrumentationSubscriber = InstrumentationSubscriber;
239
+ //# sourceMappingURL=instrumentationsubscriber.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"instrumentationsubscriber.js","sourceRoot":"","sources":["../src/instrumentationsubscriber.ts"],"names":[],"mappings":";;;AAAA,+CAAsG;AACtG,uDAAwE;AACxE,gEAA4D;AAE5D,+BAAoC;AAEpC,kDAA6C;AAC7C,gDAAmD;AACnD,MAAM,QAAQ,GAAG,IAAA,oBAAQ,GAAE,CAAA;AAM3B,MAAa,yBAA0B,SAAQ,yBAAc;IACzD,cAAc,GAAkC,EAAG,CAAC;IACpD,gBAAgB,CAAkB;IAClC,SAAS,GAAwB,IAAI,CAAC;IACtC,GAAG,CAAe;IAClB,aAAa,GAAW,SAAS,CAAC;IAElC,YAAY,OAA0C;QAClD,KAAK,CAAC,OAAO,CAAC,CAAC;QAEf,IAAI,CAAC,gBAAgB,GAAG,IAAI,iCAAe,EAAE,CAAC;QAE9C,MAAM,GAAG,GAAwB;YAC7B,QAAQ,EAAE,QAAQ,CAAC,cAAc;YACjC,OAAO,EAAE,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC;YAC1C,SAAS,EAAE,QAAQ,CAAC,gBAAgB;YACpC,YAAY,EAAE,QAAQ,CAAC,mBAAmB;YAC1C,iBAAiB,EAAE,QAAQ,CAAC,wBAAwB;YACpD,QAAQ,EAAE,QAAQ,CAAC,eAAe;YAClC,MAAM,EAAE,QAAQ,CAAC,aAAa;YAC9B,cAAc,EAAE,QAAQ,CAAC,qBAAqB;SACjD,CAAA;QACD,IAAI,QAAQ,CAAC,aAAa,EAAE;YACxB,GAAG,CAAC,GAAG,GAAG;gBACN,kBAAkB,EAAE,QAAQ,CAAC,4BAA4B;gBACzD,MAAM,EAAE,QAAQ,CAAC,gBAAgB;gBACjC,YAAY,EAAE,QAAQ,CAAC,kBAAkB;gBACzC,OAAO,EAAE,QAAQ,CAAC,iBAAiB;aACtC,CAAA;SACJ;QACD,IAAI,CAAC,GAAG,GAAG,IAAI,2BAAY,CAAC,GAAG,CAAC,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,YAA2B;QAC/C,MAAM,gBAAgB,GAAG,KAAK,IAAI,EAAE;YAChC,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC;YACjC,IAAI,KAAK,EAAE;gBACP,qCAAqC;gBACrC,OAAO,CAAC,GAAG,CAAC,gFAAgF,KAAK,CAAC,UAAU,YAAY,CAAC,MAAM,CAAC,CAAA;gBAChI,MAAM,KAAK,CAAC,aAAa,EAAE,IAAI,EAAE,CAAC;gBAClC,MAAM,KAAK,CAAC,aAAa,EAAE,UAAU,EAAE,CAAC;gBACxC,OAAO,CAAC,GAAG,CAAC,gFAAgF,KAAK,CAAC,UAAU,cAAc,CAAC,MAAM,CAAC,CAAA;aACrI;iBAAM;gBACH,OAAO,CAAC,GAAG,CAAC,gGAAgG,YAAY,CAAC,KAAK,IAAI,CAAC,OAAO,CAAC,CAAA;aAC9I;QACL,CAAC,CAAC;QACF,OAAO,gBAAgB,EAAE,CAAC;IAC9B,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,YAA2B;QAC7C,MAAM,cAAc,GAAG,KAAK,IAAI,EAAE;YAC9B,IAAI,YAAY,CAAC,GAAG,EAAE;gBAClB,MAAM,YAAY,GAAG,GAAG,IAAI,CAAC,aAAa,IAAI,YAAY,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBACpF,MAAM,YAAY,GAAG,IAAA,SAAM,GAAE,CAAC;gBAC9B,YAAY,CAAC,KAAK,GAAG;oBACjB,UAAU,EAAE,YAAY;oBACxB,YAAY,EAAE,YAAY;oBAC1B,aAAa,EAAE,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,YAAY,CAAC;iBACvD,CAAC;gBACF,OAAO,CAAC,GAAG,CAAC,0EAA0E,YAAY,CAAC,KAAK,CAAC,UAAU,YAAY,CAAC,MAAM,CAAC,CAAA;gBAEvI,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC;gBACjC,MAAM,KAAK,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC;gBACpC,MAAM,KAAK,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,KAAK,CAAE,CAAC;gBAChE,8DAA8D;gBAC9D,MAAM,KAAK,CAAC,aAAa,CAAC,sBAAsB,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;oBACvF,IAAI;wBACA,IAAI,GAAG,IAAI,EAAE,IAAI,KAAK,IAAI,EAAE,EAAE;4BAC1B,IAAI,KAAK,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;gCAChC,YAAY,CAAC,EAAE,CAAC;oCACZ,KAAK,EAAE,+BAAiB,CAAC,aAAa;oCACtC,GAAG,EAAE,YAAY,CAAC,GAAG;oCACrB,IAAI,EAAE;wCACF,UAAU,EAAE,KAAK,CAAC,UAAU;wCAC5B,YAAY,EAAE,KAAK,CAAC,YAAY;wCAChC,SAAS;wCACT,KAAK;wCACL,GAAG;wCACH,KAAK;qCACR;iCACJ,CAAC,CAAC;6BACN;iCAAM;gCACH,OAAO,CAAC,GAAG,CAAC,gGAAgG,KAAK,eAAe,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;6BAC5J;yBACJ;6BAAM;4BACH,OAAO,CAAC,GAAG,CAAC,gGAAgG,KAAK,YAAY,GAAG,cAAc,KAAK,GAAG,CAAC,GAAG,CAAC,CAAC;yBAC/J;qBACJ;oBAAC,OAAO,KAAK,EAAE;wBACZ,OAAO,CAAC,GAAG,CAAC,wDAAwD,KAAK,GAAG,CAAC,GAAG,CAAC,CAAC;qBACrF;gBACL,CAAC,CAAC,CAAC;gBACH,OAAO,CAAC,GAAG,CAAC,0EAA0E,KAAK,CAAC,UAAU,aAAa,CAAC,MAAM,CAAC,CAAA;aAC9H;iBAAM;gBACH,OAAO,CAAC,GAAG,CAAC,2GAA2G,YAAY,CAAC,KAAK,GAAG,CAAC,OAAO,CAAC,CAAC;aACzJ;QACL,CAAC,CAAA;QACD,OAAO,cAAc,EAAE,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,YAA2B;QACjD,wBAAwB;QACxB,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAI,YAAY,CAAC;QAC7C,QAAQ,KAAK,EAAE;YACf,uGAAuG;YACvG,KAAK,+BAAiB,CAAC,mBAAmB;gBACtC,OAAO,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC,6BAA6B,EAAE,CAAC;YAChF,KAAK,+BAAiB,CAAC,QAAQ;gBAC3B,OAAO,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC,yBAAyB,EAAE,CAAC;YAC5E,KAAK,+BAAiB,CAAC,gBAAgB;gBACnC,IAAI,GAAG,EAAE,EAAE,mBAAmB;oBAC1B,OAAO,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC,kCAAkC,CAAC,GAAG,CAAC,CAAC;iBACvF;qBAAM;oBACH,MAAM,IAAI,KAAK,CAAC,4DAA4D,KAAK,GAAG,CAAC,CAAC;iBACzF;YACL,KAAK,+BAAiB,CAAC,eAAe;gBAClC,IAAI,GAAG,EAAE,EAAE,4BAA4B;oBACnC,OAAO,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC,iCAAiC,CAAC,GAAG,CAAC,CAAC;iBACtF;qBAAM;oBACH,MAAM,IAAI,KAAK,CAAC,4DAA4D,KAAK,GAAG,CAAC,CAAC;iBACzF;YAED,0GAA0G;YAC9G,KAAK,+BAAiB,CAAC,iBAAiB;gBACpC,OAAO,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,2BAA2B,EAAE,CAAC;YAC5E,KAAM,+BAAiB,CAAC,MAAM;gBAC1B,OAAO,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,uBAAuB,EAAE,CAAC;YACxE,KAAM,+BAAiB,CAAC,YAAY;gBAChC,IAAI,GAAG,EAAE,EAAE,0BAA0B;oBACjC,OAAO,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,8BAA8B,CAAC,GAAG,CAAC,CAAC;iBACjF;qBAAM;oBACH,MAAM,IAAI,KAAK,CAAC,4DAA4D,KAAK,GAAG,CAAC,CAAC;iBACzF;YACL,KAAK,+BAAiB,CAAC,WAAW;gBAC9B,IAAI,GAAG,IAAI,MAAM,EAAE,EAAE,6DAA6D;oBAC9E,OAAO,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,6BAA6B,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;iBACxF;qBAAM;oBACH,MAAM,IAAI,KAAK,CAAC,0EAA0E,KAAK,GAAG,CAAC,CAAC;iBACvG;YAED,2GAA2G;YAC/G,KAAK,+BAAiB,CAAC,aAAa;gBAChC,qEAAqE;gBACrE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YACjC;gBACI,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;SAChC;IACL,CAAC;IAED,qBAAqB,GAAG,KAAK,IAAI,EAAE;QAC/B,MAAM,SAAS,GAAoC,EAAG,CAAC;QACvD,MAAM,UAAU,GAAmB,EAAG,CAAC;QAEvC,KAAK,MAAM,CAAC,EAAE,YAAY,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE;YAChE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,YAAY,CAAC,CAAC,CAAC;YACvD,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;SACjC;QACD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC5C,IAAI;YACA,KAAK,IAAI,CAAC,GAAC,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBAClC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;aAC/B;SACJ;QAAC,OAAO,KAAK,EAAE;YACZ,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;SAC5D;IACL,CAAC,CAAA;IAED,SAAS,CAAC,aAA6B;QACnC,IAAI,IAAI,CAAC,SAAS,EAAE;YAChB,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;SACjC;QACD,KAAK,IAAI,CAAC,GAAC,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACzC,IAAI,IAAI,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE;gBAC7C,yCAAyC;gBACzC,OAAO,CAAC,GAAG,CAAC,gFAAgF,CAAC,OAAO,CAAC,CAAC;aACzG;iBAAM;gBACH,IAAI,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;gBAC/D,OAAO,CAAC,GAAG,CAAC,qDAAqD,aAAa,CAAC,CAAC,CAAC,CAAC,KAAK,4BAA4B,CAAC,IAAI,CAAC,CAAC;gBAC1H,IAAI,aAAa,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,+BAAiB,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE;oBAC7E,IAAI,CAAC,eAAe,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;iBAC1C;aACJ;SACJ;QAED,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE;YACjB,uDAAuD;YACvD,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAC7B,IAAI,QAAQ,GAAG,IAAI,CAAC,CAAC,UAAU;YAC/B,IAAI,IAAI,CAAC,OAAO,EAAE;gBACd,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAA;aAC/C;YACD,IAAI,CAAC,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE;gBAC9B,IAAI,CAAC,qBAAqB,EAAE,CAAC;YACjC,CAAC,EAAE,QAAQ,CAAC,CAAC;SAChB;IACL,CAAC;IAED,WAAW,CAAC,aAA6B;QACrC,KAAK,IAAI,CAAC,GAAC,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACzC,IAAI,aAAa,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,+BAAiB,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE;gBAC7E,IAAI,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;aAC5C;YACD,OAAO,IAAI,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;SACtD;QACD,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE;YAC/C,IAAI,IAAI,CAAC,SAAS,EAAE;gBAChB,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBAC9B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;aACzB;SACJ;IACL,CAAC;IAED,eAAe;QACX,IAAI,IAAI,CAAC,SAAS,EAAE;YAChB,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC9B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;SACzB;QACD,KAAK,MAAM,CAAC,EAAE,YAAY,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE;YAChE,IAAI,YAAY,CAAC,KAAK,CAAC,aAAa,CAAC,+BAAiB,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE;gBACzE,IAAI,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;aACxC;SACJ;QAED,IAAI,CAAC,cAAc,GAAG,EAAG,CAAC;IAC9B,CAAC;IAED,KAAK,CAAC,KAAK;QACP,OAAO,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC;IACzC,CAAC;IAED,KAAK,CAAC,IAAI;QACN,IAAI,CAAC,eAAe,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC;IAC7C,CAAC;CACJ;AAzOD,8DAyOC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@nsshunt/stsappframework",
3
- "version": "3.0.20",
3
+ "version": "3.0.21",
4
4
  "description": "",
5
5
  "main": "dist/index.js",
6
6
  "types": "./types/index.d.ts",
@@ -0,0 +1,248 @@
1
+ import { ISubscriptions, ISubscription, ISubscriptionPayload, SubscriptionTopic } from './commonTypes'
2
+ import { KafkaManager, IKafkaManagerConfig } from './kafka/kafkamanager'
3
+ import { InfluxDBManager } from './influxdb/influxDBManager'
4
+
5
+ import { v4 as uuidv4 } from 'uuid';
6
+
7
+ import { $Options } from '@nsshunt/stsconfig'
8
+ import { STSOptionsBase } from '@nsshunt/stsutils';
9
+ const goptions = $Options()
10
+
11
+ export interface IInstrumentationSubscriberOptions {
12
+ SubscriptionInterval: number
13
+ }
14
+
15
+ export class InstrumentationSubscriber extends STSOptionsBase {
16
+ #subscriptions: Record<string, ISubscription> = { };
17
+ #influxDBManager: InfluxDBManager;
18
+ #interval: NodeJS.Timer | null = null;
19
+ #km: KafkaManager;
20
+ #KAFKA_PREFIX: string = '__STS__';
21
+
22
+ constructor(options: IInstrumentationSubscriberOptions) {
23
+ super(options);
24
+
25
+ this.#influxDBManager = new InfluxDBManager();
26
+
27
+ const kmc: IKafkaManagerConfig = {
28
+ clientId: goptions.kafka_clientId,
29
+ brokers: goptions.kafka_brokers.split(','),
30
+ keepAlive: goptions.kafka_keep_alive,
31
+ adminTimeout: goptions.kafka_admin_timeout,
32
+ connectionTimeout: goptions.kafka_connection_timeout,
33
+ logLevel: goptions.kafka_log_level,
34
+ useSSL: goptions.kafka_use_ssl,
35
+ requestTimeout: goptions.kafka_request_timeout
36
+ }
37
+ if (goptions.kafka_use_ssl) {
38
+ kmc.ssl = {
39
+ rejectUnauthorized: goptions.kafka_ssl_rejectUnauthorized,
40
+ cafile: goptions.kafka_ssl_cafile,
41
+ certfileFile: goptions.kafka_ssl_certfile,
42
+ keyfile: goptions.kafka_ssl_keyfile
43
+ }
44
+ }
45
+ this.#km = new KafkaManager(kmc);
46
+ }
47
+
48
+ async #UnSubscribeKafka(subscription: ISubscription): Promise<void> {
49
+ const unsubscribeKafka = async () => {
50
+ const kafka = subscription.kafka;
51
+ if (kafka) {
52
+ // Un-Subscribe from this kafka topic
53
+ console.log(`InstrumentationSubscriber:#UnSubscribeKafka(): Unsubscribe from Kafka topc: [${kafka.kafkaTopic}] Starting`.yellow)
54
+ await kafka.kafkaConsumer?.Stop();
55
+ await kafka.kafkaConsumer?.Disconnect();
56
+ console.log(`InstrumentationSubscriber:#UnSubscribeKafka(): Unsubscribe from Kafka topc: [${kafka.kafkaTopic}] Completed.`.yellow)
57
+ } else {
58
+ console.log(`InstrumentationSubscriber:#UnSubscribeKafka(): Kafka details do not eixsting for this topc: [${subscription.topic}].`.magenta)
59
+ }
60
+ };
61
+ return unsubscribeKafka();
62
+ }
63
+
64
+ async #SubscribeKafka(subscription: ISubscription): Promise<void> {
65
+ const kafkaSubscribe = async () => {
66
+ if (subscription.key) {
67
+ const kafkaSafeKey = `${this.#KAFKA_PREFIX}_${subscription.key}`.replace(/@/g, '_');
68
+ const kafkaGroupId = uuidv4();
69
+ subscription.kafka = {
70
+ kafkaTopic: kafkaSafeKey,
71
+ kafkaGroupId: kafkaGroupId,
72
+ kafkaConsumer: this.#km.CreateConsumer(kafkaGroupId)
73
+ };
74
+ console.log(`InstrumentationSubscriber:#SubscribeKafka(): Subscribe to Kafka topc: [${subscription.kafka.kafkaTopic}] Starting`.yellow)
75
+
76
+ const kafka = subscription.kafka;
77
+ await kafka.kafkaConsumer.Connect();
78
+ await kafka.kafkaConsumer.Subscribe([kafka.kafkaTopic], false );
79
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
80
+ await kafka.kafkaConsumer.StartConsumingMessages((topic, key, partition, value, headers) => {
81
+ try {
82
+ if (key != "" && value != "") {
83
+ if (topic.localeCompare(key) === 0) {
84
+ subscription.cb({
85
+ topic: SubscriptionTopic.LogProcessing,
86
+ key: subscription.key, // Original requested topic (may be kafka un-safe)
87
+ data: {
88
+ kafkaTopic: kafka.kafkaTopic, // Safe kafka topic key with prefix. kafkaTopic, topic and key should be same value
89
+ kafkaGroupId: kafka.kafkaGroupId,
90
+ partition,
91
+ topic, // kafkaTopic, topic and key should be same value
92
+ key, // kafkaTopic, topic and key should be same value
93
+ value
94
+ }
95
+ });
96
+ } else {
97
+ console.log(`InstrumentationSubscriber:#SubscribeKafka(): Invalid message format from Kafka data, topic: [${topic}] and key: [${key}] do not match`.red);
98
+ }
99
+ } else {
100
+ console.log(`InstrumentationSubscriber:#SubscribeKafka(): Invalid message format from Kafka data, topic: [${topic}], key: [${key}], value: [${value}]`.red);
101
+ }
102
+ } catch (error) {
103
+ console.log(`InstrumentationSubscriber:#SubscribeKafka(): Error: [${error}]`.red);
104
+ }
105
+ });
106
+ console.log(`InstrumentationSubscriber:#SubscribeKafka(): Subscribe to Kafka topc: [${kafka.kafkaTopic}] Completed`.yellow)
107
+ } else {
108
+ console.log(`InstrumentationSubscriber:#SubscribeKafka(): Could not subscribe to Kafka topc, missing key for topic: [${subscription.topic}]`.magenta);
109
+ }
110
+ }
111
+ return kafkaSubscribe();
112
+ }
113
+
114
+ async #OutputSubscription(subscription: ISubscription): Promise<ISubscriptionPayload> {
115
+ // Service subscriptions
116
+ const { topic, key, subkey } = subscription;
117
+ switch (topic) {
118
+ // Services -------------------------------------------------------------------------------------------
119
+ case SubscriptionTopic.AllServicesCombined :
120
+ return this.#influxDBManager.serviceManager.GetInfluxDBResultsRootService();
121
+ case SubscriptionTopic.Services :
122
+ return this.#influxDBManager.serviceManager.GetInfluxDBResultsService();
123
+ case SubscriptionTopic.ServiceInstances :
124
+ if (key) { // key = service_id
125
+ return this.#influxDBManager.serviceManager.GetInfluxDBResultsServiceInstances(key);
126
+ } else {
127
+ throw new Error(`#OutputSubscription: key not provided for subscription: [${topic}]`);
128
+ }
129
+ case SubscriptionTopic.ServiceInstance :
130
+ if (key) { // key = service instance id
131
+ return this.#influxDBManager.serviceManager.GetInfluxDBResultsServiceInstance(key);
132
+ } else {
133
+ throw new Error(`#OutputSubscription: key not provided for subscription: [${topic}]`);
134
+ }
135
+
136
+ // User Agents -------------------------------------------------------------------------------------------
137
+ case SubscriptionTopic.AllAgentsCombined :
138
+ return this.#influxDBManager.agentManager.GetInfluxDBResultsRootAgent();
139
+ case SubscriptionTopic.Agents :
140
+ return this.#influxDBManager.agentManager.GetInfluxDBResultsAgent();
141
+ case SubscriptionTopic.AgentWorkers :
142
+ if (key) { // key = agent instance id
143
+ return this.#influxDBManager.agentManager.GetInfluxDBResultsAgentThreads(key);
144
+ } else {
145
+ throw new Error(`#OutputSubscription: key not provided for subscription: [${topic}]`);
146
+ }
147
+ case SubscriptionTopic.AgentWorker :
148
+ if (key && subkey) { // key = agent instance id, subkey = agent instance worker id
149
+ return this.#influxDBManager.agentManager.GetInfluxDBResultsAgentThread(key, subkey);
150
+ } else {
151
+ throw new Error(`#OutputSubscription: key and/or subkey not provided for subscription: [${topic}]`);
152
+ }
153
+
154
+ // Logging (both service and agent) -----------------------------------------------------------------------
155
+ case SubscriptionTopic.LogProcessing :
156
+ // Kafka processing will be handled by a dedicated kafka run consumer
157
+ return { topic, data: null };
158
+ default :
159
+ return { topic, data: null };
160
+ }
161
+ }
162
+
163
+ #ExecuteSubscriptions = async () => {
164
+ const promArray: Promise<ISubscriptionPayload>[] = [ ];
165
+ const indexArray: ISubscriptions = [ ];
166
+
167
+ for (const [, subscription] of Object.entries(this.#subscriptions)) {
168
+ promArray.push(this.#OutputSubscription(subscription));
169
+ indexArray.push(subscription);
170
+ }
171
+ const retVal = await Promise.all(promArray);
172
+ try {
173
+ for (let i=0; i < retVal.length; i++) {
174
+ indexArray[i].cb(retVal[i]);
175
+ }
176
+ } catch (error) {
177
+ console.log(`${error} -- ${JSON.stringify(retVal)}`.red);
178
+ }
179
+ }
180
+
181
+ Subscribe(subscriptions: ISubscriptions) {
182
+ if (this.#interval) {
183
+ clearInterval(this.#interval);
184
+ }
185
+ for (let i=0; i < subscriptions.length; i++) {
186
+ if (this.#subscriptions[subscriptions[i].topic]) {
187
+ // Subscription already exists - ignoring
188
+ console.log(`InstrumentationSubscriber.Subscribe(): Subscription already exists - ignoring.`.magenta);
189
+ } else {
190
+ this.#subscriptions[subscriptions[i].topic] = subscriptions[i];
191
+ console.log(`InstrumentationSubscriber.Subscribe(): topicKey: [${subscriptions[i].topic}] successfully subscribed.`.grey);
192
+ if (subscriptions[i].topic.localeCompare(SubscriptionTopic.LogProcessing) === 0) {
193
+ this.#SubscribeKafka(subscriptions[i]);
194
+ }
195
+ }
196
+ }
197
+
198
+ if (!this.#interval) {
199
+ // Output subscription immediately and then iterate ...
200
+ this.#ExecuteSubscriptions();
201
+ let interval = 1000; // default
202
+ if (this.options) {
203
+ interval = this.options.SubscriptionInterval
204
+ }
205
+ this.#interval = setInterval(() => {
206
+ this.#ExecuteSubscriptions();
207
+ }, interval);
208
+ }
209
+ }
210
+
211
+ UnSubscribe(subscriptions: ISubscriptions) {
212
+ for (let i=0; i < subscriptions.length; i++) {
213
+ if (subscriptions[i].topic.localeCompare(SubscriptionTopic.LogProcessing) === 0) {
214
+ this.#UnSubscribeKafka(subscriptions[i]);
215
+ }
216
+ delete this.#subscriptions[subscriptions[i].topic];
217
+ }
218
+ if (Object.keys(this.#subscriptions).length === 0) {
219
+ if (this.#interval) {
220
+ clearInterval(this.#interval);
221
+ this.#interval = null;
222
+ }
223
+ }
224
+ }
225
+
226
+ #UnSubscribeAll() {
227
+ if (this.#interval) {
228
+ clearInterval(this.#interval);
229
+ this.#interval = null;
230
+ }
231
+ for (const [, subscription] of Object.entries(this.#subscriptions)) {
232
+ if (subscription.topic.localeCompare(SubscriptionTopic.LogProcessing) === 0) {
233
+ this.#UnSubscribeKafka(subscription);
234
+ }
235
+ }
236
+
237
+ this.#subscriptions = { };
238
+ }
239
+
240
+ async Start() {
241
+ return this.#influxDBManager.Start();
242
+ }
243
+
244
+ async Stop() {
245
+ this.#UnSubscribeAll();
246
+ return this.#influxDBManager.Terminate();
247
+ }
248
+ }
@@ -0,0 +1,14 @@
1
+ import { ISubscriptions } from './commonTypes';
2
+ import { STSOptionsBase } from '@nsshunt/stsutils';
3
+ export interface IInstrumentationSubscriberOptions {
4
+ SubscriptionInterval: number;
5
+ }
6
+ export declare class InstrumentationSubscriber extends STSOptionsBase {
7
+ #private;
8
+ constructor(options: IInstrumentationSubscriberOptions);
9
+ Subscribe(subscriptions: ISubscriptions): void;
10
+ UnSubscribe(subscriptions: ISubscriptions): void;
11
+ Start(): Promise<void>;
12
+ Stop(): Promise<void>;
13
+ }
14
+ //# sourceMappingURL=instrumentationsubscriber.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"instrumentationsubscriber.d.ts","sourceRoot":"","sources":["../src/instrumentationsubscriber.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAA0D,MAAM,eAAe,CAAA;AAOtG,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAGnD,MAAM,WAAW,iCAAiC;IAC9C,oBAAoB,EAAE,MAAM,CAAA;CAC/B;AAED,qBAAa,yBAA0B,SAAQ,cAAc;;gBAO7C,OAAO,EAAE,iCAAiC;IA+JtD,SAAS,CAAC,aAAa,EAAE,cAAc;IA8BvC,WAAW,CAAC,aAAa,EAAE,cAAc;IA6BnC,KAAK;IAIL,IAAI;CAIb"}