@nsshunt/stsappframework 3.0.20 → 3.0.22

package/dist/index.js

@@ -35,4 +35,5 @@ __exportStar(require("./influxdb/influxDBManagerBase"), exports);
 __exportStar(require("./influxdb/influxDBManagerService"), exports);
 __exportStar(require("./influxdb/influxDBManagerAgent"), exports);
 __exportStar(require("./publishertransports/publishTransportUtils"), exports);
+__exportStar(require("./instrumentationsubscriber"), exports);
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,gDAA6B;AAC7B,mDAAgC;AAChC,6CAA0B;AAC1B,kDAA+B;AAC/B,kDAA+B;AAC/B,sDAAmC;AACnC,sDAAmC;AACnC,sDAAmC;AACnC,sDAAmC;AACnC,yDAAsC;AACtC,yDAAsC;AACtC,oDAAiC;AACjC,uDAAoC;AACpC,wDAAqC;AACrC,wDAAqC;AACrC,yDAAsC;AACtC,6DAA0C;AAC1C,iEAA8C;AAC9C,oEAAiD;AACjD,kEAA+C;AAC/C,8EAA2D"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,gDAA6B;AAC7B,mDAAgC;AAChC,6CAA0B;AAC1B,kDAA+B;AAC/B,kDAA+B;AAC/B,sDAAmC;AACnC,sDAAmC;AACnC,sDAAmC;AACnC,sDAAmC;AACnC,yDAAsC;AACtC,yDAAsC;AACtC,oDAAiC;AACjC,uDAAoC;AACpC,wDAAqC;AACrC,wDAAqC;AACrC,yDAAsC;AACtC,6DAA0C;AAC1C,iEAA8C;AAC9C,oEAAiD;AACjD,kEAA+C;AAC/C,8EAA2D;AAC3D,8DAA2C"}

package/dist/instrumentationsubscriber.js

@@ -0,0 +1,239 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.InstrumentationSubscriber = void 0;
+const commonTypes_1 = require("./commonTypes");
+const kafkamanager_1 = require("./kafka/kafkamanager");
+const influxDBManager_1 = require("./influxdb/influxDBManager");
+const uuid_1 = require("uuid");
+const stsconfig_1 = require("@nsshunt/stsconfig");
+const stsutils_1 = require("@nsshunt/stsutils");
+const goptions = (0, stsconfig_1.$Options)();
+class InstrumentationSubscriber extends stsutils_1.STSOptionsBase {
+    #subscriptions = {};
+    #influxDBManager;
+    #interval = null;
+    #km;
+    #KAFKA_PREFIX = '__STS__';
+    constructor(options) {
+        super(options);
+        this.#influxDBManager = new influxDBManager_1.InfluxDBManager();
+        const kmc = {
+            clientId: goptions.kafka_clientId,
+            brokers: goptions.kafka_brokers.split(','),
+            keepAlive: goptions.kafka_keep_alive,
+            adminTimeout: goptions.kafka_admin_timeout,
+            connectionTimeout: goptions.kafka_connection_timeout,
+            logLevel: goptions.kafka_log_level,
+            useSSL: goptions.kafka_use_ssl,
+            requestTimeout: goptions.kafka_request_timeout
+        };
+        if (goptions.kafka_use_ssl) {
+            kmc.ssl = {
+                rejectUnauthorized: goptions.kafka_ssl_rejectUnauthorized,
+                cafile: goptions.kafka_ssl_cafile,
+                certfileFile: goptions.kafka_ssl_certfile,
+                keyfile: goptions.kafka_ssl_keyfile
+            };
+        }
+        this.#km = new kafkamanager_1.KafkaManager(kmc);
+    }
+    async #UnSubscribeKafka(subscription) {
+        const unsubscribeKafka = async () => {
+            const kafka = subscription.kafka;
+            if (kafka) {
+                // Un-Subscribe from this kafka topic
+                console.log(`InstrumentationSubscriber:#UnSubscribeKafka(): Unsubscribe from Kafka topc: [${kafka.kafkaTopic}] Starting`.yellow);
+                await kafka.kafkaConsumer?.Stop();
+                await kafka.kafkaConsumer?.Disconnect();
+                console.log(`InstrumentationSubscriber:#UnSubscribeKafka(): Unsubscribe from Kafka topc: [${kafka.kafkaTopic}] Completed.`.yellow);
+            }
+            else {
+                console.log(`InstrumentationSubscriber:#UnSubscribeKafka(): Kafka details do not eixsting for this topc: [${subscription.topic}].`.magenta);
+            }
+        };
+        return unsubscribeKafka();
+    }
+    async #SubscribeKafka(subscription) {
+        const kafkaSubscribe = async () => {
+            if (subscription.key) {
+                const kafkaSafeKey = `${this.#KAFKA_PREFIX}_${subscription.key}`.replace(/@/g, '_');
+                const kafkaGroupId = (0, uuid_1.v4)();
+                subscription.kafka = {
+                    kafkaTopic: kafkaSafeKey,
+                    kafkaGroupId: kafkaGroupId,
+                    kafkaConsumer: this.#km.CreateConsumer(kafkaGroupId)
+                };
+                console.log(`InstrumentationSubscriber:#SubscribeKafka(): Subscribe to Kafka topc: [${subscription.kafka.kafkaTopic}] Starting`.yellow);
+                const kafka = subscription.kafka;
+                await kafka.kafkaConsumer.Connect();
+                await kafka.kafkaConsumer.Subscribe([kafka.kafkaTopic], false);
+                // eslint-disable-next-line  @typescript-eslint/no-unused-vars
+                await kafka.kafkaConsumer.StartConsumingMessages((topic, key, partition, value, headers) => {
+                    try {
+                        if (key != "" && value != "") {
+                            if (topic.localeCompare(key) === 0) {
+                                subscription.cb({
+                                    topic: commonTypes_1.SubscriptionTopic.LogProcessing,
+                                    key: subscription.key,
+                                    data: {
+                                        kafkaTopic: kafka.kafkaTopic,
+                                        kafkaGroupId: kafka.kafkaGroupId,
+                                        partition,
+                                        topic,
+                                        key,
+                                        value
+                                    }
+                                });
+                            }
+                            else {
+                                console.log(`InstrumentationSubscriber:#SubscribeKafka(): Invalid message format from Kafka data, topic: [${topic}] and key: [${key}] do not match`.red);
+                            }
+                        }
+                        else {
+                            console.log(`InstrumentationSubscriber:#SubscribeKafka(): Invalid message format from Kafka data, topic: [${topic}], key: [${key}], value: [${value}]`.red);
+                        }
+                    }
+                    catch (error) {
+                        console.log(`InstrumentationSubscriber:#SubscribeKafka(): Error: [${error}]`.red);
+                    }
+                });
+                console.log(`InstrumentationSubscriber:#SubscribeKafka(): Subscribe to Kafka topc: [${kafka.kafkaTopic}] Completed`.yellow);
+            }
+            else {
+                console.log(`InstrumentationSubscriber:#SubscribeKafka(): Could not subscribe to Kafka topc, missing key for topic: [${subscription.topic}]`.magenta);
+            }
+        };
+        return kafkaSubscribe();
+    }
+    async #OutputSubscription(subscription) {
+        // Service subscriptions
+        const { topic, key, subkey } = subscription;
+        switch (topic) {
+            // Services -------------------------------------------------------------------------------------------
+            case commonTypes_1.SubscriptionTopic.AllServicesCombined:
+                return this.#influxDBManager.serviceManager.GetInfluxDBResultsRootService();
+            case commonTypes_1.SubscriptionTopic.Services:
+                return this.#influxDBManager.serviceManager.GetInfluxDBResultsService();
+            case commonTypes_1.SubscriptionTopic.ServiceInstances:
+                if (key) { // key = service_id
+                    return this.#influxDBManager.serviceManager.GetInfluxDBResultsServiceInstances(key);
+                }
+                else {
+                    throw new Error(`#OutputSubscription: key not provided for subscription: [${topic}]`);
+                }
+            case commonTypes_1.SubscriptionTopic.ServiceInstance:
+                if (key) { // key = service instance id
+                    return this.#influxDBManager.serviceManager.GetInfluxDBResultsServiceInstance(key);
+                }
+                else {
+                    throw new Error(`#OutputSubscription: key not provided for subscription: [${topic}]`);
+                }
+            // User Agents -------------------------------------------------------------------------------------------
+            case commonTypes_1.SubscriptionTopic.AllAgentsCombined:
+                return this.#influxDBManager.agentManager.GetInfluxDBResultsRootAgent();
+            case commonTypes_1.SubscriptionTopic.Agents:
+                return this.#influxDBManager.agentManager.GetInfluxDBResultsAgent();
+            case commonTypes_1.SubscriptionTopic.AgentWorkers:
+                if (key) { // key = agent instance id
+                    return this.#influxDBManager.agentManager.GetInfluxDBResultsAgentThreads(key);
+                }
+                else {
+                    throw new Error(`#OutputSubscription: key not provided for subscription: [${topic}]`);
+                }
+            case commonTypes_1.SubscriptionTopic.AgentWorker:
+                if (key && subkey) { // key = agent instance id, subkey = agent instance worker id
+                    return this.#influxDBManager.agentManager.GetInfluxDBResultsAgentThread(key, subkey);
+                }
+                else {
+                    throw new Error(`#OutputSubscription: key and/or subkey not provided for subscription: [${topic}]`);
+                }
+            // Logging (both service and agent) -----------------------------------------------------------------------
+            case commonTypes_1.SubscriptionTopic.LogProcessing:
+                // Kafka processing will be handled by a dedicated kafka run consumer
+                return { topic, data: null };
+            default:
+                return { topic, data: null };
+        }
+    }
+    #ExecuteSubscriptions = async () => {
+        const promArray = [];
+        const indexArray = [];
+        for (const [, subscription] of Object.entries(this.#subscriptions)) {
+            promArray.push(this.#OutputSubscription(subscription));
+            indexArray.push(subscription);
+        }
+        const retVal = await Promise.all(promArray);
+        try {
+            for (let i = 0; i < retVal.length; i++) {
+                indexArray[i].cb(retVal[i]);
+            }
+        }
+        catch (error) {
+            console.log(`${error} -- ${JSON.stringify(retVal)}`.red);
+        }
+    };
+    Subscribe(subscriptions) {
+        if (this.#interval) {
+            clearInterval(this.#interval);
+        }
+        for (let i = 0; i < subscriptions.length; i++) {
+            if (this.#subscriptions[subscriptions[i].topic]) {
+                // Subscription already exists - ignoring
+                console.log(`InstrumentationSubscriber.Subscribe(): Subscription already exists - ignoring.`.magenta);
+            }
+            else {
+                this.#subscriptions[subscriptions[i].topic] = subscriptions[i];
+                console.log(`InstrumentationSubscriber.Subscribe(): topicKey: [${subscriptions[i].topic}] successfully subscribed.`.grey);
+                if (subscriptions[i].topic.localeCompare(commonTypes_1.SubscriptionTopic.LogProcessing) === 0) {
+                    this.#SubscribeKafka(subscriptions[i]);
+                }
+            }
+        }
+        if (!this.#interval) {
+            // Output subscription immediately and then iterate ...
+            this.#ExecuteSubscriptions();
+            let interval = 1000; // default
+            if (this.options) {
+                interval = this.options.SubscriptionInterval;
+            }
+            this.#interval = setInterval(() => {
+                this.#ExecuteSubscriptions();
+            }, interval);
+        }
+    }
+    UnSubscribe(subscriptions) {
+        for (let i = 0; i < subscriptions.length; i++) {
+            if (subscriptions[i].topic.localeCompare(commonTypes_1.SubscriptionTopic.LogProcessing) === 0) {
+                this.#UnSubscribeKafka(subscriptions[i]);
+            }
+            delete this.#subscriptions[subscriptions[i].topic];
+        }
+        if (Object.keys(this.#subscriptions).length === 0) {
+            if (this.#interval) {
+                clearInterval(this.#interval);
+                this.#interval = null;
+            }
+        }
+    }
+    #UnSubscribeAll() {
+        if (this.#interval) {
+            clearInterval(this.#interval);
+            this.#interval = null;
+        }
+        for (const [, subscription] of Object.entries(this.#subscriptions)) {
+            if (subscription.topic.localeCompare(commonTypes_1.SubscriptionTopic.LogProcessing) === 0) {
+                this.#UnSubscribeKafka(subscription);
+            }
+        }
+        this.#subscriptions = {};
+    }
+    async Start() {
+        return this.#influxDBManager.Start();
+    }
+    async Stop() {
+        this.#UnSubscribeAll();
+        return this.#influxDBManager.Terminate();
+    }
+}
+exports.InstrumentationSubscriber = InstrumentationSubscriber;
+//# sourceMappingURL=instrumentationsubscriber.js.map

package/dist/instrumentationsubscriber.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"instrumentationsubscriber.js","sourceRoot":"","sources":["../src/instrumentationsubscriber.ts"],"names":[],"mappings":";;;AAAA,+CAAsG;AACtG,uDAAwE;AACxE,gEAA4D;AAE5D,+BAAoC;AAEpC,kDAA6C;AAC7C,gDAAmD;AACnD,MAAM,QAAQ,GAAG,IAAA,oBAAQ,GAAE,CAAA;AAM3B,MAAa,yBAA0B,SAAQ,yBAAc;IACzD,cAAc,GAAkC,EAAG,CAAC;IACpD,gBAAgB,CAAkB;IAClC,SAAS,GAAwB,IAAI,CAAC;IACtC,GAAG,CAAe;IAClB,aAAa,GAAW,SAAS,CAAC;IAElC,YAAY,OAA0C;QAClD,KAAK,CAAC,OAAO,CAAC,CAAC;QAEf,IAAI,CAAC,gBAAgB,GAAG,IAAI,iCAAe,EAAE,CAAC;QAE9C,MAAM,GAAG,GAAwB;YAC7B,QAAQ,EAAE,QAAQ,CAAC,cAAc;YACjC,OAAO,EAAE,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC;YAC1C,SAAS,EAAE,QAAQ,CAAC,gBAAgB;YACpC,YAAY,EAAE,QAAQ,CAAC,mBAAmB;YAC1C,iBAAiB,EAAE,QAAQ,CAAC,wBAAwB;YACpD,QAAQ,EAAE,QAAQ,CAAC,eAAe;YAClC,MAAM,EAAE,QAAQ,CAAC,aAAa;YAC9B,cAAc,EAAE,QAAQ,CAAC,qBAAqB;SACjD,CAAA;QACD,IAAI,QAAQ,CAAC,aAAa,EAAE;YACxB,GAAG,CAAC,GAAG,GAAG;gBACN,kBAAkB,EAAE,QAAQ,CAAC,4BAA4B;gBACzD,MAAM,EAAE,QAAQ,CAAC,gBAAgB;gBACjC,YAAY,EAAE,QAAQ,CAAC,kBAAkB;gBACzC,OAAO,EAAE,QAAQ,CAAC,iBAAiB;aACtC,CAAA;SACJ;QACD,IAAI,CAAC,GAAG,GAAG,IAAI,2BAAY,CAAC,GAAG,CAAC,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,YAA2B;QAC/C,MAAM,gBAAgB,GAAG,KAAK,IAAI,EAAE;YAChC,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC;YACjC,IAAI,KAAK,EAAE;gBACP,qCAAqC;gBACrC,OAAO,CAAC,GAAG,CAAC,gFAAgF,KAAK,CAAC,UAAU,YAAY,CAAC,MAAM,CAAC,CAAA;gBAChI,MAAM,KAAK,CAAC,aAAa,EAAE,IAAI,EAAE,CAAC;gBAClC,MAAM,KAAK,CAAC,aAAa,EAAE,UAAU,EAAE,CAAC;gBACxC,OAAO,CAAC,GAAG,CAAC,gFAAgF,KAAK,CAAC,UAAU,cAAc,CAAC,MAAM,CAAC,CAAA;aACrI;iBAAM;gBACH,OAAO,CAAC,GAAG,CAAC,gGAAgG,YAAY,CAAC,KAAK,IAAI,CAAC,OAAO,CAAC,CAAA;aAC9I;QACL,CAAC,CAAC;QACF,OAAO,gBAAgB,EAAE,CAAC;IAC9B,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,YAA2B;QAC7C,MAAM,cAAc,GAAG,KAAK,IAAI,EAAE;YAC9B,IAAI,YAAY,CAAC,GAAG,EAAE;gBAClB,MAAM,YAAY,GAAG,GAAG,IAAI,CAAC,aAAa,IAAI,YAAY,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBACpF,MAAM,YAAY,GAAG,IAAA,SAAM,GAAE,CAAC;gBAC9B,YAAY,CAAC,KAAK,GAAG;oBACjB,UAAU,EAAE,YAAY;oBACxB,YAAY,EAAE,YAAY;oBAC1B,aAAa,EAAE,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,YAAY,CAAC;iBACvD,CAAC;gBACF,OAAO,CAAC,GAAG,CAAC,0EAA0E,YAAY,CAAC,KAAK,CAAC,UAAU,YAAY,CAAC,MAAM,CAAC,CAAA;gBAEvI,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC;gBACjC,MAAM,KAAK,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC;gBACpC,MAAM,KAAK,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,KAAK,CAAE,CAAC;gBAChE,8DAA8D;gBAC9D,MAAM,KAAK,CAAC,aAAa,CAAC,sBAAsB,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;oBACvF,IAAI;wBACA,IAAI,GAAG,IAAI,EAAE,IAAI,KAAK,IAAI,EAAE,EAAE;4BAC1B,IAAI,KAAK,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;gCAChC,YAAY,CAAC,EAAE,CAAC;oCACZ,KAAK,EAAE,+BAAiB,CAAC,aAAa;oCACtC,GAAG,EAAE,YAAY,CAAC,GAAG;oCACrB,IAAI,EAAE;wCACF,UAAU,EAAE,KAAK,CAAC,UAAU;wCAC5B,YAAY,EAAE,KAAK,CAAC,YAAY;wCAChC,SAAS;wCACT,KAAK;wCACL,GAAG;wCACH,KAAK;qCACR;iCACJ,CAAC,CAAC;6BACN;iCAAM;gCACH,OAAO,CAAC,GAAG,CAAC,gGAAgG,KAAK,eAAe,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;6BAC5J;yBACJ;6BAAM;4BACH,OAAO,CAAC,GAAG,CAAC,gGAAgG,KAAK,YAAY,GAAG,cAAc,KAAK,GAAG,CAAC,GAAG,CAAC,CAAC;yBAC/J;qBACJ;oBAAC,OAAO,KAAK,EAAE;wBACZ,OAAO,CAAC,GAAG,CAAC,wDAAwD,KAAK,GAAG,CAAC,GAAG,CAAC,CAAC;qBACrF;gBACL,CAAC,CAAC,CAAC;gBACH,OAAO,CAAC,GAAG,CAAC,0EAA0E,KAAK,CAAC,UAAU,aAAa,CAAC,MAAM,CAAC,CAAA;aAC9H;iBAAM;gBACH,OAAO,CAAC,GAAG,CAAC,2GAA2G,YAAY,CAAC,KAAK,GAAG,CAAC,OAAO,CAAC,CAAC;aACzJ;QACL,CAAC,CAAA;QACD,OAAO,cAAc,EAAE,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,YAA2B;QACjD,wBAAwB;QACxB,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAI,YAAY,CAAC;QAC7C,QAAQ,KAAK,EAAE;YACf,uGAAuG;YACvG,KAAK,+BAAiB,CAAC,mBAAmB;gBACtC,OAAO,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC,6BAA6B,EAAE,CAAC;YAChF,KAAK,+BAAiB,CAAC,QAAQ;gBAC3B,OAAO,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC,yBAAyB,EAAE,CAAC;YAC5E,KAAK,+BAAiB,CAAC,gBAAgB;gBACnC,IAAI,GAAG,EAAE,EAAE,mBAAmB;oBAC1B,OAAO,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC,kCAAkC,CAAC,GAAG,CAAC,CAAC;iBACvF;qBAAM;oBACH,MAAM,IAAI,KAAK,CAAC,4DAA4D,KAAK,GAAG,CAAC,CAAC;iBACzF;YACL,KAAK,+BAAiB,CAAC,eAAe;gBAClC,IAAI,GAAG,EAAE,EAAE,4BAA4B;oBACnC,OAAO,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC,iCAAiC,CAAC,GAAG,CAAC,CAAC;iBACtF;qBAAM;oBACH,MAAM,IAAI,KAAK,CAAC,4DAA4D,KAAK,GAAG,CAAC,CAAC;iBACzF;YAED,0GAA0G;YAC9G,KAAK,+BAAiB,CAAC,iBAAiB;gBACpC,OAAO,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,2BAA2B,EAAE,CAAC;YAC5E,KAAM,+BAAiB,CAAC,MAAM;gBAC1B,OAAO,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,uBAAuB,EAAE,CAAC;YACxE,KAAM,+BAAiB,CAAC,YAAY;gBAChC,IAAI,GAAG,EAAE,EAAE,0BAA0B;oBACjC,OAAO,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,8BAA8B,CAAC,GAAG,CAAC,CAAC;iBACjF;qBAAM;oBACH,MAAM,IAAI,KAAK,CAAC,4DAA4D,KAAK,GAAG,CAAC,CAAC;iBACzF;YACL,KAAK,+BAAiB,CAAC,WAAW;gBAC9B,IAAI,GAAG,IAAI,MAAM,EAAE,EAAE,6DAA6D;oBAC9E,OAAO,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,6BAA6B,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;iBACxF;qBAAM;oBACH,MAAM,IAAI,KAAK,CAAC,0EAA0E,KAAK,GAAG,CAAC,CAAC;iBACvG;YAED,2GAA2G;YAC/G,KAAK,+BAAiB,CAAC,aAAa;gBAChC,qEAAqE;gBACrE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YACjC;gBACI,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;SAChC;IACL,CAAC;IAED,qBAAqB,GAAG,KAAK,IAAI,EAAE;QAC/B,MAAM,SAAS,GAAoC,EAAG,CAAC;QACvD,MAAM,UAAU,GAAmB,EAAG,CAAC;QAEvC,KAAK,MAAM,CAAC,EAAE,YAAY,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE;YAChE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,YAAY,CAAC,CAAC,CAAC;YACvD,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;SACjC;QACD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC5C,IAAI;YACA,KAAK,IAAI,CAAC,GAAC,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBAClC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;aAC/B;SACJ;QAAC,OAAO,KAAK,EAAE;YACZ,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;SAC5D;IACL,CAAC,CAAA;IAED,SAAS,CAAC,aAA6B;QACnC,IAAI,IAAI,CAAC,SAAS,EAAE;YAChB,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;SACjC;QACD,KAAK,IAAI,CAAC,GAAC,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACzC,IAAI,IAAI,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE;gBAC7C,yCAAyC;gBACzC,OAAO,CAAC,GAAG,CAAC,gFAAgF,CAAC,OAAO,CAAC,CAAC;aACzG;iBAAM;gBACH,IAAI,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;gBAC/D,OAAO,CAAC,GAAG,CAAC,qDAAqD,aAAa,CAAC,CAAC,CAAC,CAAC,KAAK,4BAA4B,CAAC,IAAI,CAAC,CAAC;gBAC1H,IAAI,aAAa,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,+BAAiB,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE;oBAC7E,IAAI,CAAC,eAAe,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;iBAC1C;aACJ;SACJ;QAED,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE;YACjB,uDAAuD;YACvD,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAC7B,IAAI,QAAQ,GAAG,IAAI,CAAC,CAAC,UAAU;YAC/B,IAAI,IAAI,CAAC,OAAO,EAAE;gBACd,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAA;aAC/C;YACD,IAAI,CAAC,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE;gBAC9B,IAAI,CAAC,qBAAqB,EAAE,CAAC;YACjC,CAAC,EAAE,QAAQ,CAAC,CAAC;SAChB;IACL,CAAC;IAED,WAAW,CAAC,aAA6B;QACrC,KAAK,IAAI,CAAC,GAAC,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACzC,IAAI,aAAa,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,+BAAiB,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE;gBAC7E,IAAI,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;aAC5C;YACD,OAAO,IAAI,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;SACtD;QACD,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE;YAC/C,IAAI,IAAI,CAAC,SAAS,EAAE;gBAChB,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBAC9B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;aACzB;SACJ;IACL,CAAC;IAED,eAAe;QACX,IAAI,IAAI,CAAC,SAAS,EAAE;YAChB,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC9B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;SACzB;QACD,KAAK,MAAM,CAAC,EAAE,YAAY,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE;YAChE,IAAI,YAAY,CAAC,KAAK,CAAC,aAAa,CAAC,+BAAiB,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE;gBACzE,IAAI,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;aACxC;SACJ;QAED,IAAI,CAAC,cAAc,GAAG,EAAG,CAAC;IAC9B,CAAC;IAED,KAAK,CAAC,KAAK;QACP,OAAO,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC;IACzC,CAAC;IAED,KAAK,CAAC,IAAI;QACN,IAAI,CAAC,eAAe,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC;IAC7C,CAAC;CACJ;AAzOD,8DAyOC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nsshunt/stsappframework",
-  "version": "3.0.20",
+  "version": "3.0.22",
   "description": "",
   "main": "dist/index.js",
   "types": "./types/index.d.ts",

package/src/index.ts

@@ -19,3 +19,4 @@ export * from './influxdb/influxDBManagerBase'
 export * from './influxdb/influxDBManagerService'
 export * from './influxdb/influxDBManagerAgent'
 export * from './publishertransports/publishTransportUtils'
+export * from './instrumentationsubscriber'

package/src/instrumentationsubscriber.ts

@@ -0,0 +1,248 @@
+import { ISubscriptions, ISubscription, ISubscriptionPayload, SubscriptionTopic } from './commonTypes'
+import { KafkaManager, IKafkaManagerConfig } from './kafka/kafkamanager'
+import { InfluxDBManager } from './influxdb/influxDBManager'
+
+import { v4 as uuidv4 } from 'uuid';
+
+import { $Options } from '@nsshunt/stsconfig'
+import { STSOptionsBase } from '@nsshunt/stsutils';
+const goptions = $Options()
+
+export interface IInstrumentationSubscriberOptions {
+    SubscriptionInterval: number
+}
+
+export class InstrumentationSubscriber extends STSOptionsBase {
+    #subscriptions: Record<string, ISubscription> = { };
+    #influxDBManager: InfluxDBManager;
+    #interval: NodeJS.Timer | null = null;
+    #km: KafkaManager;
+    #KAFKA_PREFIX: string = '__STS__';
+
+    constructor(options: IInstrumentationSubscriberOptions) {
+        super(options);
+
+        this.#influxDBManager = new InfluxDBManager();
+
+        const kmc: IKafkaManagerConfig = {
+            clientId: goptions.kafka_clientId,
+            brokers: goptions.kafka_brokers.split(','),
+            keepAlive: goptions.kafka_keep_alive,
+            adminTimeout: goptions.kafka_admin_timeout,
+            connectionTimeout: goptions.kafka_connection_timeout,
+            logLevel: goptions.kafka_log_level,
+            useSSL: goptions.kafka_use_ssl,
+            requestTimeout: goptions.kafka_request_timeout
+        }
+        if (goptions.kafka_use_ssl) {
+            kmc.ssl = {
+                rejectUnauthorized: goptions.kafka_ssl_rejectUnauthorized,
+                cafile: goptions.kafka_ssl_cafile,
+                certfileFile: goptions.kafka_ssl_certfile,
+                keyfile: goptions.kafka_ssl_keyfile
+            }
+        }
+        this.#km = new KafkaManager(kmc);
+    }
+
+    async #UnSubscribeKafka(subscription: ISubscription): Promise<void> {
+        const unsubscribeKafka = async () => {
+            const kafka = subscription.kafka;
+            if (kafka) {
+                // Un-Subscribe from this kafka topic
+                console.log(`InstrumentationSubscriber:#UnSubscribeKafka(): Unsubscribe from Kafka topc: [${kafka.kafkaTopic}] Starting`.yellow)
+                await kafka.kafkaConsumer?.Stop();
+                await kafka.kafkaConsumer?.Disconnect();
+                console.log(`InstrumentationSubscriber:#UnSubscribeKafka(): Unsubscribe from Kafka topc: [${kafka.kafkaTopic}] Completed.`.yellow)
+            } else {
+                console.log(`InstrumentationSubscriber:#UnSubscribeKafka(): Kafka details do not eixsting for this topc: [${subscription.topic}].`.magenta)
+            }
+        };
+        return unsubscribeKafka();
+    }
+
+    async #SubscribeKafka(subscription: ISubscription): Promise<void> {
+        const kafkaSubscribe = async () => {
+            if (subscription.key) {
+                const kafkaSafeKey = `${this.#KAFKA_PREFIX}_${subscription.key}`.replace(/@/g, '_');
+                const kafkaGroupId = uuidv4();
+                subscription.kafka = {
+                    kafkaTopic: kafkaSafeKey,
+                    kafkaGroupId: kafkaGroupId,
+                    kafkaConsumer: this.#km.CreateConsumer(kafkaGroupId)
+                };
+                console.log(`InstrumentationSubscriber:#SubscribeKafka(): Subscribe to Kafka topc: [${subscription.kafka.kafkaTopic}] Starting`.yellow)
+
+                const kafka = subscription.kafka;
+                await kafka.kafkaConsumer.Connect();
+                await kafka.kafkaConsumer.Subscribe([kafka.kafkaTopic], false );
+                // eslint-disable-next-line  @typescript-eslint/no-unused-vars
+                await kafka.kafkaConsumer.StartConsumingMessages((topic, key, partition, value, headers) => {
+                    try {
+                        if (key != "" && value != "") {
+                            if (topic.localeCompare(key) === 0) {
+                                subscription.cb({
+                                    topic: SubscriptionTopic.LogProcessing,
+                                    key: subscription.key, // Original requested topic (may be kafka un-safe)
+                                    data: { 
+                                        kafkaTopic: kafka.kafkaTopic, // Safe kafka topic key with prefix. kafkaTopic, topic and key should be same value
+                                        kafkaGroupId: kafka.kafkaGroupId,
+                                        partition,
+                                        topic, // kafkaTopic, topic and key should be same value
+                                        key, // kafkaTopic, topic and key should be same value
+                                        value
+                                    }
+                                });
+                            } else {
+                                console.log(`InstrumentationSubscriber:#SubscribeKafka(): Invalid message format from Kafka data, topic: [${topic}] and key: [${key}] do not match`.red);
+                            }
+                        } else {
+                            console.log(`InstrumentationSubscriber:#SubscribeKafka(): Invalid message format from Kafka data, topic: [${topic}], key: [${key}], value: [${value}]`.red);
+                        }
+                    } catch (error) {
+                        console.log(`InstrumentationSubscriber:#SubscribeKafka(): Error: [${error}]`.red);
+                    }
+                });
+                console.log(`InstrumentationSubscriber:#SubscribeKafka(): Subscribe to Kafka topc: [${kafka.kafkaTopic}] Completed`.yellow)
+            } else {
+                console.log(`InstrumentationSubscriber:#SubscribeKafka(): Could not subscribe to Kafka topc, missing key for topic: [${subscription.topic}]`.magenta);
+            }
+        }
+        return kafkaSubscribe();
+    }
+
+    async #OutputSubscription(subscription: ISubscription): Promise<ISubscriptionPayload> {
+        // Service subscriptions
+        const { topic, key, subkey }  = subscription;
+        switch (topic) {
+        // Services -------------------------------------------------------------------------------------------
+        case SubscriptionTopic.AllServicesCombined :
+            return this.#influxDBManager.serviceManager.GetInfluxDBResultsRootService();
+        case SubscriptionTopic.Services :
+            return this.#influxDBManager.serviceManager.GetInfluxDBResultsService();
+        case SubscriptionTopic.ServiceInstances :
+            if (key) { // key = service_id
+                return this.#influxDBManager.serviceManager.GetInfluxDBResultsServiceInstances(key);
+            } else {
+                throw new Error(`#OutputSubscription: key not provided for subscription: [${topic}]`);
+            }
+        case SubscriptionTopic.ServiceInstance :
+            if (key) { // key = service instance id
+                return this.#influxDBManager.serviceManager.GetInfluxDBResultsServiceInstance(key);
+            } else {
+                throw new Error(`#OutputSubscription: key not provided for subscription: [${topic}]`);
+            }
+
+            // User Agents -------------------------------------------------------------------------------------------
+        case SubscriptionTopic.AllAgentsCombined :
+            return this.#influxDBManager.agentManager.GetInfluxDBResultsRootAgent();
+        case  SubscriptionTopic.Agents :
+            return this.#influxDBManager.agentManager.GetInfluxDBResultsAgent();
+        case  SubscriptionTopic.AgentWorkers :
+            if (key) { // key = agent instance id
+                return this.#influxDBManager.agentManager.GetInfluxDBResultsAgentThreads(key);
+            } else {
+                throw new Error(`#OutputSubscription: key not provided for subscription: [${topic}]`);
+            }
+        case SubscriptionTopic.AgentWorker :
+            if (key && subkey) { // key = agent instance id, subkey = agent instance worker id
+                return this.#influxDBManager.agentManager.GetInfluxDBResultsAgentThread(key, subkey);
+            } else {
+                throw new Error(`#OutputSubscription: key and/or subkey not provided for subscription: [${topic}]`);
+            }
+
+            // Logging (both service and agent) -----------------------------------------------------------------------
+        case SubscriptionTopic.LogProcessing :
+            // Kafka processing will be handled by a dedicated kafka run consumer
+            return { topic, data: null };
+        default :
+            return { topic, data: null };
+        }   
+    }
+
+    #ExecuteSubscriptions = async () => {
+        const promArray: Promise<ISubscriptionPayload>[] = [ ];
+        const indexArray: ISubscriptions = [ ];
+
+        for (const [, subscription] of Object.entries(this.#subscriptions)) {
+            promArray.push(this.#OutputSubscription(subscription));
+            indexArray.push(subscription);
+        }
+        const retVal = await Promise.all(promArray);
+        try {
+            for (let i=0; i < retVal.length; i++) {
+                indexArray[i].cb(retVal[i]);
+            }
+        } catch (error) {
+            console.log(`${error} -- ${JSON.stringify(retVal)}`.red);
+        }
+    }
+
+    Subscribe(subscriptions: ISubscriptions) {
+        if (this.#interval) {
+            clearInterval(this.#interval);
+        }
+        for (let i=0; i < subscriptions.length; i++) {
+            if (this.#subscriptions[subscriptions[i].topic]) {
+                // Subscription already exists - ignoring
+                console.log(`InstrumentationSubscriber.Subscribe(): Subscription already exists - ignoring.`.magenta);
+            } else {
+                this.#subscriptions[subscriptions[i].topic] = subscriptions[i];
+                console.log(`InstrumentationSubscriber.Subscribe(): topicKey: [${subscriptions[i].topic}] successfully subscribed.`.grey);
+                if (subscriptions[i].topic.localeCompare(SubscriptionTopic.LogProcessing) === 0) {
+                    this.#SubscribeKafka(subscriptions[i]);
+                }
+            }
+        }
+
+        if (!this.#interval) {
+            // Output subscription immediately and then iterate ...
+            this.#ExecuteSubscriptions();
+            let interval = 1000; // default
+            if (this.options) {
+                interval = this.options.SubscriptionInterval
+            }
+            this.#interval = setInterval(() => {
+                this.#ExecuteSubscriptions();
+            }, interval);
+        }
+    }
+
+    UnSubscribe(subscriptions: ISubscriptions) {
+        for (let i=0; i < subscriptions.length; i++) {
+            if (subscriptions[i].topic.localeCompare(SubscriptionTopic.LogProcessing) === 0) {
+                this.#UnSubscribeKafka(subscriptions[i]);
+            }
+            delete this.#subscriptions[subscriptions[i].topic];
+        }
+        if (Object.keys(this.#subscriptions).length === 0) {
+            if (this.#interval) {
+                clearInterval(this.#interval);
+                this.#interval = null;
+            }
+        }
+    }
+
+    #UnSubscribeAll() {
+        if (this.#interval) {
+            clearInterval(this.#interval);
+            this.#interval = null;
+        }
+        for (const [, subscription] of Object.entries(this.#subscriptions)) {
+            if (subscription.topic.localeCompare(SubscriptionTopic.LogProcessing) === 0) {
+                this.#UnSubscribeKafka(subscription);
+            }
+        }
+
+        this.#subscriptions = { };
+    }
+
+    async Start() {
+        return this.#influxDBManager.Start();
+    }
+
+    async Stop() {
+        this.#UnSubscribeAll();
+        return this.#influxDBManager.Terminate();
+    }
+}

package/types/index.d.ts

@@ -19,4 +19,5 @@ export * from './influxdb/influxDBManagerBase';
 export * from './influxdb/influxDBManagerService';
 export * from './influxdb/influxDBManagerAgent';
 export * from './publishertransports/publishTransportUtils';
+export * from './instrumentationsubscriber';
 //# sourceMappingURL=index.d.ts.map

package/types/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAA;AAC7B,cAAc,kBAAkB,CAAA;AAChC,cAAc,YAAY,CAAA;AAC1B,cAAc,iBAAiB,CAAA;AAC/B,cAAc,iBAAiB,CAAA;AAC/B,cAAc,qBAAqB,CAAA;AACnC,cAAc,qBAAqB,CAAA;AACnC,cAAc,qBAAqB,CAAA;AACnC,cAAc,qBAAqB,CAAA;AACnC,cAAc,wBAAwB,CAAA;AACtC,cAAc,wBAAwB,CAAA;AACtC,cAAc,mBAAmB,CAAA;AACjC,cAAc,sBAAsB,CAAA;AACpC,cAAc,uBAAuB,CAAA;AACrC,cAAc,uBAAuB,CAAA;AACrC,cAAc,wBAAwB,CAAA;AACtC,cAAc,4BAA4B,CAAA;AAC1C,cAAc,gCAAgC,CAAA;AAC9C,cAAc,mCAAmC,CAAA;AACjD,cAAc,iCAAiC,CAAA;AAC/C,cAAc,6CAA6C,CAAA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAA;AAC7B,cAAc,kBAAkB,CAAA;AAChC,cAAc,YAAY,CAAA;AAC1B,cAAc,iBAAiB,CAAA;AAC/B,cAAc,iBAAiB,CAAA;AAC/B,cAAc,qBAAqB,CAAA;AACnC,cAAc,qBAAqB,CAAA;AACnC,cAAc,qBAAqB,CAAA;AACnC,cAAc,qBAAqB,CAAA;AACnC,cAAc,wBAAwB,CAAA;AACtC,cAAc,wBAAwB,CAAA;AACtC,cAAc,mBAAmB,CAAA;AACjC,cAAc,sBAAsB,CAAA;AACpC,cAAc,uBAAuB,CAAA;AACrC,cAAc,uBAAuB,CAAA;AACrC,cAAc,wBAAwB,CAAA;AACtC,cAAc,4BAA4B,CAAA;AAC1C,cAAc,gCAAgC,CAAA;AAC9C,cAAc,mCAAmC,CAAA;AACjD,cAAc,iCAAiC,CAAA;AAC/C,cAAc,6CAA6C,CAAA;AAC3D,cAAc,6BAA6B,CAAA"}

package/types/instrumentationsubscriber.d.ts

@@ -0,0 +1,14 @@
+import { ISubscriptions } from './commonTypes';
+import { STSOptionsBase } from '@nsshunt/stsutils';
+export interface IInstrumentationSubscriberOptions {
+    SubscriptionInterval: number;
+}
+export declare class InstrumentationSubscriber extends STSOptionsBase {
+    #private;
+    constructor(options: IInstrumentationSubscriberOptions);
+    Subscribe(subscriptions: ISubscriptions): void;
+    UnSubscribe(subscriptions: ISubscriptions): void;
+    Start(): Promise<void>;
+    Stop(): Promise<void>;
+}
+//# sourceMappingURL=instrumentationsubscriber.d.ts.map

package/types/instrumentationsubscriber.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"instrumentationsubscriber.d.ts","sourceRoot":"","sources":["../src/instrumentationsubscriber.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAA0D,MAAM,eAAe,CAAA;AAOtG,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAGnD,MAAM,WAAW,iCAAiC;IAC9C,oBAAoB,EAAE,MAAM,CAAA;CAC/B;AAED,qBAAa,yBAA0B,SAAQ,cAAc;;gBAO7C,OAAO,EAAE,iCAAiC;IA+JtD,SAAS,CAAC,aAAa,EAAE,cAAc;IA8BvC,WAAW,CAAC,aAAa,EAAE,cAAc;IA6BnC,KAAK;IAIL,IAAI;CAIb"}