@blazedpath/commons 0.3.1 → 0.4.0

package/blz-jwt/index.js

@@ -0,0 +1,121 @@
+const jwt = require('jsonwebtoken');
+
+class JwtSecretKeyNotDefined extends Error {
+    constructor() {
+        super('JWT_SECRET_KEY is not defined.');
+        this.name = 'JwtSecretKeyNotDefined';
+        this.code = 'JwtSecretKeyNotDefined';
+    }
+}
+
+function getJwtSecretKey() {
+    const secret = process.env.JWT_SECRET_KEY;
+    if (!secret || secret.trim() === '') {
+        throw new JwtSecretKeyNotDefined();
+    }
+    return secret;
+}
+
+module.exports = {
+    JwtSecretKeyNotDefined,
+
+    jwtSign: function (payload, secret, algorithm) {
+        try {
+            switch (algorithm) {
+                case 'HMAC-SHA256':
+                    return jwt.sign(payload, secret, { algorithm: 'HS256' });
+                case 'HMAC-SHA384':
+                    return jwt.sign(payload, secret, { algorithm: 'HS384' });
+                case 'HMAC-SHA512':
+                    return jwt.sign(payload, secret, { algorithm: 'HS512' });
+                default:
+                    throw new Error('Invalid encryption algorithm');
+            }
+        } catch (jwterr) {
+            if (jwterr.name === 'TokenExpiredError') {
+                let err = new Error();
+                err.code = 'JwtExpired';
+                err.data = { expiredAt: jwterr.expiredAt };
+                throw err;
+            } else {
+                throw jwterr;
+            }
+        }
+    },
+
+    jwtVerify: function (token, secret) {
+        try {
+            return jwt.verify(token, secret);
+        } catch (jwterr) {
+            if (jwterr.name === 'TokenExpiredError') {
+                let err = new Error();
+                err.code = 'JwtExpired';
+                err.data = { expiredAt: jwterr.expiredAt };
+                throw err;
+            } else {
+                let err = new Error();
+                err.code = 'JwtError';
+                err.data = {};
+                throw err;
+            }
+        }
+    },
+
+    jwtDecode: function (token) {
+        try {
+            return jwt.decode(token);
+        } catch (jwterr) {
+            if (jwterr.name === 'TokenExpiredError') {
+                let err = new Error();
+                err.code = 'JwtExpired';
+                err.data = { expiredAt: jwterr.expiredAt };
+                throw err;
+            } else {
+                let err = new Error();
+                err.code = 'JwtError';
+                err.data = {};
+                throw err;
+            }
+        }
+    },
+
+    jwtCreateToken: function (id, expiresIn = '10s') {
+        return jwt.sign({ id }, getJwtSecretKey(), { expiresIn });
+    },
+
+    jwtGetId: function (token) {
+        try {
+            const decoded = jwt.verify(token, getJwtSecretKey());
+            return decoded.id;
+        } catch (err) {
+            const error = new Error('Invalid or expired token');
+            error.code = 'JwtInvalid';
+            error.data = err.message;
+            throw error;
+        }
+    },
+
+    jwtEncodeUrl: function (url, expiresIn = '10s') {
+        const [path, query] = url.split('?');
+        const parts = path.split('/').map(part => {
+            if (/^\d+$/.test(part)) {
+                return jwt.sign({ id: parseInt(part) }, getJwtSecretKey(), { expiresIn });
+            }
+            return part;
+        });
+        return parts.join('/') + (query ? `?${query}` : '');
+    },
+
+    jwtDecodeUrl: function (url) {
+        const [path, query] = url.split('?');
+        const parts = path.split('/').map(part => {
+            try {
+                const decoded = jwt.verify(part, getJwtSecretKey());
+                return decoded.id?.toString() ?? part;
+            } catch {
+                return part;
+            }
+        });
+        return parts.join('/') + (query ? `?${query}` : '');
+    }
+};

package/blz-kafka/index.js

@@ -0,0 +1,522 @@
+// This code has functions for the backend.
+const Kafkajs = require('kafkajs');
+const Fs = require('fs-extra');
+const retry = require('async-retry');
+const logger = require('pino')();
+// let blzCorepath= (true) ? '../../blz-core/dual/index' : '../../blz-core/sources-dual/index';
+// const BlzCore = require(blzCorepath);
+let convertToString = function (value) {
+    if (value === null)
+        return null;
+    let valueType = toString.call(value);
+    if (valueType === '[object String]')
+        return value;
+    if (valueType === '[object Number]')
+        return value.toString();
+    if (valueType === '[object Uint8Array]')
+        return value.toString();
+    if (valueType === '[object Boolean]')
+        return value ? 'true' : 'false';
+    if (valueType === '[object Date]')
+        return value.toJSON();
+    if (valueType === '[object Object]' && value.type === 'Buffer' && value.data)
+        return Buffer.from(value.data).toString();
+    throw this.errorInvalidConversion(value, 'string');
+}
+
+let producers = {};
+
+const getConfig = async function (connection) {
+    let kafkaConfig = {};
+    if (connection.brokers)
+        kafkaConfig.brokers = connection.brokers.split(',');
+    kafkaConfig.connectionTimeout = connection.connectionTimeout || 3000;
+    kafkaConfig.authenticationTimeout= connection.authenticationTimeout  || 1000;
+    kafkaConfig.reauthenticationThreshold = connection.reauthenticationThreshold || 10000;
+    if (connection.requestTimeout)
+        kafkaConfig.requestTimeout = connection.requestTimeout;
+    if (connection.connectionRetries || connection.connectionInitialRetryTime || connection.connectionMaxRetryTime || connection.connectionRetryFactor){
+        kafkaConfig.retry = {};
+        if (connection.connectionRetries)
+            kafkaConfig.retry.retries = connection.connectionRetries;
+        if (connection.connectionInitialRetryTime)
+            kafkaConfig.retry.initialRetryTime = connection.connectionInitialRetryTime;
+        if (connection.connectionMaxRetryTime)
+            kafkaConfig.retry.maxRetryTime = connection.connectionMaxRetryTime;
+        if (connection.connectionRetryFactor)
+            kafkaConfig.retry.factor = connection.connectionRetryFactor;
+    }
+    if (connection.ssl) {
+        kafkaConfig.ssl = {};
+        if (connection.sslPfx)
+            kafkaConfig.ssl.pfx = await convertFileArg(connection.sslPfx);
+        if (connection.sslKey)
+            kafkaConfig.ssl.key = await convertFileArg(connection.sslKey);
+        if (connection.sslPassphrase)
+            kafkaConfig.ssl.passphrase = connection.sslPassphrase;
+        if (connection.sslCert)
+            kafkaConfig.ssl.cert = await convertFileArg(connection.sslCert);
+        if (connection.sslCa)
+            kafkaConfig.ssl.ca = await convertFileArg(connection.sslCa);
+        if (connection.sslCrl)
+            kafkaConfig.ssl.crl = await convertFileArg(connection.sslCrl);
+        if (connection.sslCiphers)
+            kafkaConfig.ssl.ciphers = connection.sslCiphers;
+        if (connection.sslSecureProtocol)
+            kafkaConfig.ssl.secureProtocol = connection.sslSecureProtocol;
+    }
+    if (connection.sasl) {
+        kafkaConfig.sasl = {
+            mechanism: connection.saslMechanism,
+            username: connection.saslUsername,
+            password: connection.saslPassword,
+        }
+    }
+    kafkaConfig.logLevel = Kafkajs.logLevel.ERROR;
+    return kafkaConfig;
+}
+
+let getOrCreateProducer = async function (connection) {
+    let producer = producers[connection.name];
+    if (producer) {
+        return producer;
+    }
+    let kafkaConfig = await getConfig(connection);    
+    let kafka = new Kafkajs.Kafka(kafkaConfig);
+    producer = kafka.producer({createPartitioner: Kafkajs.Partitioners.LegacyPartitioner});
+    await producer.connect();
+    producers[connection.name] = producer;
+    return producer;
+};
+
+let getOrCreateAdmin = async function (connection, config) {
+    let kafkaConfig = await getConfig(connection);    
+    let kafka = new Kafkajs.Kafka(kafkaConfig);
+    admin = kafka.admin();
+    await admin.connect();
+    producers[connection.name] = producer;
+    return producer;
+};
+
+let convertFileArg = async function (fileArg) {
+    if (fileArg) {
+        let fileArgParts = fileArg.split(',');
+        if (fileArgParts.length === 1) {
+            return await Fs.readFile(fileArgParts[0], 'utf-8');
+        }
+        else if (fileArgParts.length > 1) {
+            let promises = [];
+            for (let i = 0; i < fileArgParts.length; i++) {
+                let fileArgPart = fileArgParts[0];
+                promises.push(Fs.readFile(fileArgParts[i], 'utf-8'));
+            }
+            return await Promise.all(promises);
+        }
+    }
+    return null;
+};
+
+// createKafkaInstances: Returns an obj with configured new Kafkajs.Kafka(kafkaCondig), using key = systemName
+// Return obj can only be loged with console.dir()
+// Parameters:
+// - kafkaConsumers: array con el json serializado de cada consumidor de kafka.
+// - connections(blz-suite) viene de config.connections donde config es toda la config cuando le di START
+//   Adentro tienen el nombre de la connexion que se va a usar.
+// - buildConfigItems = config.buildConfigItems;
+async function createKafkaInstances(kafkaConsumers, connections, buildConfigItems, runHelper) {
+    let kafkas = {};
+    // Iterate all consumers to make sure all kafka connections will be up and running exactly 1 time
+    for (let kafkaConsumerName in kafkaConsumers) {
+        let buildKafkaConsumer = kafkaConsumers[kafkaConsumerName];
+        let kafka = kafkas[buildKafkaConsumer.systemName];
+        // If connection is present, skip
+        if (kafka) {
+            continue
+        }
+
+        let connection = connections[buildKafkaConsumer.systemName]; 
+        // Check if kafka connection info is undefined
+        if (!connection) {
+            const kafkaBuildSystemConfig = buildConfigItems[buildKafkaConsumer.systemName]
+            if (kafkaBuildSystemConfig && kafkaBuildSystemConfig.action !== 'omit') {
+                console.log(`Connection ${buildKafkaConsumer.systemName} undefined`)
+            } else if (!kafkaBuildSystemConfig) {
+                console.log(`Config ${buildKafkaConsumer.systemName} undefined`)
+            }
+            continue
+        }
+        if (connection.provider) {
+            kafka = connection;
+        }
+        else {
+            let kafkaConfig = {};
+            if (connection.brokers)
+                kafkaConfig.brokers = connection.brokers.split(',');
+            kafkaConfig.connectionTimeout = connection.connectionTimeout || 3000;
+            kafkaConfig.authenticationTimeout= connection.authenticationTimeout  || 1000;
+            kafkaConfig.reauthenticationThreshold = connection.reauthenticationThreshold || 10000;
+            if (connection.requestTimeout)
+                kafkaConfig.requestTimeout = connection.requestTimeout;
+            if (connection.connectionRetries || connection.connectionInitialRetryTime || connection.connectionMaxRetryTime || connection.connectionRetryFactor){
+                kafkaConfig.retry = {};
+                if (connection.connectionRetries)
+                    kafkaConfig.retry.retries = connection.connectionRetries;
+                if (connection.connectionInitialRetryTime)
+                    kafkaConfig.retry.initialRetryTime = connection.connectionInitialRetryTime;
+                if (connection.connectionMaxRetryTime)
+                    kafkaConfig.retry.maxRetryTime = connection.connectionMaxRetryTime;
+                if (connection.connectionRetryFactor)
+                    kafkaConfig.retry.factor = connection.connectionRetryFactor;
+            }
+            if (connection.ssl) {
+                kafkaConfig.ssl = {};
+                if (connection.sslPfx)
+                    kafkaConfig.ssl.pfx = await convertFileArg(connection.sslPfx);
+                if (connection.sslKey)
+                    kafkaConfig.ssl.key = await convertFileArg(connection.sslKey);
+                if (connection.sslPassphrase)
+                    kafkaConfig.ssl.passphrase = connection.sslPassphrase;
+                if (connection.sslCert)
+                    kafkaConfig.ssl.cert = await convertFileArg(connection.sslCert);
+                if (connection.sslCa)
+                    kafkaConfig.ssl.ca = await convertFileArg(connection.sslCa);
+                if (connection.sslCrl)
+                    kafkaConfig.ssl.crl = await convertFileArg(connection.sslCrl);
+                if (connection.sslCiphers)
+                    kafkaConfig.ssl.ciphers = connection.sslCiphers;
+                if (connection.sslSecureProtocol)
+                    kafkaConfig.ssl.secureProtocol = connection.sslSecureProtocol;
+            }
+            if (connection.sasl) {
+                kafkaConfig.sasl = {
+                    mechanism: connection.saslMechanism,
+                    username: connection.saslUsername,
+                    password: connection.saslPassword,
+                }
+            }
+            kafkaConfig.logLevel = Kafkajs.logLevel.ERROR;
+            kafkaConfig.logCreator = ()=> {
+                return ({ namespace, level, label, log }) => {
+                    if (level === 1 && log.message && log.message.includes('The group is rebalancing')) {
+                        // Do nothing or log at a different level for rebalancing errors
+                        return;
+                    } else if (level === 1 ) {
+                        runHelper.logError(`[${namespace}] : ${JSON.stringify(log.message)}`, namespace, log);
+                    } else {
+                        runHelper.log(`[${namespace}] : ${log.message}`);
+                    }                   
+                }
+            };
+            kafka = new Kafkajs.Kafka(kafkaConfig);
+        }
+        kafkas[buildKafkaConsumer.systemName] = kafka;
+    }
+    return kafkas;
+}
+// Funtion to start the kafka consumers
+// - kafkas: obj con instancias new Kafkajs.Kafka(kafkaCondig), configuradas, usando key = systemName
+// - config: en blz-suite es toda la environment config del proyecto cuando le di start.
+//           en blz-builder es ... TODO
+// - kafkaConsumers: objeto con todos los consumers. Cada property del objeto es consumer
+// - connections: objeto con todas las conneciones con key
+async function startKafkaConsumers (kafkas,config, kafkaConsumers, connections, runHelper) {
+
+    for (let kafkaConsumerName in kafkaConsumers) {
+        let buildKafkaConsumer = kafkaConsumers[kafkaConsumerName];
+        // Probe kafka connection configuration for message error options
+        let connection = connections[buildKafkaConsumer.systemName];
+        if (!connection) {
+            console.log(`Kafka connection : ${buildKafkaConsumer.systemName} not defined or ommited`);
+            continue;
+        }
+        let kafkaErrorOptions = {};
+        kafkaErrorOptions.errorOptions = connection.errorOptions ?? false;
+        kafkaErrorOptions.manualCommit = connection.messageManualCommit ?? false;
+        kafkaErrorOptions.retries =  connection.messageRetryCount ?? 5;
+        kafkaErrorOptions.initialRetryTime = connection.initialRetryTime ?? 300;
+        kafkaErrorOptions.factor = connection.messageRetryFactor ?? 2;
+        kafkaErrorOptions.multiplier = connection.messageRetryMultiplier ?? 2;
+        kafkaErrorOptions.traceAll = connection.traceAll ?? false;
+        kafkaErrorOptions.retryTopic = (buildKafkaConsumer.useErrorTopic && buildKafkaConsumer.parameterizedErrorTopic)? config.parameters[ buildKafkaConsumer.parameterizedErrorTopic.configParameterName ] : '';
+
+        let topics = [];
+        if (buildKafkaConsumer.topics){
+            for (let i = 0; i < buildKafkaConsumer.topics.length; i++) {
+                let topic = buildKafkaConsumer.topics[i];
+                topics.push(topic);
+            }
+        }
+        if (buildKafkaConsumer.parameterizedTopics){
+            for (let i = 0; i < buildKafkaConsumer.parameterizedTopics.length; i++) {
+                let parameterizedTopic = buildKafkaConsumer.parameterizedTopics[i];
+                const topic = config.parameters[parameterizedTopic.configParameterName];
+                if (!topic) {
+                    console.error(`Kafka: ${parameterizedTopic.configParameterName} parameter not had topic defined`)
+                    continue
+                }
+                topics.push(config.parameters[parameterizedTopic.configParameterName]);
+            }
+        }
+        let kafka = kafkas[buildKafkaConsumer.systemName];
+        if (!kafka) {
+            console.error(`Kafka ${buildKafkaConsumer.systemName} not found`)
+            continue
+        }
+        const groupId = buildKafkaConsumer.isParameterizedGroupId
+            ? config.parameters[ buildKafkaConsumer.parameterizedGroupId.configParameterName ]
+            : buildKafkaConsumer.groupId
+        
+        if (groupId === undefined || groupId === null || groupId.trim() === '') {
+            console.error(`Kafka: ${kafkaConsumerName} consumer not had groupId defined`)
+            continue
+        }        
+        if (topics.length === 0) {
+            console.error(`Kafka: ${kafkaConsumerName} consumer not had topics defined`)
+            continue
+        }
+        // When Kafka = mock, use this section
+        if (kafka.provider) {
+            setInterval(async function () {
+                let consumeData = await kafka.provider.consume(kafka, groupId, topics);
+                while (consumeData) {
+                    let callContext = runHelper.createCallContext('Running Kafka Consumer [' + kafkaConsumerName + ']...');
+                    try {
+                        if (kafkaErrorOptions.traceAll) {
+                            console.log(
+                                'KAFKA CONSUMER | ' + kafkaConsumerName +
+                                ' | TOPIC: ' + consumeData.topic +
+                                (consumeData.partition ? ' | PARTITION: ' + consumeData.partition : '') +
+                                (consumeData.message.key ? ' | MESSAGE KEY: ' + convertToString(consumeData.message.key) : '') +
+                                (consumeData.message.value ? ' | MESSAGE VALUE: ' + convertToString(consumeData.message.value) : '')
+                            );
+                        }
+                        let ctx0 = {
+                            topic: consumeData.topic,
+                            partition: consumeData.partition,
+                            message: { key: consumeData.message.key, value: consumeData.message.value, headers: consumeData.message.headers }
+                        };
+                        await runHelper.runAlgorithm(callContext, { elementType: 'KafkaConsumer', elementName: kafkaConsumerName, path: buildKafkaConsumer.path, aliasesMatches: buildKafkaConsumer.aliasesMatches }, null, buildKafkaConsumer.algorithm, ctx0);
+						if (kafkaErrorOptions.manualCommit) {
+                            await consumer.commitOffsets([{ topic, partition, offset: (Number(message.offset) + 1).toString() }]);
+                        }
+                    }
+                    catch (err) {
+                        runHelper.logError(err);
+                    }
+                    await runHelper.terminateCallContext(callContext);
+                    consumeData = await kafka.provider.consume(kafka, groupId, topics);
+                }
+            }, 4000);
+        }
+        // If Kafka is an actual server (!= mock). Use this section
+        else {
+            try{
+                let consumerOptions = {}
+                let consumerOptionsDefault = {
+                    sessionTimeout: 60000,                // 60 seconds
+                    heartbeatInterval: 6000,              // 6 seconds
+                    maxBytesPerPartition: 2097152,        // 2 MB
+                    maxBytes: 20971520,                   // 20 MB
+                    maxWaitTimeInMs: 5000,                // 1 second
+                    minBytes: 1,                          // 1 byte
+                    maxInFlightRequests: 10,               // 10 in-flight requests
+                    retry: {
+                        initialRetryTime: 100,            // 100 milliseconds
+                        retries: 5,                       // Increase to 5 retries
+                        factor: 2,                        // Exponential backoff factor
+                        multiplier: 2                     // Backoff multiplier
+                    }
+                };
+                if (connection.consumerOptions){
+                    consumerOptions = { ...consumerOptionsDefault, ...connection.consumerOptions };
+                }
+                let consumer = kafka.consumer({ groupId: groupId });
+                await consumer.connect(consumerOptions);
+                for (let i = 0; i < topics.length; i++) {
+                    const topic = topics[i];
+                    if (!topic || topic.trim() === '') {
+                        runHelper.logError(new Error(`Kafka: ${kafkaConsumerName} consumer not had topic defined`));
+                        continue
+                    }
+                    try {
+                        await consumer.subscribe({ topic: topic, fromBeginning: buildKafkaConsumer.fromBeginning });
+                    } catch(err) {
+                        runHelper.logError(new Error(`Kafka: error whit topic ${topic}: ${err.message} `,err));                
+                    }
+                }
+                // Handle group join and leave events
+                consumer.on(consumer.events.GROUP_JOIN, async e => {
+                    try {
+                        const group = (e && e.payload && e.payload.groupId) ? e.payload.groupId : 'undefined';
+                        const memberId = (e && e.payload && e.payload.memberId) ? e.payload.memberId : 'undefined';
+                        console.log(`Consumer with member ID ${memberId} has joined the group: ${group}`);
+                    } catch (error) {
+                        console.error('Error in GROUP_JOIN event handler:', error);
+                    }
+                });
+                consumer.on(consumer.events.REBALANCING, async e => {
+                    try {
+                        const group = (e && e.payload && e.payload.groupId) ? e.payload.groupId : 'undefined';
+                        const memberId = (e && e.payload && e.payload.memberId) ? e.payload.memberId : 'undefined';
+                        console.log(`Consumer with member ID ${memberId} has joined the group: ${group}`);
+                    } catch (error) {
+                        console.error('Error in GROUP_JOIN event handler:', error);
+                    }
+                });
+                // consumer.on(consumer.events.CONNECT, async e => {
+                //     runHelper.log(`Consumer connected: ${JSON.stringify(e)}`);
+                // });
+                consumer.on(consumer.events.DISCONNECT, async e => {
+                    runHelper.log(`Consumer disconnected: ${JSON.stringify(e)}`);
+                    await consumer.connect();
+                });
+                consumer.on(consumer.events.CRASH, async e => {
+                    runHelper.logError(`Consumer crashed: ${e.error.message}`, e.error);
+                    await consumer.disconnect();
+                    await consumer.connect();
+                });
+                // consumer run configuration
+                await consumer.run({
+                    autoCommit: !kafkaErrorOptions.manualCommit,
+                    eachMessage: async function ({ topic, partition, message }) {
+                        let callContext = runHelper.createCallContext('Running Kafka Consumer [' + kafkaConsumerName + ']...');
+                        try {
+                            await retry(async (bail, attempt) => {
+                                if (kafkaErrorOptions.traceAll) {
+                                    console.log(`Attempt ${attempt} | ` + 'KAFKA CONSUMER | ' + kafkaConsumerName + ' | TOPIC: ' + topic + (partition ? ' | PARTITION: ' + partition : '') + (message.key ? ' | MESSAGE KEY: ' + convertToString(message.key) : '') + (message.value ? ' | MESSAGE VALUE: ' + convertToString(message.value) : ''));
+                                }
+                                let ctx0 = {
+                                    topic: topic,
+                                    partition: partition,
+                                    message: { key: message.key, value: message.value, headers: message.headers }
+                                };
+                                await runHelper.runAlgorithm(callContext, { elementType: 'KafkaConsumer', elementName: kafkaConsumerName, path: buildKafkaConsumer.path, aliasesMatches: buildKafkaConsumer.aliasesMatches }, null, buildKafkaConsumer.algorithm, ctx0);
+                            }, {
+                                retries: kafkaErrorOptions.retries, // Maximum number of retries
+                                minTimeout: kafkaErrorOptions.initialRetryTime, // Minimum wait time between retries (in ms)
+                            });
+                            if (kafkaErrorOptions.manualCommit) {
+                                await consumer.commitOffsets([{ topic, partition, offset: (Number(message.offset) + 1).toString() }]);
+                            }
+                        } catch (err) {
+                            if(kafkaErrorOptions.traceAll) {
+                                console.error(err);
+                            }
+                            if(kafkaErrorOptions.retryTopic && kafkaErrorOptions.retryTopic !== '') {
+                                // sends message with error to retry topic
+                                await module.exports.kafkaSend(connection, kafkaErrorOptions.retryTopic, [message]);
+                                // marks the message as read on original topic
+                                await consumer.commitOffsets([{ topic, partition, offset: (Number(message.offset) + 1).toString() }]);
+                           }
+                        }
+                        await runHelper.terminateCallContext(callContext);
+                    }, 
+                    retry: {
+                        retries: kafkaErrorOptions.retries,
+                        initialRetryTime: kafkaErrorOptions.initialRetryTime,
+                        factor: kafkaErrorOptions.factor,
+                        multiplier: kafkaErrorOptions.multiplier,
+                    },
+                    rebalance: {
+                        eachGroupJoin: async ({ groupId, memberId, groupAssignment }) => {
+                            console.log(`KAFKA CONSUMER | ${kafkaConsumerName} | memberId: ${memberId} joined the group ${groupId}`);
+                        },
+                        eachGroupLeave: async ({ groupId, memberId }) => {
+                            console.log(`KAFKA CONSUMER | ${kafkaConsumerName} | memberId: ${memberId} left the group ${groupId}`);
+                        }
+                    }
+                });
+            } catch (err) {
+                runHelper.logError(err);
+            }
+        }        
+        if (process.env.blz_traceAll === 'true' || process.env.blz_traceKafkaConsumers === 'true') {
+            console.log('KAFKA CONSUMER | ' + kafkaConsumerName + ' | Start');
+        }
+        
+    }
+}
+// END: functions to initialize kafka consumers
+
+module.exports = {
+    kafkaConfig: async function (connection, config) {
+        if(!config || !config.topicsConfig || !config.topicsConfig.topics || !config.topicsConfig.topics.length || config.topicsConfig.topics.length === 0) {
+            throw new Error('Invalid config');
+        }
+        for (let i = 0; i < config.topicsConfig.topics.length; i++) {
+            let topic = config.topicsConfig.topics[i];
+            if (!topic.topic || topic.topic.trim().length === 0) {
+                throw new Error(`Invalid topic ${i+1} config`);
+            }
+        }
+        let result = false
+        let error = null;
+        let admin = null;  
+        try {
+            let kafkaConfig = await getConfig(connection);    
+            let kafka = new Kafkajs.Kafka(kafkaConfig);            
+            admin = kafka.admin();
+            await admin.connect();
+            const currentTopics = await admin.listTopics();
+            let newsTopics = config.topicsConfig.topics.filter(topic => !currentTopics.includes(topic.topic));            
+            if(newsTopics && newsTopics.length > 0) {
+                config.topicsConfig.topics = newsTopics;
+                result = await admin.createTopics(config.topicsConfig);
+            } 
+        }catch(e) {
+            error = e;
+        } finally {
+            if(admin) {
+                await admin.disconnect();
+            }  
+        }
+        if(error) {
+            throw error;
+        }
+        return result;                
+    },
+    kafkaGetTopics: async function (connection) {
+        let kafkaConfig = await getConfig(connection);    
+        let kafka = new Kafkajs.Kafka(kafkaConfig);
+        const admin = kafka.admin();
+        await admin.connect();
+        const topics = await admin.listTopics();
+        await admin.disconnect();
+        return topics;
+    },
+    kafkaSend: async function (connection, topic, messages) {
+        if (connection.provider) {
+            await connection.provider.produce(connection, topic, messages);
+            return null;
+        }
+        let producer = await getOrCreateProducer(connection);
+        await producer.send({ topic: topic, messages: messages });
+        return null;
+    },
+    kafkaConnections: async function(kafkaConsumers, connections, buildConfigItems, runHelper){
+        return createKafkaInstances(kafkaConsumers, connections, buildConfigItems, runHelper);
+    },
+    startKafkaConsumers: async function(kafkas,config, kafkaConsumers, connections, runHelper){
+        startKafkaConsumers(kafkas, config, kafkaConsumers, connections, runHelper); 
+    },
+    kafkaConnectionHealthCheck: async function (connection) {
+        if (!connection.brokers) {
+            return { status: "error", message: "Kafka broker list not defined" }
+        }
+        const kafka = new Kafkajs.Kafka({ brokers: [connection.brokers] });
+        try {
+            // Try to get metadata from Kafka to check the connection
+            const admin = kafka.admin();
+            await admin.connect();
+            // Fetch metadata (this will trigger a request to Kafka to see if it's reachable)
+            const metadata = await admin.fetchTopicMetadata({ topics: [] });
+            logger.info({message: "Kafka is reachable!", metadata:metadata});
+            await admin.disconnect();  // Disconnect after metadata check
+            return { status: "ok", message: "Kafka connected, metadata vissible" };
+        } catch (error) {
+            logger.info({ message: "Kafka is not reachable", error: error });
+            return { status: "error", message: error.message };
+        }
+
+    }
+};