mqtt-devices-parser 1.0.18 → 1.0.20

package/Changelog.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## 1.0.20
+	Adds Kafka consumer support with optional MQTT message parsing (#5)
+	supports tables for mqtt templates
+
+## 1.0.19
+	Removes adding project on launching process
+	models/models:
+	 - seedCommonLwM2MObjects: avoids error on description property not defined on object
+	 - changes lwm2m seed functions name
+	 
 ## 1.0.18
 	Fix and improve Sensors, checks uid length
 	Adds seedCommonLwM2MObjects and seedCommonLwM2MResources functions

package/Readme.md

@@ -4,6 +4,7 @@
 ## NPM Module
   - MQTT parser
   - Parses topics coming from devices and updates data on dB
+  - **NEW**: Kafka consumer support for multi-process message parsing
 
 ## Deploy
   ```
@@ -25,6 +26,7 @@
 
 - mysql8.0
 - MQTT
+- **NEW**: Apache Kafka (optional)
 
 ## Front end
 - Node service running the following [project](https://github.com/zimbora/mgmt-iot-web)
@@ -37,3 +39,81 @@ In your program set the same struct and pass it as argument on the module init c
 An mqtt account will be created using the following settings
 mqtt.user - mqtt user login
 mqtt.pwd - mqtt pwd login
+
+### Kafka Consumer Configuration (NEW)
+
+The module now supports consuming messages from Kafka topics as an alternative or complement to MQTT. This enables multi-process message parsing using Kafka's shared subscription mechanism.
+
+#### Environment Variables
+
+**Kafka Settings:**
+- `KAFKA_ENABLED` - Set to `'true'` to enable Kafka consumer (default: `false`)
+- `KAFKA_BROKERS` - Comma-separated list of Kafka brokers (default: `'localhost:9092'`)
+- `KAFKA_GROUP_ID` - Consumer group ID for shared subscriptions (default: `'mqtt-devices-parser-group'`)
+- `KAFKA_CLIENT_ID` - Kafka client identifier (default: `'mqtt-devices-parser'`)
+
+**Authentication (SASL):**
+- `KAFKA_SASL_MECHANISM` - SASL mechanism: `'plain'`, `'scram-sha-256'`, `'scram-sha-512'` (default: `'plain'`)
+- `KAFKA_SASL_USERNAME` - SASL username (default: empty)
+- `KAFKA_SASL_PASSWORD` - SASL password (default: empty)
+
+**Security:**
+- `KAFKA_SSL` - Set to `'true'` to enable SSL (default: `false`)
+
+**Connection Settings:**
+- `KAFKA_CONNECTION_TIMEOUT` - Connection timeout in ms (default: `3000`)
+- `KAFKA_REQUEST_TIMEOUT` - Request timeout in ms (default: `30000`)
+
+**MQTT Parsing Control:**
+- `MQTT_PARSE_MESSAGES` - Set to `'false'` to disable direct MQTT message parsing (default: `true`)
+
+#### Usage Examples
+
+**Example 1: Enable Kafka with basic authentication**
+```bash
+export KAFKA_ENABLED=true
+export KAFKA_BROKERS=kafka1:9092,kafka2:9092
+export KAFKA_SASL_USERNAME=myuser
+export KAFKA_SASL_PASSWORD=mypassword
+export KAFKA_SASL_MECHANISM=scram-sha-256
+```
+
+**Example 2: Use Kafka only (disable MQTT parsing)**
+```bash
+export KAFKA_ENABLED=true
+export MQTT_PARSE_MESSAGES=false
+export KAFKA_BROKERS=kafka-cluster:9092
+```
+
+**Example 3: Multi-process deployment**
+```bash
+# Start multiple instances with the same group ID for load balancing
+export KAFKA_ENABLED=true
+export KAFKA_GROUP_ID=my-parser-group
+export KAFKA_BROKERS=kafka1:9092,kafka2:9092
+# Each process will consume different partitions automatically
+```
+
+#### Kafka Topic Structure
+
+Kafka topics should correspond to project names configured in your `projects` array. Messages are expected to have:
+
+- **Topic**: The project name (e.g., `demo`, `freeRTOS2`)
+- **Key**: Device path part (e.g., `uid:123/status`)
+- **Value**: Message payload (same format as MQTT payloads)
+
+The consumer will automatically format messages as `{project}/{key}` to match the expected MQTT topic structure.
+
+#### Shared Subscriptions
+
+When multiple instances use the same `KAFKA_GROUP_ID`, Kafka automatically distributes messages across consumers, enabling:
+
+- **Load balancing**: Messages are distributed across multiple parser instances
+- **High availability**: If one instance fails, others continue processing
+- **Scalability**: Add more instances to handle increased load
+
+#### Message Flow
+
+1. **MQTT Mode** (traditional): MQTT messages → `$.device.parseMessage`
+2. **Kafka Mode**: Kafka messages → Topic formatting → `$.device.parseMessage`  
+3. **Hybrid Mode**: Both MQTT and Kafka messages → `$.device.parseMessage`

package/config/index.js

@@ -13,7 +13,22 @@ module.exports = {
     user:process.env.MQTT_USER || 'admin',
     pwd:process.env.MQTT_PWD || 'admin',
     client:process.env.MQTT_CLIENT || 'mqtt-devices-parser',
-    logs_path: process.env.MQTT_LOGS || 'uServices'
+    logs_path: process.env.MQTT_LOGS || 'uServices',
+    parseMessages: process.env.MQTT_PARSE_MESSAGES !== 'false' // Default to true for backward compatibility
+  },
+  kafka: {
+    enabled: process.env.KAFKA_ENABLED === 'true' ? true : true,
+    brokers: (process.env.KAFKA_BROKERS || 'localhost:9092').split(','),
+    groupId: process.env.KAFKA_GROUP_ID || 'mqtt-devices-parser-group',
+    clientId: process.env.KAFKA_CLIENT_ID || 'mqtt-devices-parser',
+    sasl: {
+      mechanism: process.env.KAFKA_SASL_MECHANISM || 'plain', // plain, scram-sha-256, scram-sha-512
+      username: process.env.KAFKA_SASL_USERNAME || '',
+      password: process.env.KAFKA_SASL_PASSWORD || ''
+    },
+    ssl: process.env.KAFKA_SSL === 'true' ? true : false,
+    connectionTimeout: parseInt(process.env.KAFKA_CONNECTION_TIMEOUT) || 3000,
+    requestTimeout: parseInt(process.env.KAFKA_REQUEST_TIMEOUT) || 30000
   },
   mysqldb: {
     conn_limit: process.env.DB_CONN_LIMIT || 15,
@@ -27,7 +42,7 @@ module.exports = {
   projects: {
     demo : process.env.demo || false,
   },
-  demo : {
+  demo : { // not need anymore. Can be created on dashboard
     uidPrefix: 'uid:',
     uidLength: 16 // max len for uid
   }

package/example/kafka-example.js

@@ -0,0 +1,181 @@
+/**
+ * Example showing how to use mqtt-devices-parser with Kafka consumer support
+ * 
+ * This example demonstrates:
+ * 1. Traditional MQTT-only mode
+ * 2. Kafka-only mode 
+ * 3. Hybrid mode (both MQTT and Kafka)
+ */
+
+const parser = require('../index.js');
+const fs = require('fs');
+
+global.BASE_DIR = process.cwd();
+
+// Example configuration with Kafka support
+const configWithKafka = {
+  version: "1.0.0",
+  dev: true,
+  web: {
+    protocol: "http://",
+    domain: 'localhost',
+    port: 8002,
+  },
+  mqtt: {
+    protocol: 'MQTT',
+    host: 'localhost',
+    port: '1883',
+    user: 'admin',
+    pwd: 'admin',
+    client: 'mqtt-devices-parser-example',
+    logs_path: 'uServices',
+    parseMessages: true // Set to false to disable MQTT message parsing
+  },
+  kafka: {
+    enabled: true, // Enable Kafka consumer
+    brokers: ['localhost:9092'], // Your Kafka brokers
+    groupId: 'mqtt-devices-parser-group',
+    clientId: 'mqtt-devices-parser-client',
+    sasl: {
+      mechanism: 'plain', // or 'scram-sha-256', 'scram-sha-512'
+      username: '', // Your Kafka username
+      password: ''  // Your Kafka password
+    },
+    ssl: false, // Set to true for SSL/TLS
+    connectionTimeout: 3000,
+    requestTimeout: 30000
+  },
+  mysqldb: {
+    conn_limit: 15,
+    host: 'localhost',
+    port: 3306,
+    user: 'user',
+    pwd: 'user_pwd',
+    name: 'mqtt-aedes',
+  },
+  sync_main_tables: true,
+  projects: {
+    demo: true,
+    // Add your project names here
+    // These will become Kafka topic names when Kafka is enabled
+  },
+  demo: {
+    uidPrefix: 'uid:',
+    uidLength: 16
+  }
+};
+
+// Example: MQTT-only configuration (traditional mode)
+const configMQTTOnly = {
+  ...configWithKafka,
+  kafka: {
+    ...configWithKafka.kafka,
+    enabled: false // Disable Kafka
+  },
+  mqtt: {
+    ...configWithKafka.mqtt,
+    parseMessages: true // Keep MQTT parsing enabled
+  }
+};
+
+// Example: Kafka-only configuration
+const configKafkaOnly = {
+  ...configWithKafka,
+  kafka: {
+    ...configWithKafka.kafka,
+    enabled: true // Enable Kafka
+  },
+  mqtt: {
+    ...configWithKafka.mqtt,
+    parseMessages: false // Disable MQTT parsing
+  }
+};
+
+// Load projects from directories
+var projectsPath = '../projects/';
+var projects = [];
+
+try {
+  fs.readdirSync(projectsPath)
+    .filter((file) => {
+      return (file.indexOf('.') == -1);
+    })
+    .forEach((project) => {
+      if (configWithKafka.projects[project]) {
+        projects.push(project);
+      }
+    });
+} catch (error) {
+  console.log('Projects directory not found, using default projects');
+  projects = ['demo'];
+}
+
+async function runExample() {
+  try {
+    console.log('=== MQTT Devices Parser with Kafka Support Example ===\n');
+    
+    // Choose which configuration to use
+    const selectedConfig = configWithKafka; // Change this to test different modes
+    
+    console.log('Configuration mode:');
+    console.log('- MQTT parsing enabled:', selectedConfig.mqtt.parseMessages);
+    console.log('- Kafka consumer enabled:', selectedConfig.kafka.enabled);
+    console.log('- Projects:', projects.join(', '));
+    console.log();
+    
+    if (selectedConfig.kafka.enabled) {
+      console.log('Kafka configuration:');
+      console.log('- Brokers:', selectedConfig.kafka.brokers.join(', '));
+      console.log('- Group ID:', selectedConfig.kafka.groupId);
+      console.log('- SASL enabled:', !!(selectedConfig.kafka.sasl.username && selectedConfig.kafka.sasl.password));
+      console.log();
+    }
+    
+    // Initialize the parser
+    console.log('Initializing parser...');
+    await parser.init(selectedConfig, projects);
+    
+    console.log('Parser initialized successfully!');
+    console.log();
+    
+    if (selectedConfig.kafka.enabled) {
+      console.log('Kafka consumer is running and listening for messages on topics:', projects.join(', '));
+      console.log('Send messages to these Kafka topics with format:');
+      console.log('- Topic: <project_name> (e.g., "demo")');
+      console.log('- Key: <device_path> (e.g., "uid:123/status")');
+      console.log('- Value: <message_payload> (e.g., "online")');
+      console.log();
+    }
+    
+    if (selectedConfig.mqtt.parseMessages) {
+      console.log('MQTT client is connected and parsing messages');
+      console.log('Subscribed to MQTT topics:', projects.map(p => p + '/#').join(', '));
+      console.log();
+    }
+    
+    console.log('Press Ctrl+C to stop...');
+    
+    // Keep the process running
+    await new Promise(resolve => {
+      process.on('SIGINT', () => {
+        console.log('\nShutting down gracefully...');
+        resolve();
+      });
+    });
+    
+  } catch (error) {
+    console.error('Error running example:', error);
+    process.exit(1);
+  }
+}
+
+// Handle unhandled rejections
+process.on('unhandledRejection', (reason, promise) => {
+  console.error('Unhandled Rejection at:', promise, 'reason:', reason);
+  if (configWithKafka.dev) {
+    process.exit(1);
+  }
+});
+
+// Run the example
+runExample();

package/index.js

@@ -7,6 +7,7 @@ $.md5 = require('md5');
 $.config = require('./config');
 $.models = require('./models/models.js');
 $.device = require('./src/device/device.js');
+$.kafka_consumer = require('./src/kafka/consumer.js');
 $.db = require('./src/db/db');
 $.db_data = require('./src/db/data');
 $.db_project = require('./src/db/project');
@@ -40,6 +41,12 @@ var self = module.exports = {
       //auth.init();
       await $.device.init($.config,projects);
 
+      // Initialize Kafka consumer if enabled
+      if ($.config.kafka.enabled) {
+        await $.kafka_consumer.init($.config, projects);
+        await $.kafka_consumer.start();
+      }
+
       console.log("mqtt connecting..")
       mqtt_connect();
 
@@ -107,6 +114,7 @@ var self = module.exports = {
       // adds client with credentials admin@admin
       await $.models.insertClient("admin","admin",user_id); // dashboard login
       
+      /* Add projects using dashboard !!
       for(name of projects){
         project = $.config[name];
         if (project) {
@@ -115,6 +123,7 @@ var self = module.exports = {
           await $.models.insertProject(projectData);
         }
       }
+      */
     }
 
     // project tables
@@ -179,8 +188,11 @@ function mqtt_connect(){
   });
 
   $.mqtt_client.on("message", (topic, payload, packet) => {
-    // payload is Buffer
-    $.device.parseMessage($.mqtt_client,topic.toString(),payload.toString(),packet.retain);
+    // Only parse MQTT messages if enabled in configuration
+    if ($.config.mqtt.parseMessages) {
+      // payload is Buffer
+      $.device.parseMessage($.mqtt_client,topic.toString(),payload.toString(),packet.retain);
+    }
   });
 
   $.mqtt_client.on("reconnect",()=>{

package/models/models.js

@@ -253,7 +253,7 @@ var self = module.exports = {
     });
   },
 
-  seedCommonLwM2MObjects: async(objects) => {
+  seedLwM2MObjects: async(objects) => {
     try {
       const results = await Promise.all(
         objects.map(async (object) => {
@@ -261,7 +261,7 @@ var self = module.exports = {
             {
               objectId: object.objectId,
               name: object.name,
-              description: object.description,
+              description: object?.description,
             },
             {
               where: { objectId: object.objectId }, // Ensure the record exists based on objectId
@@ -317,7 +317,7 @@ var self = module.exports = {
  * @param {import('sequelize').Sequelize} sequelize - An initialized Sequelize instance.
  * @returns {Promise<Array<{ objectId:number, resourceId:number, created:boolean, updated:boolean }>>}
  */
-  seedCommonLwM2MResources: async (resources) => {
+  seedLwM2MResources: async (resources) => {
     if (!sequelize || !sequelize.models || !sequelize.models['lwm2mResources']) {
       throw new Error("sequelize instance with model 'lwm2mResources' is required");
     }

package/models/mqtt.models.js

@@ -0,0 +1,41 @@
+
+module.exports = (sequelize,DataTypes)=>{
+	return sequelize.define("mqtt", {
+		device_id: {
+			type: DataTypes.INTEGER,
+			allowNull: true
+		},
+		topic: {
+			type: DataTypes.STRING,
+			allowNull: false
+		},
+		description: {
+			type: DataTypes.JSON,
+			allowNull: false
+		},
+		defaultData: {
+			type: DataTypes.JSON,
+			allowNull: true	
+		},
+		remoteData: {
+			type: DataTypes.JSON,
+			allowNull: true	
+		},
+		localData: {
+			type: DataTypes.JSON,
+			allowNull: true	
+		},
+		readInterval: {
+			type: DataTypes.INTEGER,
+			allowNull: true
+		},
+		template_id: {
+			type: DataTypes.INTEGER,
+			allowNull: true,
+		}
+	},
+	{
+		tableName: 'mqtt',
+		freezeTableName: true
+	})
+}

package/models/mqttTemplate.models.js

@@ -0,0 +1,29 @@
+
+module.exports = (sequelize,DataTypes)=>{
+	return sequelize.define("mqttTemplate", {
+		topic: {
+			type: DataTypes.STRING,
+			allowNull: false
+		},
+		description: {
+			type: DataTypes.JSON,
+			allowNull: false
+		},
+		defaultData: {
+			type: DataTypes.JSON,
+			allowNull: true	
+		},
+		readInterval: {
+			type: DataTypes.INTEGER,
+			allowNull: true
+		},
+		template_id: {
+			type: DataTypes.INTEGER,
+			allowNull: false
+		},
+	},
+	{
+		tableName: 'mqttTemplate',
+		freezeTableName: true
+	})
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mqtt-devices-parser",
-  "version": "1.0.18",
+  "version": "1.0.20",
   "description": "",
   "main": "index.js",
   "scripts": {
@@ -11,6 +11,7 @@
   "license": "ISC",
   "dependencies": {
     "async": "^3.2.4",
+    "kafkajs": "^2.2.4",
     "lodash": "^4.17.21",
     "md5": "^2.3.0",
     "moment": "^2.29.4",

package/src/device/device.js

@@ -35,6 +35,10 @@ var self = module.exports = {
 
   parseMessage : async (client,topic,payload,retain)=>{
 
+    try{
+      payload = JSON.parse(payload)
+    }catch(err){};
+    
     let topic_bck = topic;
 
     //let index = topic.indexOf(MACRO_UID_PREFIX);

package/src/kafka/consumer.js

@@ -0,0 +1,176 @@
+const { Kafka, logLevel } = require('kafkajs');
+
+var kafka = null;
+var consumer = null;
+var running = false;
+
+var self = module.exports = {
+
+  init: async (config, projects) => {
+
+    return new Promise(async (resolve, reject) => {
+      try {
+        if (!config.kafka.enabled) {
+          console.log("Kafka consumer is disabled");
+          return resolve();
+        }
+
+        console.log("Initializing Kafka consumer...");
+
+        // Build Kafka configuration
+        const kafkaConfig = {
+          clientId: config.kafka.clientId,
+          brokers: config.kafka.brokers,
+          connectionTimeout: config.kafka.connectionTimeout,
+          requestTimeout: config.kafka.requestTimeout,
+          logLevel: logLevel.INFO
+        };
+
+        // Add SASL authentication if credentials provided
+        if (config.kafka.sasl.username && config.kafka.sasl.password) {
+          kafkaConfig.sasl = {
+            mechanism: config.kafka.sasl.mechanism,
+            username: config.kafka.sasl.username,
+            password: config.kafka.sasl.password
+          };
+        }
+
+        // Add SSL configuration if enabled
+        if (config.kafka.ssl) {
+          kafkaConfig.ssl = true;
+        }
+
+        kafka = new Kafka(kafkaConfig);
+
+        // Create consumer with shared subscription support
+        consumer = kafka.consumer({
+          groupId: config.kafka.groupId,
+          sessionTimeout: 30000,
+          rebalanceTimeout: 60000,
+          heartbeatInterval: 3000,
+          maxBytesPerPartition: 1048576,
+          minBytes: 1,
+          maxBytes: 10485760,
+          maxWaitTimeInMs: 5000,
+          retry: {
+            initialRetryTime: 100,
+            retries: 8
+          }
+        });
+
+        // Subscribe to project topics
+        const topics = projects.map(project => project.toString());
+        
+        if (topics.length === 0) {
+          console.log("No Kafka topics to subscribe to");
+          return resolve();
+        }
+
+        await consumer.subscribe({ 
+          topics: topics,
+          fromBeginning: false 
+        });
+
+        console.log(`Kafka consumer subscribed to topics: ${topics.join(', ')}`);
+
+        return resolve();
+      } catch (error) {
+        console.error("Failed to initialize Kafka consumer:", error);
+        return reject(error);
+      }
+    });
+  },
+
+  start: async () => {
+    if (!consumer || running) {
+      return;
+    }
+
+    try {
+      console.log("Starting Kafka consumer...");
+      running = true;
+
+      await consumer.run({
+        eachMessage: async ({ topic, partition, message }) => {
+          try {
+            let payload = message.value ? message.value.toString() : '';
+            const kafkaTopic = topic.toString();
+            
+            // Format message to look like MQTT format for compatibility
+            // Kafka topic becomes the project name, message key becomes the device path
+            const messageKey = message.key ? message.key.toString() : '';
+            
+            let formattedTopic;
+            if (messageKey) {
+              // If message has a key, use it as the device path: project/device_path
+              formattedTopic = `${kafkaTopic}/${messageKey}`;
+            } else {
+              // If no key, try to extract from headers or use a default format
+              const headers = message.headers || {};
+              const devicePath = headers.device_path ? headers.device_path.toString() : 'unknown';
+              formattedTopic = `${kafkaTopic}/${devicePath}`;
+            }
+
+            try{
+              payload = JSON.parse(payload);
+            }catch(error){}
+
+            // Call the device parser with formatted topic
+            await $.device.parseMessage(payload.client.id, formattedTopic, payload.mqtt.payload, payload.mqtt.retain);
+
+          } catch (error) {
+            console.error('[Kafka] Error processing message:', {
+              topic: topic.toString(),
+              partition,
+              offset: message.offset ? message.offset.toString() : 'unknown',
+              error: error.message
+            });
+          }
+        },
+      });
+
+      console.log("Kafka consumer started successfully");
+    } catch (error) {
+      console.error("Failed to start Kafka consumer:", error);
+      running = false;
+      throw error;
+    }
+  },
+
+  stop: async () => {
+    if (!consumer || !running) {
+      return;
+    }
+
+    try {
+      console.log("Stopping Kafka consumer...");
+      running = false;
+      await consumer.stop();
+      await consumer.disconnect();
+      console.log("Kafka consumer stopped successfully");
+    } catch (error) {
+      console.error("Error stopping Kafka consumer:", error);
+    }
+  },
+
+  isRunning: () => {
+    return running;
+  }
+};
+
+// Graceful shutdown handling
+process.on('SIGINT', async () => {
+  if (running) {
+    console.log('Received SIGINT, gracefully shutting down Kafka consumer...');
+    await self.stop();
+  }
+  process.exit(0);
+});
+
+process.on('SIGTERM', async () => {
+  if (running) {
+    console.log('Received SIGTERM, gracefully shutting down Kafka consumer...');
+    await self.stop();
+  }
+  process.exit(0);
+});

package/src/unitTest/kafka-integration.test.js

@@ -0,0 +1,118 @@
+const fs = require('fs');
+
+$ = {};
+var path = "/../..";
+$.config = require(__dirname + path + '/config');
+
+// Mock device module to prevent actual parsing
+$.device = {
+  parseMessage: async (client, topic, payload, retain) => {
+    console.log('Mock parseMessage called:', { topic, payload, retain });
+    return Promise.resolve();
+  }
+};
+
+async function testConfigurationValues() {
+  console.log('\n=== Testing Configuration Values ===');
+  
+  // Test default values
+  console.log('Default MQTT parseMessages:', $.config.mqtt.parseMessages);
+  console.log('Default Kafka enabled:', $.config.kafka.enabled);
+  console.log('Default Kafka brokers:', $.config.kafka.brokers);
+  console.log('Default Kafka groupId:', $.config.kafka.groupId);
+  console.log('Default Kafka SASL mechanism:', $.config.kafka.sasl.mechanism);
+  
+  // Test with environment variables
+  process.env.MQTT_PARSE_MESSAGES = 'false';
+  process.env.KAFKA_ENABLED = 'true';
+  process.env.KAFKA_BROKERS = 'broker1:9092,broker2:9092';
+  process.env.KAFKA_GROUP_ID = 'test-group';
+  process.env.KAFKA_SASL_MECHANISM = 'scram-sha-256';
+  process.env.KAFKA_SASL_USERNAME = 'testuser';
+  process.env.KAFKA_SASL_PASSWORD = 'testpass';
+  
+  // Reload config to pick up env variables
+  delete require.cache[require.resolve(__dirname + path + '/config')];
+  const newConfig = require(__dirname + path + '/config');
+  
+  console.log('\nWith environment variables:');
+  console.log('MQTT parseMessages:', newConfig.mqtt.parseMessages);
+  console.log('Kafka enabled:', newConfig.kafka.enabled);
+  console.log('Kafka brokers:', newConfig.kafka.brokers);
+  console.log('Kafka groupId:', newConfig.kafka.groupId);
+  console.log('Kafka SASL mechanism:', newConfig.kafka.sasl.mechanism);
+  console.log('Kafka SASL username:', newConfig.kafka.sasl.username);
+  console.log('Kafka SASL password:', newConfig.kafka.sasl.password ? '[SET]' : '[NOT SET]');
+  
+  console.log('✓ Configuration values test passed!');
+}
+
+async function testKafkaConsumerModule() {
+  console.log('\n=== Testing Kafka Consumer Module Structure ===');
+  
+  try {
+    const kafkaConsumer = require(__dirname + path + '/src/kafka/consumer.js');
+    
+    // Test module exports
+    console.log('Module has init method:', typeof kafkaConsumer.init === 'function');
+    console.log('Module has start method:', typeof kafkaConsumer.start === 'function');
+    console.log('Module has stop method:', typeof kafkaConsumer.stop === 'function');
+    console.log('Module has isRunning method:', typeof kafkaConsumer.isRunning === 'function');
+    
+    // Test initial state
+    console.log('Initial running state:', kafkaConsumer.isRunning());
+    
+    // Test with disabled Kafka
+    const configDisabled = { kafka: { enabled: false } };
+    await kafkaConsumer.init(configDisabled, ['test']);
+    console.log('Init with disabled config succeeded');
+    
+    console.log('✓ Kafka consumer module structure test passed!');
+  } catch (error) {
+    console.error('✗ Kafka consumer module test failed:', error.message);
+  }
+}
+
+async function testMQTTMessageParsing() {
+  console.log('\n=== Testing MQTT Message Parsing Logic ===');
+  
+  // Test the logic that would be used in index.js
+  const testConfigs = [
+    { mqtt: { parseMessages: true } },
+    { mqtt: { parseMessages: false } }
+  ];
+  
+  for (const config of testConfigs) {
+    console.log(`\nTesting with parseMessages: ${config.mqtt.parseMessages}`);
+    
+    // Simulate the MQTT message handler logic
+    const mockTopic = "test/uid:123/status";
+    const mockPayload = "online";
+    const mockPacket = { retain: false };
+    
+    if (config.mqtt.parseMessages) {
+      console.log('Would call device.parseMessage');
+      await $.device.parseMessage(null, mockTopic, mockPayload, mockPacket.retain);
+    } else {
+      console.log('Would skip device.parseMessage');
+    }
+  }
+  
+  console.log('✓ MQTT message parsing logic test passed!');
+}
+
+async function runTests() {
+  console.log('Starting Kafka Integration Tests...');
+  
+  await testConfigurationValues();
+  await testKafkaConsumerModule();
+  await testMQTTMessageParsing();
+  
+  console.log('\n=== All Tests Completed Successfully ===');
+  process.exit(0);
+}
+
+runTests().catch(error => {
+  console.error('Test suite failed:', error);
+  process.exit(1);
+});

package/src/unitTest/kafka.test.js

@@ -0,0 +1,72 @@
+const fs = require('fs');
+
+$ = {};
+var path = "/../..";
+$.config = require(__dirname + path + '/config');
+$.kafka_consumer = require(__dirname + path + '/src/kafka/consumer.js');
+$.device = {
+  parseMessage: async (client, topic, payload, retain) => {
+    console.log('Mock parseMessage called:', { topic, payload, retain });
+  }
+};
+
+async function testKafkaConsumerInit() {
+  console.log('\n=== Testing Kafka Consumer Initialization ===');
+  
+  try {
+    // Test with Kafka disabled
+    console.log('1. Testing with Kafka disabled...');
+    const configDisabled = { ...$.config, kafka: { ...$.config.kafka, enabled: false } };
+    await $.kafka_consumer.init(configDisabled, ['test-project']);
+    console.log('✓ Kafka consumer init with disabled config passed');
+    
+    // Test with Kafka enabled but no authentication
+    console.log('2. Testing with Kafka enabled (no auth)...');
+    const configEnabled = { 
+      ...$.config, 
+      kafka: { 
+        ...$.config.kafka, 
+        enabled: true,
+        sasl: { username: '', password: '' }
+      } 
+    };
+    await $.kafka_consumer.init(configEnabled, ['test-project']);
+    console.log('✓ Kafka consumer init with enabled config passed');
+    
+    // Test isRunning method
+    console.log('3. Testing isRunning method...');
+    console.log('Is running:', $.kafka_consumer.isRunning());
+    console.log('✓ isRunning method works');
+    
+    console.log('\n✓ All Kafka consumer tests passed!');
+  } catch (error) {
+    console.error('✗ Kafka consumer test failed:', error.message);
+  }
+}
+
+async function testConfiguration() {
+  console.log('\n=== Testing Configuration ===');
+  
+  console.log('MQTT parseMessages:', $.config.mqtt.parseMessages);
+  console.log('Kafka enabled:', $.config.kafka.enabled);
+  console.log('Kafka brokers:', $.config.kafka.brokers);
+  console.log('Kafka groupId:', $.config.kafka.groupId);
+  console.log('Kafka SASL mechanism:', $.config.kafka.sasl.mechanism);
+  
+  console.log('✓ Configuration test passed!');
+}
+
+async function runTests() {
+  console.log('Starting Kafka Consumer Tests...');
+  
+  await testConfiguration();
+  await testKafkaConsumerInit();
+  
+  console.log('\n=== All Tests Completed ===');
+  process.exit(0);
+}
+
+runTests().catch(error => {
+  console.error('Test suite failed:', error);
+  process.exit(1);
+});