mqtt-devices-parser 1.0.20 → 1.0.22

package/Changelog.md

@@ -1,5 +1,21 @@
 # Changelog
 
+## 1.0.22
+	Adds SSL and SASL support for kafka
+		mqtt.parseMessages removed
+	src/kafka/consumer: adds random number to kafka client Id
+	Parse mqtt lwm2m (#6)
+		* in progress: splits parsing messages base on protocol
+		* supports local unix timestamp, fix sensor data parse
+		models/logs_sensor.models: add localUnixTs
+		src/devices/device: fix object parse
+		* fix: mqtt parser
+		Re-structs parser
+
+## 1.0.21
+	src/kafka/consumer: fixes parseMessage call
+	improves: logs
+	
 ## 1.0.20
 	Adds Kafka consumer support with optional MQTT message parsing (#5)
 	supports tables for mqtt templates

package/Readme.md

@@ -26,7 +26,7 @@
 
 - mysql8.0
 - MQTT
-- **NEW**: Apache Kafka (optional)
+- **NEW**: Confluentic Kafka (optional)
 
 ## Front end
 - Node service running the following [project](https://github.com/zimbora/mgmt-iot-web)
@@ -58,7 +58,7 @@ The module now supports consuming messages from Kafka topics as an alternative o
 - `KAFKA_SASL_PASSWORD` - SASL password (default: empty)
 
 **Security:**
-- `KAFKA_SSL` - Set to `'true'` to enable SSL (default: `false`)
+- `KAFKA_SSL_ENABLED` - Set to `'true'` to enable SSL (default: `false`)
 
 **Connection Settings:**
 - `KAFKA_CONNECTION_TIMEOUT` - Connection timeout in ms (default: `3000`)
@@ -69,23 +69,47 @@ The module now supports consuming messages from Kafka topics as an alternative o
 
 #### Usage Examples
 
-**Example 1: Enable Kafka with basic authentication**
+**Example 1: Use Kafka with SSL and SCRAM-SHA-256 authentication - Most Secure**
 ```bash
-export KAFKA_ENABLED=true
+export KAFKA_SSL_ENABLED=true
+export KAFKA_SSL_CERTFILE='./certs/ca.cert'
+export KAFKA_SSL_REJECT_UNAUTHORIZED=true
+export KAFKA_BROKERS=kafka1:9092,kafka2:9092
+export KAFKA_SASL_USERNAME=myuser
+export KAFKA_SASL_PASSWORD=mypassword
+export KAFKA_SASL_MECHANISM=scram-sha-256
+```
+**Example 2: Use Kafka with SSL No trust and SCRAM-SHA-256 authentication.
+Prone to MITM attacks**
+```bash
+export KAFKA_SSL_ENABLED=true
+export KAFKA_SSL_CERTFILE=''
+export KAFKA_SSL_REJECT_UNAUTHORIZED=false
+export KAFKA_BROKERS=kafka1:9092,kafka2:9092
+export KAFKA_SASL_USERNAME=myuser
+export KAFKA_SASL_PASSWORD=mypassword
+export KAFKA_SASL_MECHANISM=scram-sha-256
+```
+**Example 3: Use Kafka without SSL and SCRAM-SHA-256 authentication. Basic authentication with no encryption. Credentials are not shared**
+```bash
+export KAFKA_SSL_ENABLED=false
+export KAFKA_SSL_CERTFILE=''
+export KAFKA_SSL_REJECT_UNAUTHORIZED=true
 export KAFKA_BROKERS=kafka1:9092,kafka2:9092
 export KAFKA_SASL_USERNAME=myuser
 export KAFKA_SASL_PASSWORD=mypassword
 export KAFKA_SASL_MECHANISM=scram-sha-256
 ```
 
-**Example 2: Use Kafka only (disable MQTT parsing)**
+**Example 4: Use KAFKA without authentication. Not secure at all**
 ```bash
 export KAFKA_ENABLED=true
-export MQTT_PARSE_MESSAGES=false
-export KAFKA_BROKERS=kafka-cluster:9092
+export KAFKA_SSL_ENABLED=false
+export KAFKA_SASL_USERNAME=''
+export KAFKA_SASL_PASSWORD=''
 ```
 
-**Example 3: Multi-process deployment**
+**Example 5: KAFKA Multi-process deployment**
 ```bash
 # Start multiple instances with the same group ID for load balancing
 export KAFKA_ENABLED=true
@@ -94,6 +118,11 @@ export KAFKA_BROKERS=kafka1:9092,kafka2:9092
 # Each process will consume different partitions automatically
 ```
 
+**Example 6: Use MQTT only (disable KAFKA parsing)**
+```bash
+export KAFKA_ENABLED=false
+```
+
 #### Kafka Topic Structure
 
 Kafka topics should correspond to project names configured in your `projects` array. Messages are expected to have:

package/config/index.js

@@ -14,19 +14,22 @@ module.exports = {
     pwd:process.env.MQTT_PWD || 'admin',
     client:process.env.MQTT_CLIENT || 'mqtt-devices-parser',
     logs_path: process.env.MQTT_LOGS || 'uServices',
-    parseMessages: process.env.MQTT_PARSE_MESSAGES !== 'false' // Default to true for backward compatibility
   },
   kafka: {
     enabled: process.env.KAFKA_ENABLED === 'true' ? true : true,
     brokers: (process.env.KAFKA_BROKERS || 'localhost:9092').split(','),
     groupId: process.env.KAFKA_GROUP_ID || 'mqtt-devices-parser-group',
     clientId: process.env.KAFKA_CLIENT_ID || 'mqtt-devices-parser',
+    ssl: {
+      enabled : process.env.KAFKA_SSL_ENABLED || false,
+      file: process.env.KAFKA_SSL_CERTFILE || './certs/kafka.crt',
+      rejectUnauthorized: process.env.KAFKA_SSL_REJECT_UNAUTHORIZED || true, // set to false to accept unstrusted certs
+    },
     sasl: {
       mechanism: process.env.KAFKA_SASL_MECHANISM || 'plain', // plain, scram-sha-256, scram-sha-512
       username: process.env.KAFKA_SASL_USERNAME || '',
       password: process.env.KAFKA_SASL_PASSWORD || ''
     },
-    ssl: process.env.KAFKA_SSL === 'true' ? true : false,
     connectionTimeout: parseInt(process.env.KAFKA_CONNECTION_TIMEOUT) || 3000,
     requestTimeout: parseInt(process.env.KAFKA_REQUEST_TIMEOUT) || 30000
   },

package/example/kafka-example.js

@@ -28,8 +28,6 @@ const configWithKafka = {
     user: 'admin',
     pwd: 'admin',
     client: 'mqtt-devices-parser-example',
-    logs_path: 'uServices',
-    parseMessages: true // Set to false to disable MQTT message parsing
   },
   kafka: {
     enabled: true, // Enable Kafka consumer
@@ -74,7 +72,6 @@ const configMQTTOnly = {
   },
   mqtt: {
     ...configWithKafka.mqtt,
-    parseMessages: true // Keep MQTT parsing enabled
   }
 };
 
@@ -87,7 +84,6 @@ const configKafkaOnly = {
   },
   mqtt: {
     ...configWithKafka.mqtt,
-    parseMessages: false // Disable MQTT parsing
   }
 };
 
@@ -118,7 +114,6 @@ async function runExample() {
     const selectedConfig = configWithKafka; // Change this to test different modes
     
     console.log('Configuration mode:');
-    console.log('- MQTT parsing enabled:', selectedConfig.mqtt.parseMessages);
     console.log('- Kafka consumer enabled:', selectedConfig.kafka.enabled);
     console.log('- Projects:', projects.join(', '));
     console.log();
@@ -147,7 +142,7 @@ async function runExample() {
       console.log();
     }
     
-    if (selectedConfig.mqtt.parseMessages) {
+    if (!selectedConfig.kafka.enabled) {
       console.log('MQTT client is connected and parsing messages');
       console.log('Subscribed to MQTT topics:', projects.map(p => p + '/#').join(', '));
       console.log();

package/index.js

@@ -47,7 +47,6 @@ var self = module.exports = {
         await $.kafka_consumer.start();
       }
 
-      console.log("mqtt connecting..")
       mqtt_connect();
 
       setInterval(async ()=>{
@@ -133,6 +132,8 @@ var self = module.exports = {
 
 function mqtt_connect(){
 
+  console.log(`[MQTT] connecting to ${$.config.mqtt.host}:${$.config.mqtt.port}..`)
+
   const mqtt_prefix = $.config.mqtt.logs_path+"/"+$.config.mqtt.client;
   let checkFota = null;
   $.mqtt_client = mqtt.connect({
@@ -165,11 +166,21 @@ function mqtt_connect(){
           $.mqtt_client.publish(mqtt_prefix+"/"+project,"deactive",{qos:2,retain:true});
       };
     })
-    console.log(`MQTT connected to: ${$.config.mqtt.host}:${$.config.mqtt.port}`);
-    projects.map( project=>{
-      console.log("subscribing project:",project);
-      $.mqtt_client.subscribe(project+"/#")
-    })
+    console.log(`[MQTT] connected`);
+
+    if (!$.config.kafka.enabled) {
+      projects.map( project=>{
+        console.log("[MQTT] subscribing project:",project);
+        $.mqtt_client.subscribe(project+"/#", (err) => {
+          if(err){
+            console.log("[MQTT] error");
+            console.err(err);
+          }
+          else
+            console.log("[MQTT] subscribed to project:",project);
+        })
+      })
+    }
 
     // Update 'dev' devices at each 5 min 
     checkFota = setInterval(async ()=>{
@@ -189,32 +200,36 @@ function mqtt_connect(){
 
   $.mqtt_client.on("message", (topic, payload, packet) => {
     // Only parse MQTT messages if enabled in configuration
-    if ($.config.mqtt.parseMessages) {
+    if (!$.config.kafka.enabled) {
+
+      console.log(`[MQTT] rx: ${topic.toString()}`);
       // payload is Buffer
       $.device.parseMessage($.mqtt_client,topic.toString(),payload.toString(),packet.retain);
     }
   });
 
   $.mqtt_client.on("reconnect",()=>{
-    console.log("reconnect")
+    console.log("[MQTT] reconnected")
   });
 
   $.mqtt_client.on("close",()=>{
-    console.log("close")
+    console.log("[MQTT] closed")
     checkFota = null;
   });
 
   $.mqtt_client.on("offline",()=>{
-    console.log("offline")
+    console.log("[MQTT] offline")
     checkFota = null;
   });
 
   $.mqtt_client.on("disconnect",(packet)=>{
-    console.log(packet)
+    console.log(`[MQTT] disconnected`);
+    console.log(packet);
     checkFota = null;
   })
 
   $.mqtt_client.on("error",(error)=>{
-    console.log(error)
+    console.log("[MQTT] error");
+    console.err(error)
   })
 }

package/models/logs_sensor.models.js

@@ -30,6 +30,10 @@ module.exports = (sequelize,DataTypes)=>{
 			type: DataTypes.STRING,
 			allowNull: true,
 		},
+		localUnixTs: {
+			type: DataTypes.BIGINT,
+			allowNull: true,
+		}
 	},
 	{
 		tableName: 'logs_sensor',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mqtt-devices-parser",
-  "version": "1.0.20",
+  "version": "1.0.22",
   "description": "",
   "main": "index.js",
   "scripts": {
@@ -21,6 +21,6 @@
     "path": "^0.12.7",
     "semver": "^7.3.8",
     "sequelize": "^6.32.1",
-    "websocket-stream": "^5.5.2"
+    "websocket-stream": "^5.3.0"
   }
 }

package/src/db/sensor.js

@@ -98,6 +98,7 @@ var self = module.exports = {
 				value : payload?.value,
 				error : payload?.error,
 				obj : payload?.object,
+				localUnixTs : payload?.timestamp, // local timestamp
 				createdAt : moment().utc().format('YYYY-MM-DD HH:mm:ss'),
 				updatedAt : moment().utc().format('YYYY-MM-DD HH:mm:ss')
 			}

package/src/device/device.js

@@ -35,175 +35,49 @@ var self = module.exports = {
 
   parseMessage : async (client,topic,payload,retain)=>{
 
+    let topic_bck = topic;
+    let project_name = null;
+    let project = null;
+    let project_id = null;
+    let action = null;
+    let uid = null;
+    let device = null;
+
     try{
       payload = JSON.parse(payload)
     }catch(err){};
     
-    let topic_bck = topic;
-
-    //let index = topic.indexOf(MACRO_UID_PREFIX);
     // --- project ---
-    let index = topic.indexOf("/");
-    if(index == -1)
-        return;
-
-    let project_name = topic.substring(0,index);
-    let project = await $.db_project.getByName(project_name);
-
+    project_name = getFirstWord(topic);
+    project = await $.db_project.getByName(project_name);
     if(project == null)
       return;
-    
-    let project_id = project?.id;
 
-    // --- uid ---
-    topic = topic.substring(index+1);
-    index = topic.indexOf("/");
-    if(index == -1)
-        return;
+    project_id = project?.id;
+    topic = getWordAfterSlash(topic)
 
-    let uid = topic.substring(0,index);
-    topic = topic.substring(index+1);
+    // check if is from lwm2m gw
+    if(topic.startsWith("responses") || topic.startsWith("requests")){
+      action = getFirstWord(topic);
+      topic = getWordAfterSlash(topic);
+    }
 
+    // --- uid ---
+    uid = getFirstWord(topic);
     // check if topic corresponds to a device
-    if(uid.startsWith(project.uidPrefix) && uid.length <= project?.uidLength){
-
-      let device = await $.db_device.get(uid);
-
-      // Insert device if not exists on db
-      if(device == null){
-        let obj = {
-          uid : uid,
-          accept_release : "prod",
-        }
-        $.db_device.insert(obj)
-        .then(async()=>{
-          device = await $.db_device.get(uid);
-        })
-        .catch( (err) => {});
-      }
-
-      // update project id if needed
-      if(project_id != null && project_id != device?.project_id){
-        $.db_device.update(device.id,"project_id",project_id);
-        $.db_device.addLog(device.id,"project_id",project_id);
-      } 
-
-      switch (topic) {
-        case "status":
-          if (payload === "online" || payload === "offline") {
-            await $.db_device.update(device.id, "status", payload);
-            $.db_device.addLog(device.id,"status",payload);
-          }
-          break;
-        case "model":
-          let res = await $.db_model.getByName(payload);
-          let model_id = res?.id;
-          if (model_id != null && device?.tech != payload) {
-            await $.db_device.update(device.id, "model_id", model_id);
-            $.db_device.addLog(device.id,"model_id",model_id);
-          }
-          break;
-        case "tech":
-          if (device?.tech && payload != device.tech) {
-            await $.db_device.update(device.id, "tech", payload);
-            $.db_device.addLog(device.id,"tech",payload);
-          }
-          break;
-        case "version":
-          if (device?.version && payload != device.version) {
-            $.db_device.addLog(device.id,"version",payload);
-            await $.db_device.update(device.id, "version", payload);
-            await self.handleFotaSuccess(device.id);
-          }
-          break;
-        case "app_version":
-          if (device?.app_version && payload != device.app_version) {
-            $.db_device.addLog(device.id,"app_version",payload);
-            await $.db_device.update(device.id, "app_version", payload);
-            await self.handleFotaSuccess(device.id);
-          }
-          break;
-        case "fw/fota/update/status":
-          await self.handleFotaError(device.id, payload);
-          break;
-        // Optional: default case if needed
-        default:
-          // handle other topics or do nothing
-          break;
-      }
-
-      if(topic.startsWith("settings/") && topic.endsWith("/set") && payload != "" ){
-        // update local settings
-        let index = topic.indexOf("settings/");
-        topic = topic.substring(index+9);
-        self.updateLocalSettings(device,topic,payload);
-      }else if(topic.startsWith("settings/") && !topic.endsWith("/set") && payload != "" ){
-        // store remote device settings
-        let index = topic.indexOf("settings/");
-        topic = topic.substring(index+9);
-        self.updateRemoteSettings(device,topic,payload);
-      }else if(topic == "fw"){
-        try{
-          payload = JSON.parse(payload);
-        }catch(error){}
-        if(typeof payload === 'object' && payload !== null){
-          try{
-            $.db_data.updateJson("fw",device.id,payload);
-            $.db_data.addJsonLog("logs_fw",device.id,payload);
-          }catch(error){
-            console.error(error)
-          } 
-        }else if(typeof payload !== 'object' && payload !== null){
-          // change it to topic.startsWith("fw")
-          try{
-            const column = getWordAfterLastSlash(topic);
-            $.db_data.update("fw",device.id,column,payload);
-            $.db_data.addLog("logs_fw",device.id,column,payload);
-          }catch(error){
-            console.error(error)
-          } 
-        }
-      }else if(topic.startsWith("sensor/")){
-        //updateSensor(device,topic,payload)
-        index = topic.indexOf("/");
-        if(index != -1){
-          const ref = topic.substring(index+1);
-          let res = await $.db_device.getSensorByRef(device.id,ref)
-          if(res == null)
-            res = await $.db_model.getSensorByRef(device.model_id,ref)
-          if(res != null){
-            object = null;
-            value = null;
-            error = null;
-            try{
-              object = JSON.parse(payload);
-            }catch(err){
-              console.log(err);
-            }
-            if (typeof object === 'object' && object !== null) {
-              value = object?.value;
-              error = object?.error;
-            }else{
-              value = payload;
-            }
-
-            if(value || error)
-              object = null;
+    if(!uid.startsWith(project.uidPrefix) || uid.length > project?.uidLength)
+      return;
 
-            const data = {
-              object,
-              value,
-              error
-            }
+    device = await $.db_device.get(uid);
+    if(!device)
+      return;
 
-            await $.db_sensor.insert(logs_table,device.id,res.id,data);
-          }
-        }
-      }
+    topic = getWordAfterSlash(topic);
 
-      if(_project[project_name]){
-        _project[project_name]?.module?.parseMessage(client,project_name,device,topic,payload,retain,()=>{});
-      }
+    if(device.protocol.toLowerCase() === "lwm2m"){
+      parseLwm2mMessage(client, project_name, device, topic, payload, action);
+    }else if(device.protocol.toLowerCase() === "mqtt"){
+      parseMqttMessage(client, project_name, device, topic, payload, retain);
     }
 
   },
@@ -226,118 +100,6 @@ var self = module.exports = {
     }
   },
 
-  updateLocalSettings: async (device, topic, payload) => {
-
-    $.db_device.addLog(device.id,"local_settings",JSON.stringify(payload));
-
-    let route = topic.split("/");
-
-    if (route == null || route.length == 0) {
-      console.warn("updateLocalSettings: topic invalid:", topic);
-      return;
-    }
-
-    // Retrieve existing settings
-    let settings = await $.db_device.getLocalSettings(device.id);
-
-    if (!settings || typeof settings !== 'object') {
-      settings = {};
-    }
-
-    let obj = settings;
-
-    // Traverse route parts to reach the target object
-    route.slice(0, -1).forEach(property => {
-      if (
-        !Object.prototype.hasOwnProperty.call(obj, property) ||
-        (obj.hasOwnProperty(property) && typeof obj[property] !== 'object') ||
-        obj[property] === null
-      ) {
-        obj[property] = {}; // Create nested object if missing or not an object
-      }
-      obj = obj[property];
-    });
-
-    // Parse the payload JSON
-    let data = {};
-    try {
-      data = JSON.parse(payload);
-    } catch (error) {
-      console.error("Failed to parse payload JSON:", error);
-      return;
-    }
-
-    // Check if the data is a plain object, then merge
-    if (data && typeof data === 'object' && !Array.isArray(data)) {
-      Object.assign(obj, data);
-    } else {
-      console.warn("Payload is not a valid object:", payload);
-      return;
-    }
-
-    try {
-      // Update the settings in the database
-      await $.db_device.updateLocalSettings(JSON.stringify(settings), device.id);
-    } catch (err) {
-      console.error("Failed to update local settings:", err);
-    }
-  },
-
-  updateRemoteSettings : async (device,topic,payload)=>{
-
-    $.db_device.addLog(device.id,"remote_settings",JSON.stringify(payload));
-
-    let route = topic.split("/");
-
-    if(route == null){
-      console.warn("updateRemoteSettings: topic invalid:",topic);
-      return;
-    }
-    // Parse existing settings
-    let settings = await $.db_device.getRemoteSettings(device.id);
-
-    if (!settings || typeof settings !== 'object') {
-      settings = {};
-    }
-
-    let obj = settings;
-
-    // Traverse route parts
-    route.slice(0, route.length).forEach(property => {
-      if (
-        !Object.prototype.hasOwnProperty.call(obj, property) ||
-        (obj.hasOwnProperty(property) && typeof obj[property] !== 'object') ||
-        obj[property] === null
-      ) {
-        obj[property] = {}; // create nested object if missing or not an object
-      }
-      obj = obj[property];
-    });
-
-    // Merge payload into nested object
-    let data = {};
-    try {
-      data = JSON.parse(payload);
-    } catch(error) {
-    }
-
-    if (data && typeof data === 'object' && !Array.isArray(data)) {
-      // Merge data properties
-      for (const [key, value] of Object.entries(data)) {
-        obj[key] = value;
-      }
-    } else {
-      // payload isn't an object, replace the nested object
-      obj = payload;
-    }
-
-    try {
-      await $.db_device.updateRemoteSettings(JSON.stringify(settings), device.id);
-    } catch (err) {
-      console.error("Failed to update local settings:", err);
-    }
-  },
-
   checkFota : async (release = "dev")=>{
 
     const models = await $.db_model.getAll();
@@ -484,25 +246,310 @@ var self = module.exports = {
     }
   },
 
-  handleFotaSuccess : async (deviceId) => {
-    let object = {
-      "nAttempts" : 0,
-      "fUpdate" : 0,
+}
+
+async function parseLwm2mMessage(client, project_name, device, topic, payload, action){
+
+  console.log("parse lwm2m message: ",topic);
+
+  try{
+    payload = JSON.parse(payload);
+  }catch(error){}
+
+  let word = getFirstWord(topic);
+  topic = getWordAfterSlash(topic);
+
+  switch (word) {
+    case "status":
+      if (payload === "online" || payload === "offline") {
+        $.db_device.update(device.id, "status", payload);
+        $.db_device.addLog(device.id,"status",payload);
+      }
+      return;
+      break;
+    case "sensor":
+      updateSensor(device,topic,payload)
+  }
+
+  if(_project[project_name]){
+    _project[project_name]?.module?.parseMessage(client,project_name,device,`${word}/${topic}`,payload,action,()=>{});
+  }
+}
+
+async function parseMqttMessage(client, project_name, device, topic, payload, retain){
+
+  if(topic.endsWith("/get"))
+    return;
+
+  //console.log("[MQTT] parse topic: ",topic);
+
+  try{
+    payload = JSON.parse(payload);
+  }catch(error){}
+
+  let word = getFirstWord(topic);
+  topic = getWordAfterSlash(topic);
+
+  switch (word) {
+    case "status":
+      if (payload === "online" || payload === "offline") {
+        $.db_device.update(device.id, "status", payload);
+        $.db_device.addLog(device.id,"status",payload);
+      }
+      return;
+      break;
+    case "model":
+      let res = await $.db_model.getByName(payload);
+      let model_id = res?.id;
+      if (model_id != null && device?.tech != payload) {
+        $.db_device.update(device.id, "model_id", model_id);
+        $.db_device.addLog(device.id,"model_id",model_id);
+      }
+      return;
+      break;
+    case "tech":
+      if (device?.tech && payload != device.tech) {
+        $.db_device.update(device.id, "tech", payload);
+        $.db_device.addLog(device.id,"tech",payload);
+      }
+      return;
+      break;
+    case "version":
+      if (device?.version && payload != device.version) {
+        $.db_device.addLog(device.id,"version",payload);
+        $.db_device.update(device.id, "version", payload);
+        handleFotaSuccess(device.id);
+      }
+      return;
+      break;
+    case "app_version":
+      if (device?.app_version && payload != device.app_version) {
+        $.db_device.addLog(device.id,"app_version",payload);
+        $.db_device.update(device.id, "app_version", payload);
+        handleFotaSuccess(device.id);
+      }
+      return;
+      break;
+    case "fw":
+      if(topic === "fota/update/status"){
+        handleFotaError(device.id, payload);
+      }else{
+        if(typeof payload === 'object' && payload !== null){
+          try{
+            let rows = await $.db_data.updateJson("fw",device.id,payload);
+            rows = await $.db_data.addJsonLog("logs_fw",device.id,payload);
+          }catch(error){
+            console.error(error)
+          } 
+        }else if(typeof payload !== 'object' && payload !== null){
+          // change it to topic.startsWith("fw")
+          try{
+            const column = getWordAfterLastSlash(topic);
+            let rows = await $.db_data.update("fw",device.id,column,payload);
+            rows = await $.db_data.addLog("logs_fw",device.id,column,payload);
+          }catch(error){
+            console.error(error)
+          } 
+        }
+      }
+      return;
+      break;
+    case "settings":
+      if(topic.endsWith("/set")){
+        updateLocalSettings(device,topic,payload);
+      }else{
+        updateRemoteSettings(device,topic,payload);
+      }
+      return;
+      break;
+    case "sensor":
+      let ref = getFirstWord(topic)
+      updateSensor(device,ref,payload)
+      return;
+      break;
+    // Optional: default case if needed
+    default:
+      // handle other topics or do nothing
+      break;
+  }
+
+  if(_project[project_name]){
+    _project[project_name]?.module?.parseMessage(client,project_name,device,`${word}/${topic}`,payload,retain,()=>{});
+  }
+}
+
+async function updateLocalSettings(device, topic, payload){
+
+  $.db_device.addLog(device.id,"local_settings",JSON.stringify(payload));
+
+  let route = topic.split("/");
+
+  if (route == null || route.length == 0) {
+    console.warn("updateLocalSettings: topic invalid:", topic);
+    return;
+  }
+
+  // Retrieve existing settings
+  let settings = await $.db_device.getLocalSettings(device.id);
+
+  if (!settings || typeof settings !== 'object') {
+    settings = {};
+  }
+
+  let obj = settings;
+
+  // Traverse route parts to reach the target object
+  route.slice(0, -1).forEach(property => {
+    if (
+      !Object.prototype.hasOwnProperty.call(obj, property) ||
+      (obj.hasOwnProperty(property) && typeof obj[property] !== 'object') ||
+      obj[property] === null
+    ) {
+      obj[property] = {}; // Create nested object if missing or not an object
     }
-    await $.db_fota.update(deviceId,object);
-    object = {
-      success : 1,
+    obj = obj[property];
+  });
+
+  // Check if the data is a plain object, then merge
+  if (typeof payload === 'object' && !Array.isArray(payload)) {
+    Object.assign(obj, payload);
+  } else {
+    console.warn("Payload is not a valid object:", payload);
+    return;
+  }
+
+  try {
+    // Update the settings in the database
+    await $.db_device.updateLocalSettings(JSON.stringify(settings), device.id);
+  } catch (err) {
+    console.error("Failed to update local settings:", err);
+  }
+}
+
+async function updateRemoteSettings(device,topic,payload){
+
+  $.db_device.addLog(device.id,"remote_settings",JSON.stringify(payload));
+
+  let route = topic.split("/");
+
+  if(route == null){
+    console.warn("updateRemoteSettings: topic invalid:",topic);
+    return;
+  }
+  // Parse existing settings
+  let settings = await $.db_device.getRemoteSettings(device.id);
+
+  if (!settings || typeof settings !== 'object') {
+    settings = {};
+  }
+
+  let obj = settings;
+
+  // Traverse route parts
+  route.slice(0, route.length).forEach(property => {
+    if (
+      !Object.prototype.hasOwnProperty.call(obj, property) ||
+      (obj.hasOwnProperty(property) && typeof obj[property] !== 'object') ||
+      obj[property] === null
+    ) {
+      obj[property] = {}; // create nested object if missing or not an object
     }
-    await $.db_fota.updateLog(deviceId,object);
-  },
+    obj = obj[property];
+  });
 
-  handleFotaError : async (deviceId, error) => {
-    let object = {
-      error : error,
+  if (typeof payload === 'object' && !Array.isArray(payload)) {
+    // Merge payload properties
+    for (const [key, value] of Object.entries(payload)) {
+      obj[key] = value;
     }
-    await $.db_fota.updateLog(deviceId,object);
+  } else {
+    // payload isn't an object, replace the nested object
+    obj = payload;
+  }
+
+  try {
+    await $.db_device.updateRemoteSettings(JSON.stringify(settings), device.id);
+  } catch (err) {
+    console.error("Failed to update local settings:", err);
   }
 }
 
+async function updateSensor(device,ref,payload){
+  let res = await $.db_device.getSensorByRef(device.id,ref)
+  if(res == null)
+    res = await $.db_model.getSensorByRef(device.model_id,ref)
+  if(res == null)
+    return;
+
+  object = payload;
+  value = null;
+  error = null;
+  timestamp = null;
+
+  if (typeof object === 'object' && object !== null) {
+    value = object?.value || object?.v;
+    error = object?.error || object?.e;
+    timestamp = object?.timestamp || objects?.ts;
+  }else{
+    value = payload;
+  }
+
+  if(value || error)
+    object = null;
+
+  const data = {
+    object,
+    value,
+    error,
+    timestamp
+  }
+
+  $.db_sensor.insert(logs_table,device.id,res.id,data);
+  return;
+}
 
+function handleFotaSuccess (deviceId){
+  let object = {
+    "nAttempts" : 0,
+    "fUpdate" : 0,
+  }
+  $.db_fota.update(deviceId,object);
+  object = {
+    success : 1,
+  }
+  $.db_fota.updateLog(deviceId,object);
+}
 
+function handleFotaError (deviceId, error){
+  let object = {
+    error : error,
+  }
+  $.db_fota.updateLog(deviceId,object);
+}
+
+function getFirstWord(str){
+  const slashIndex = str.indexOf('/');
+  if (slashIndex === -1) {
+    // No slash found, return the original string
+    return str;
+  }
+  return str.substring(0,slashIndex);
+}
+
+function getWordAfterSlash(str){
+  const slashIndex = str.indexOf('/');
+  if (slashIndex === -1) {
+    // No slash found, return empty string
+    return "";
+  }
+  return str.substring(slashIndex + 1);
+}
+
+function getWordAfterLastSlash(str){
+  const lastSlashIndex = str.lastIndexOf('/');
+  if (lastSlashIndex === -1) {
+    // No slash found, return the original string
+    return str;
+  }
+  return str.substring(lastSlashIndex + 1);
+}

package/src/kafka/consumer.js

@@ -1,4 +1,5 @@
 const { Kafka, logLevel } = require('kafkajs');
+const fs = require('fs');
 
 var kafka = null;
 var consumer = null;
@@ -11,23 +12,43 @@ var self = module.exports = {
     return new Promise(async (resolve, reject) => {
       try {
         if (!config.kafka.enabled) {
-          console.log("Kafka consumer is disabled");
+          console.log("[Kafka] consumer is disabled");
           return resolve();
         }
 
-        console.log("Initializing Kafka consumer...");
+        console.log("[Kafka] Initializing consumer...");
+
+        // Generate a random 6-digit number
+        const randomNumber = Math.floor(100000 + Math.random() * 900000);
 
         // Build Kafka configuration
         const kafkaConfig = {
-          clientId: config.kafka.clientId,
+          clientId: `${config.kafka.clientId}-${randomNumber}`,
           brokers: config.kafka.brokers,
           connectionTimeout: config.kafka.connectionTimeout,
           requestTimeout: config.kafka.requestTimeout,
           logLevel: logLevel.INFO
         };
 
+        if(config.kafka.ssl.enabled === true){
+          console.log("[Kafka] Try to establish SSL connection..");
+          if(config.kafka.ssl.rejectUnauthorized){
+            // check certificate. Most secure
+            kafkaConfig.ssl = {
+              ca: [fs.readFileSync(config.kafka.ssl.file, 'utf8')],
+              rejectUnauthorized: config.kafka.ssl.rejectUnauthorized
+            }
+           }else{
+            // trust certificate. Prone to MITM attacks
+            kafkaConfig.ssl = {
+              rejectUnauthorized: config.kafka.ssl.rejectUnauthorized
+            }
+          }
+        }
+
         // Add SASL authentication if credentials provided
         if (config.kafka.sasl.username && config.kafka.sasl.password) {
+          console.log("[Kafka] SASL: username and password are defined");
           kafkaConfig.sasl = {
             mechanism: config.kafka.sasl.mechanism,
             username: config.kafka.sasl.username,
@@ -35,11 +56,6 @@ var self = module.exports = {
           };
         }
 
-        // Add SSL configuration if enabled
-        if (config.kafka.ssl) {
-          kafkaConfig.ssl = true;
-        }
-
         kafka = new Kafka(kafkaConfig);
 
         // Create consumer with shared subscription support
@@ -62,7 +78,7 @@ var self = module.exports = {
         const topics = projects.map(project => project.toString());
         
         if (topics.length === 0) {
-          console.log("No Kafka topics to subscribe to");
+          console.log("[Kafka] No topics to subscribe to");
           return resolve();
         }
 
@@ -71,11 +87,11 @@ var self = module.exports = {
           fromBeginning: false 
         });
 
-        console.log(`Kafka consumer subscribed to topics: ${topics.join(', ')}`);
+        console.log(`[Kafka] consumer subscribed to topics: ${topics.join(', ')}`);
 
         return resolve();
       } catch (error) {
-        console.error("Failed to initialize Kafka consumer:", error);
+        console.error("[Kafka] Failed to initialize consumer:", error);
         return reject(error);
       }
     });
@@ -87,11 +103,14 @@ var self = module.exports = {
     }
 
     try {
-      console.log("Starting Kafka consumer...");
+      console.log("[Kafka] Starting consumer...");
       running = true;
 
       await consumer.run({
         eachMessage: async ({ topic, partition, message }) => {
+
+          console.log(`[KAFKA] rx: ${message.key.toString()}`);
+
           try {
             let payload = message.value ? message.value.toString() : '';
             const kafkaTopic = topic.toString();
@@ -113,10 +132,13 @@ var self = module.exports = {
 
             try{
               payload = JSON.parse(payload);
-            }catch(error){}
+            }catch(error){
+              console.log(`[KAFKA] error: ${payload}`);
+              console.log(error)
+            }
 
             // Call the device parser with formatted topic
-            await $.device.parseMessage(payload.client.id, formattedTopic, payload.mqtt.payload, payload.mqtt.retain);
+            await $.device.parseMessage(payload.client.id, formattedTopic, JSON.stringify(payload.mqtt.payload), payload.mqtt.retain);
 
           } catch (error) {
             console.error('[Kafka] Error processing message:', {
@@ -129,9 +151,9 @@ var self = module.exports = {
         },
       });
 
-      console.log("Kafka consumer started successfully");
+      console.log("[Kafka] consumer started successfully");
     } catch (error) {
-      console.error("Failed to start Kafka consumer:", error);
+      console.error("[KAFKA] Failed to start consumer:", error);
       running = false;
       throw error;
     }
@@ -143,13 +165,13 @@ var self = module.exports = {
     }
 
     try {
-      console.log("Stopping Kafka consumer...");
+      console.log("[KAFKA] Stopping consumer...");
       running = false;
       await consumer.stop();
       await consumer.disconnect();
-      console.log("Kafka consumer stopped successfully");
+      console.log("[KAFKA] consumer stopped successfully");
     } catch (error) {
-      console.error("Error stopping Kafka consumer:", error);
+      console.error("[KAFKA] Error stopping consumer:", error);
     }
   },
 

package/src/unitTest/kafka-integration.test.js

@@ -16,7 +16,6 @@ async function testConfigurationValues() {
   console.log('\n=== Testing Configuration Values ===');
   
   // Test default values
-  console.log('Default MQTT parseMessages:', $.config.mqtt.parseMessages);
   console.log('Default Kafka enabled:', $.config.kafka.enabled);
   console.log('Default Kafka brokers:', $.config.kafka.brokers);
   console.log('Default Kafka groupId:', $.config.kafka.groupId);
@@ -36,7 +35,6 @@ async function testConfigurationValues() {
   const newConfig = require(__dirname + path + '/config');
   
   console.log('\nWith environment variables:');
-  console.log('MQTT parseMessages:', newConfig.mqtt.parseMessages);
   console.log('Kafka enabled:', newConfig.kafka.enabled);
   console.log('Kafka brokers:', newConfig.kafka.brokers);
   console.log('Kafka groupId:', newConfig.kafka.groupId);
@@ -78,19 +76,19 @@ async function testMQTTMessageParsing() {
   
   // Test the logic that would be used in index.js
   const testConfigs = [
-    { mqtt: { parseMessages: true } },
-    { mqtt: { parseMessages: false } }
+    { kafka: { enabled: true } },
+    { kafka: { enabled: false } }
   ];
   
   for (const config of testConfigs) {
-    console.log(`\nTesting with parseMessages: ${config.mqtt.parseMessages}`);
+    console.log(`\nTesting with kafka parseMessages: ${config.kafka.enabled}`);
     
     // Simulate the MQTT message handler logic
     const mockTopic = "test/uid:123/status";
     const mockPayload = "online";
     const mockPacket = { retain: false };
     
-    if (config.mqtt.parseMessages) {
+    if (!config.kafka.enabled) {
       console.log('Would call device.parseMessage');
       await $.device.parseMessage(null, mockTopic, mockPayload, mockPacket.retain);
     } else {

package/src/unitTest/kafka.test.js

@@ -47,7 +47,6 @@ async function testKafkaConsumerInit() {
 async function testConfiguration() {
   console.log('\n=== Testing Configuration ===');
   
-  console.log('MQTT parseMessages:', $.config.mqtt.parseMessages);
   console.log('Kafka enabled:', $.config.kafka.enabled);
   console.log('Kafka brokers:', $.config.kafka.brokers);
   console.log('Kafka groupId:', $.config.kafka.groupId);