mqtt-devices-parser 1.0.23 → 1.0.25

package/Changelog.md

@@ -1,5 +1,60 @@
 # Changelog
 
+## 1.0.25
+
+	Sensors template (#9)
+
+		* new table sensorsTemplate
+
+		* store topics which match a sensor
+		for each mqtt msg received, check if there is a sensor associated to that topic.
+		If there is, update current value and insert new log
+
+		* db: models/sensors
+		changes localUnixTs field to remoteUnixTs
+		adds value, error
+		changes property: allow null
+		adds foreign keys to model_id and device_id
+
+		* handle change db column localUnixTs to remoteUnixTs
+
+		* fix getSensorsByRef
+		ref is not unique so it can have more than 1 row
+
+		* src/db/data: Implements missing function pathIntoObject
+
+		* src/aux/parser: adds functions to handle mqtt topics parsing
+		index: uses global $.parser as a pointer to class src/aux/parser
+		src/device/device: fixes null iterations, sets updateSensor and handleMqttTopic as public functions. Removes local mqtt parser functions. Adapts calls to those functions
+
+	mqtt: always check if topic is registered
+		store values in db
+		
+## 1.0.24
+	src/device/device: fix log
+	src/kafka/consumer: increase initialRetryTime from 100 to 300ms
+	src/device/device: parseMqttMessage: get remote configs if not known
+	synch configs if they mismatch
+	models/devices.models: new column synch
+	synchs mqtt topics
+		Explanation
+		**
+		When a message is received, it checks if mqtt topic is associated to the device without "set" if present.
+		If topic exists and ends with set, updates localData column. If topic exists and doesn't end with set, updates remoteData column.
+		If localData was updated sets topic as unsynched
+		If remoteData was received and differs from localData and synch is enabled, tries to synch topic
+		**
+		adds synch and synched columns to mqtt table
+		Adds methods getMqttTopic, updateRemoteTopic, setSynchedTopic
+	kafka: adds random number to kafka when program is in dev mode.
+		Avoids to read all queue messages since last session
+	mqtt templates: preparing mqtt topics synch
+		new column synch and localData for mqttTemplates table
+		new column mqttTemplate_id for mqtt table
+	add property "property" to table sensors
+	src/device/device: decode json payload and get property associated
+	models/devices: add synched column (for future implementations)
+
 ## 1.0.23
 	src/device/device: fix tech, version and app_version db update
 	npm warn audit Updating websocket-stream to 5.3.0, which is a SemVer major change.

package/index.js

@@ -17,6 +17,7 @@ $.db_firmware = require('./src/db/firmware');
 $.db_sensor = require('./src/db/sensor');
 $.db_fota = require('./src/db/fota');
 $.mqtt_client = null;
+$.parser = require('./src/aux/parser')
 
 const packageJson = require(__dirname+'/package.json');
 const packageVersion = packageJson.version;
@@ -174,7 +175,7 @@ function mqtt_connect(){
         $.mqtt_client.subscribe(project+"/#", (err) => {
           if(err){
             console.log("[MQTT] error");
-            console.err(err);
+            console.error(err);
           }
           else
             console.log("[MQTT] subscribed to project:",project);
@@ -230,6 +231,6 @@ function mqtt_connect(){
 
   $.mqtt_client.on("error",(error)=>{
     console.log("[MQTT] error");
-    console.err(error)
+    console.error(error)
   })
 }

package/models/devices.models.js

@@ -71,6 +71,16 @@ module.exports = (sequelize,DataTypes)=>{
 			type: DataTypes.STRING,
 			allowNull: true
 		},
+		synch: { // set to 1 to enable auto synch
+			type: DataTypes.INTEGER,
+			default: 0,
+			allowNull: true
+		},
+		synched: { // handle internally
+			type: DataTypes.INTEGER,
+			default: 0,
+			allowNull: true
+		},
 	},
 	{
 		tableName: 'devices',

package/models/logs_sensor.models.js

@@ -30,7 +30,7 @@ module.exports = (sequelize,DataTypes)=>{
 			type: DataTypes.STRING,
 			allowNull: true,
 		},
-		localUnixTs: {
+		remoteUnixTs: {
 			type: DataTypes.BIGINT,
 			allowNull: true,
 		}

package/models/mqtt.models.js

@@ -32,6 +32,20 @@ module.exports = (sequelize,DataTypes)=>{
 		template_id: {
 			type: DataTypes.INTEGER,
 			allowNull: true,
+		},
+		mqttTemplate_id: {
+			type: DataTypes.INTEGER,
+			allowNull: true,
+		},
+		synch: {
+			type: DataTypes.INTEGER,
+			default: 0,
+			allowNull: true,
+		},
+		synched: {
+			type: DataTypes.INTEGER,
+			default: 0,
+			allowNull: true,
 		}
 	},
 	{

package/models/mqttTemplate.models.js

@@ -13,10 +13,18 @@ module.exports = (sequelize,DataTypes)=>{
 			type: DataTypes.JSON,
 			allowNull: true	
 		},
+		localData: {
+			type: DataTypes.JSON,
+			allowNull: true	
+		},
 		readInterval: {
 			type: DataTypes.INTEGER,
 			allowNull: true
 		},
+		synch: {
+			type: DataTypes.INTEGER,
+			allowNull: true
+		},
 		template_id: {
 			type: DataTypes.INTEGER,
 			allowNull: false

package/models/sensors.models.js

@@ -3,11 +3,19 @@ module.exports = (sequelize,DataTypes)=>{
 	return sequelize.define("sensors", {
 		model_id: { // reference a model
 			type: DataTypes.INTEGER,
-			allowNull: true,
+			allowNull: false,
+			references: {
+				model: 'models',
+				key: 'id'
+			}
 		},
 		device_id: { // reference a device id
 			type: DataTypes.INTEGER,
-			allowNull: true,
+			allowNull: false,
+			references: {
+				model: 'devices',
+				key: 'id'
+			}
 		},
 		active: {
 			type: DataTypes.BOOLEAN,
@@ -26,6 +34,22 @@ module.exports = (sequelize,DataTypes)=>{
 			type: DataTypes.STRING,
 			allowNull: false,
 		},
+		property: {
+			type: DataTypes.STRING,
+			allowNull: true
+		},
+		value: {
+			type: DataTypes.STRING,
+			allowNull: true
+		},
+		error: {
+			type: DataTypes.STRING,
+			allowNull: true,
+		},
+		remoteUnixTs: {
+			type: DataTypes.BIGINT,
+			allowNull: true,
+		},
 		graph: {
 			type: DataTypes.JSON,
 			allowNull: true,

package/models/sensorsTemplate.models.js

@@ -0,0 +1,43 @@
+
+module.exports = (sequelize,DataTypes)=>{
+	return sequelize.define("sensorsTemplate", {
+		model_id: { // reference a model
+			type: DataTypes.INTEGER,
+			allowNull: false,
+			references: {
+				model: 'models',
+				key: 'id'
+			}
+		},
+		active: {
+			type: DataTypes.BOOLEAN,
+			allowNull: false,
+			defaultValue: true
+		},
+		ref: {
+			type: DataTypes.STRING,
+			allowNull: false,
+		},
+		name: {
+			type: DataTypes.STRING,
+			allowNull: false,
+		},
+		type: {
+			type: DataTypes.STRING,
+			allowNull: false,
+		},
+		property: {
+			type: DataTypes.STRING,
+			allowNull: true
+		},
+		graph: {
+			type: DataTypes.JSON,
+			allowNull: true,
+		}
+	},
+	{
+		tableName: 'sensorsTemplate',
+		freezeTableName: true
+	})
+}
+

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mqtt-devices-parser",
-  "version": "1.0.23",
+  "version": "1.0.25",
   "description": "",
   "main": "index.js",
   "scripts": {

package/src/aux/parser.js

@@ -16,5 +16,41 @@ module.exports = {
 			}
 		});
 		return obj;
+	},
+
+	getFirstWord : (str)=>{
+		const slashIndex = str.indexOf('/');
+		if (slashIndex === -1) {
+			// No slash found, return the original string
+			return str;
+		}
+		return str.substring(0,slashIndex);
+	},
+
+	getTopicAfterSlash : (str)=>{
+		const slashIndex = str.indexOf('/');
+		if (slashIndex === -1) {
+		// No slash found, return empty string
+		return "";
+		}
+		return str.substring(slashIndex + 1);
+	},
+
+	getWordAfterLastSlash : (str)=>{
+		const lastSlashIndex = str.lastIndexOf('/');
+		if (lastSlashIndex === -1) {
+			// No slash found, return the original string
+			return str;
+		}
+		return str.substring(lastSlashIndex + 1);
+	},
+
+	getWordBeforeLastSlash : (str)=>{
+		const lastSlashIndex = str.lastIndexOf('/');
+		if (lastSlashIndex === -1) {
+			// No slash found, return the original string
+			return str;
+		}
+		return str.substring(0,lastSlashIndex);
 	}
 }

package/src/db/data.js

@@ -372,3 +372,59 @@ var self = module.exports = {
 	  });
 	}
 };
+
+/**
+ * Build a nested object from a path string and a leaf value.
+ *
+ * Example:
+ *   pathIntoObject('foo/bar/baz', 123) -> { foo: { bar: { baz: 123 } } }
+ *
+ * Notes:
+ * - By default, splits on '/'.
+ * - Leading/trailing/duplicate delimiters are ignored (e.g., '/a//b/' -> ['a','b']).
+ * - Value is used as-is. If you pass JSON.stringify(data), the leaf will be that string.
+ *
+ * @param {string} path - The path to turn into nested object keys (e.g., 'a/b/c').
+ * @param {*} value - The value to place at the final key.
+ * @param {Object} [options]
+ * @param {string} [options.delimiter='/'] - Path delimiter.
+ * @returns {*}
+ */
+const parser = {
+  pathIntoObject(path, value, options = {}) {
+    const delimiter = options.delimiter || '/';
+
+    if (typeof path !== 'string') {
+      throw new TypeError('path must be a string');
+    }
+
+    // Normalize path: remove empty segments caused by leading/trailing or duplicate delimiters
+    const segments = path
+      .split(delimiter)
+      .map(s => s.trim())
+      .filter(Boolean);
+
+    // If no segments, just return the value directly
+    if (segments.length === 0) return value;
+
+    // Build nested object
+    let root = {};
+    let cursor = root;
+
+    for (let i = 0; i < segments.length; i++) {
+      const key = segments[i];
+      const isLast = i === segments.length - 1;
+
+      if (isLast) {
+        cursor[key] = value;
+      } else {
+        // Create the next level if not present
+        cursor[key] = {};
+        cursor = cursor[key];
+      }
+    }
+
+    return root;
+  },
+};
+

package/src/db/device.js

@@ -194,7 +194,6 @@ var self = module.exports = {
 		});
 	},
 
-
 	addLog : async(id,column,value)=>{
 		return new Promise((resolve,reject) => {
 
@@ -255,26 +254,26 @@ var self = module.exports = {
 
 	getLocalSettings : async(deviceId)=>{
 
-  		return new Promise((resolve,reject) => {
+		return new Promise((resolve,reject) => {
 
-		    let query = "SELECT local_settings FROM ?? where id = ?";
-		    let table = ["devices", deviceId];
-		    query = mysql.format(query,table);
+	    let query = "SELECT local_settings FROM ?? where id = ?";
+	    let table = ["devices", deviceId];
+	    query = mysql.format(query,table);
 
-		    $.db.queryRow(query)
-		    .then( rows => {
-		    	if(rows?.length > 0)
-			  		return resolve(rows[0]?.local_settings);
-			  	else
-			  		return resolve(null);
-		    })
-		    .catch( err => {
-		      return reject(err);
-		    });
+	    $.db.queryRow(query)
+	    .then( rows => {
+	    	if(rows?.length > 0)
+		  		return resolve(rows[0]?.local_settings);
+		  	else
+		  		return resolve(null);
+	    })
+	    .catch( err => {
+	      return reject(err);
+	    });
 		});
-  	},
+	},
 
-  updateLocalSettings : async(settings, deviceId)=>{
+	updateLocalSettings : async(settings, deviceId)=>{
 
 		return new Promise((resolve,reject) => {
 			let obj = {
@@ -299,7 +298,7 @@ var self = module.exports = {
 		});
 	},
 
-  getRemoteSettings : async(deviceId)=>{
+	getRemoteSettings : async(deviceId)=>{
 
 		return new Promise((resolve,reject) => {
 
@@ -389,13 +388,34 @@ var self = module.exports = {
 		});
 	},
 
-	getSensorByRef : async(deviceId, ref)=>{
+	getSensorsByRef : async(deviceId, ref)=>{
 		return new Promise((resolve,reject) => {
 
 	    let query = "SELECT * FROM ?? where device_id = ? and ref = ?";
 	    let args = ["sensors",deviceId,ref];
 	    query = mysql.format(query,args);
 
+	    $.db.queryRow(query)
+	    .then( rows => {
+	      if(rows.length > 0)
+	        return resolve(rows);
+	      else
+	        return resolve(null);
+	    })
+	    .catch( err => {
+	      console.log(err);
+	      return resolve(null);
+	    });
+	  });
+	},
+
+	getMqttTopic : async(deviceId, topic)=>{
+		return new Promise((resolve,reject) => {
+
+	    let query = "SELECT * FROM ?? where device_id = ? and topic = ?";
+	    let args = ["mqtt",deviceId,topic];
+	    query = mysql.format(query,args);
+
 	    $.db.queryRow(query)
 	    .then( rows => {
 	      if(rows.length > 0)
@@ -408,6 +428,82 @@ var self = module.exports = {
 	      return resolve(null);
 	    });
 	  });
-	}
+	},
+
+	updateLocalTopic : async(id, data)=>{
+
+		let jsonData = {
+			value : data
+		};
+		
+		return new Promise((resolve,reject) => {
+			let obj = {
+				localData : JSON.stringify(jsonData),
+				updatedAt : moment().utc().format('YYYY-MM-DD HH:mm:ss')
+			}
+
+			let filter = {
+				id
+			};
+
+		    $.db.update("mqtt",obj,filter)
+		    .then (rows => {
+		      return resolve(rows[0]);
+		    })
+		    .catch(error => {
+		      return reject(error);
+		    });
+		    
+		});
+	},
+
+	updateRemoteTopic : async(id, data)=>{
 
+		let jsonData = {
+			value : data
+		};
+
+		return new Promise((resolve,reject) => {
+			let obj = {
+				remoteData : JSON.stringify(jsonData),
+				updatedAt : moment().utc().format('YYYY-MM-DD HH:mm:ss')
+			}
+
+			let filter = {
+				id
+			};
+			
+		    $.db.update("mqtt",obj,filter)
+		    .then (rows => {
+		      return resolve(rows[0]);
+		    })
+		    .catch(error => {
+		      return reject(error);
+		    });
+		    
+		});
+	},
+
+	setSynchedTopic : async(id, data)=>{
+		
+		return new Promise((resolve,reject) => {
+			let obj = {
+				synched : data ? 1 : 0,
+				updatedAt : moment().utc().format('YYYY-MM-DD HH:mm:ss')
+			}
+
+			let filter = {
+				id,
+			};
+
+		    $.db.update("mqtt",obj,filter)
+		    .then (rows => {
+		      return resolve(rows[0]);
+		    })
+		    .catch(error => {
+		      return reject(error);
+		    });
+		    
+		});
+	},
 }

package/src/db/model.js

@@ -112,7 +112,7 @@ var self = module.exports = {
 		});
   	},
 
-  	getSensorByRef : async(modelId, ref)=>{
+  	getSensorsByRef : async(modelId, ref)=>{
 		return new Promise((resolve,reject) => {
 
 	    let query = "SELECT * FROM ?? where model_id = ? and ref = ?";
@@ -122,7 +122,7 @@ var self = module.exports = {
 	    $.db.queryRow(query)
 	    .then( rows => {
 	      if(rows.length > 0)
-	        return resolve(rows[0]);
+	        return resolve(rows);
 	      else
 	        return resolve(null);
 	    })

package/src/db/sensor.js

@@ -26,6 +26,7 @@ var self = module.exports = {
 	  });
 	},
 
+	// not used !!
 	getByRef : async (ref)=>{
 
 	  return new Promise((resolve,reject) => {
@@ -98,7 +99,7 @@ var self = module.exports = {
 				value : payload?.value,
 				error : payload?.error,
 				obj : payload?.object,
-				localUnixTs : payload?.timestamp, // local timestamp
+				remoteUnixTs : payload?.timestamp, // local timestamp
 				createdAt : moment().utc().format('YYYY-MM-DD HH:mm:ss'),
 				updatedAt : moment().utc().format('YYYY-MM-DD HH:mm:ss')
 			}
@@ -113,4 +114,16 @@ var self = module.exports = {
 		});
   	},
 
+	update : async(table,obj,filter)=>{
+		return new Promise((resolve,reject) => {
+
+			$.db.update(table,obj,filter)
+			.then (rows => {
+				return resolve(rows);
+			})
+			.catch(error => {
+				return reject(error);
+			});
+		});
+	},
 }

package/src/device/device.js

@@ -1,6 +1,8 @@
 const moment = require('moment');
+const { isDeepStrictEqual } = require('node:util');
 
 var _project = [];
+const table = "sensors"
 const logs_table = "logs_sensor";
 const semver = require('semver');
 
@@ -29,8 +31,6 @@ var self = module.exports = {
         })
       })
     });
-
-
   },
 
   parseMessage : async (client,topic,payload,retain)=>{
@@ -48,22 +48,22 @@ var self = module.exports = {
     }catch(err){};
     
     // --- project ---
-    project_name = getFirstWord(topic);
+    project_name = $.parser.getFirstWord(topic);
     project = await $.db_project.getByName(project_name);
     if(project == null)
       return;
 
     project_id = project?.id;
-    topic = getWordAfterSlash(topic)
+    topic = $.parser.getTopicAfterSlash(topic)
 
     // check if is from lwm2m gw
     if(topic.startsWith("responses") || topic.startsWith("requests")){
-      action = getFirstWord(topic);
-      topic = getWordAfterSlash(topic);
+      action = $.parser.getFirstWord(topic);
+      topic = $.parser.getTopicAfterSlash(topic);
     }
 
     // --- uid ---
-    uid = getFirstWord(topic);
+    uid = $.parser.getFirstWord(topic);
     // check if topic corresponds to a device
     if(!uid.startsWith(project.uidPrefix) || uid.length > project?.uidLength)
       return;
@@ -72,14 +72,13 @@ var self = module.exports = {
     if(!device)
       return;
 
-    topic = getWordAfterSlash(topic);
+    topic = $.parser.getTopicAfterSlash(topic);
 
     if(device.protocol.toLowerCase() === "lwm2m"){
       parseLwm2mMessage(client, project_name, device, topic, payload, action);
     }else if(device.protocol.toLowerCase() === "mqtt"){
       parseMqttMessage(client, project_name, device, topic, payload, retain);
     }
-
   },
 
   deleteLogs : async()=>{
@@ -104,6 +103,9 @@ var self = module.exports = {
 
     const models = await $.db_model.getAll();
 
+    if(!models || !models?.length)
+        return;
+
     for (const model of models) {
       
       //console.log("model:",model?.name);
@@ -180,6 +182,9 @@ var self = module.exports = {
 
     const devices = await $.db_fota.getUpdatable(release);
 
+    if(!devices || !!devices?.length)
+      return;
+
     const batchSize = 10;
     for (let i = 0; i < devices.length; i += batchSize) {
       const batch = devices.slice(i, i + batchSize);
@@ -246,6 +251,91 @@ var self = module.exports = {
     }
   },
 
+
+  updateSensor : async (device,ref,payload)=>{
+
+    let sensors = await $.db_device.getSensorsByRef(device.id,ref)
+
+    if(!sensors?.length)
+      return;
+
+    object = payload;
+    value = null;
+    error = null;
+    timestamp = null;
+    
+    sensors.map( (sensor,index) =>{
+      let value = null;
+      let error = null;
+      let remoteUnixTs = null;
+      if(sensor?.type === "json" && typeof object === 'object'){
+        if(object.hasOwnProperty(sensor?.property)){
+          value = object[sensor.property];
+        }
+      }else{
+        if (typeof object === 'object') {
+          value = object?.value || object?.v;
+          error = object?.error || object?.e;
+          remoteUnixTs = object?.timestamp || objects?.ts;
+        }else{
+          value = payload;
+        }
+      }
+      if(value || error){
+
+        if(!remoteUnixTs)
+          remoteUnixTs = moment().unix()
+
+        const data = {
+          value,
+          error,
+          remoteUnixTs
+        }
+
+        let filter = {
+          id : sensor.id
+        }
+
+        $.db_sensor.update(table,data,filter);    
+        $.db_sensor.insert(logs_table,device.id,sensor.id,data);    
+      }
+    })
+  },
+
+  handleMqttTopic : async(device, dbTopic, payload, set)=>{
+    if(set){
+      // update local topic
+      $.db_device.updateLocalTopic(dbTopic.id,payload)
+      .then(()=>{
+        // set as not synched
+        $.db_device.setSynchedTopic(dbTopic.id,false);
+      })
+      .catch((error)=>{
+        console.error(error);
+      })
+    }else{
+      // update remote topic
+      $.db_device.updateRemoteTopic(dbTopic.id,payload)
+      .then(async (res)=>{
+        // check if topics mismatch
+        if(isDeepStrictEqual(dbTopic?.localData, dbTopic?.remoteData)){
+          $.db_device.setSynchedTopic(dbTopic.id,true);
+        }else{
+          // set as not synched
+          await $.db_device.setSynchedTopic(dbTopic.id,false);
+          if(dbTopic?.synch){ // check if sync is enabled
+            // synch topic
+            synchMqttTopic(device,dbTopic);
+          }
+        }
+      })
+      .catch((error)=>{
+        console.error(error);
+      })
+
+    }
+  },
+
 }
 
 async function parseLwm2mMessage(client, project_name, device, topic, payload, action){
@@ -256,8 +346,8 @@ async function parseLwm2mMessage(client, project_name, device, topic, payload, a
     payload = JSON.parse(payload);
   }catch(error){}
 
-  let word = getFirstWord(topic);
-  topic = getWordAfterSlash(topic);
+  let word = $.parser.getFirstWord(topic);
+  topic = $.parser.getTopicAfterSlash(topic);
 
   switch (word) {
     case "status":
@@ -268,7 +358,7 @@ async function parseLwm2mMessage(client, project_name, device, topic, payload, a
       return;
       break;
     case "sensor":
-      updateSensor(device,topic,payload)
+      self.updateSensor(device,topic,payload)
   }
 
   if(_project[project_name]){
@@ -281,14 +371,15 @@ async function parseMqttMessage(client, project_name, device, topic, payload, re
   if(topic.endsWith("/get"))
     return;
 
+  const topicBck = topic;
   //console.log("[MQTT] parse topic: ",topic);
 
   try{
     payload = JSON.parse(payload);
   }catch(error){}
 
-  let word = getFirstWord(topic);
-  topic = getWordAfterSlash(topic);
+  let word = $.parser.getFirstWord(topic);
+  topic = $.parser.getTopicAfterSlash(topic);
 
   switch (word) {
     case "status":
@@ -296,7 +387,32 @@ async function parseMqttMessage(client, project_name, device, topic, payload, re
         $.db_device.update(device.id, "status", payload);
         $.db_device.addLog(device.id,"status",payload);
       }
-      return;
+
+      // list mqtt topics and do calls to readable topics
+      if (payload === "online"){
+        let mqtt_prefix = `${project_name}/${device.uid}`;
+        // check if remote settings are known, if not require it
+        if(!device.remote_settings?.log){
+          let topic = `${mqtt_prefix}/settings/log/get`
+          $.mqtt_client.publish(topic,"",{qos:1,retain:false});
+        }
+        if(!device.remote_settings?.keepalive){
+          let topic = `${mqtt_prefix}/settings/keepalive/get`
+          $.mqtt_client.publish(topic,"",{qos:1,retain:false});
+        }
+        if(!device.remote_settings?.modem){
+          let topic = `${mqtt_prefix}/settings/modem/get`
+          $.mqtt_client.publish(topic,"",{qos:1,retain:false});
+        }
+        if(!device.remote_settings?.wifi){
+          let topic = `${mqtt_prefix}/settings/wifi/get`
+          $.mqtt_client.publish(topic,"",{qos:1,retain:false});
+        }
+        if(!device.remote_settings?.mqtt){
+          let topic = `${mqtt_prefix}/settings/mqtt/get`
+          $.mqtt_client.publish(topic,"",{qos:1,retain:false});
+        }
+      }
       break;
     case "model":
       let res = await $.db_model.getByName(payload);
@@ -305,14 +421,12 @@ async function parseMqttMessage(client, project_name, device, topic, payload, re
         $.db_device.update(device.id, "model_id", model_id);
         $.db_device.addLog(device.id,"model_id",model_id);
       }
-      return;
       break;
     case "tech":
       if (payload != null && payload != device?.tech) {
         $.db_device.update(device.id, "tech", payload);
         $.db_device.addLog(device.id,"tech",payload);
       }
-      return;
       break;
     case "version":
       if (payload != null && payload != device?.version) {
@@ -320,7 +434,6 @@ async function parseMqttMessage(client, project_name, device, topic, payload, re
         $.db_device.update(device.id, "version", payload);
         handleFotaSuccess(device.id);
       }
-      return;
       break;
     case "app_version":
       if (payload != null && payload != device?.app_version) {
@@ -328,7 +441,6 @@ async function parseMqttMessage(client, project_name, device, topic, payload, re
         $.db_device.update(device.id, "app_version", payload);
         handleFotaSuccess(device.id);
       }
-      return;
       break;
     case "fw":
       if(topic === "fota/update/status"){
@@ -344,7 +456,7 @@ async function parseMqttMessage(client, project_name, device, topic, payload, re
         }else if(typeof payload !== 'object' && payload !== null){
           // change it to topic.startsWith("fw")
           try{
-            const column = getWordAfterLastSlash(topic);
+            const column = $.parser.getWordAfterLastSlash(topic);
             let rows = await $.db_data.update("fw",device.id,column,payload);
             rows = await $.db_data.addLog("logs_fw",device.id,column,payload);
           }catch(error){
@@ -352,7 +464,6 @@ async function parseMqttMessage(client, project_name, device, topic, payload, re
           } 
         }
       }
-      return;
       break;
     case "settings":
       if(topic.endsWith("/set")){
@@ -360,25 +471,54 @@ async function parseMqttMessage(client, project_name, device, topic, payload, re
       }else{
         updateRemoteSettings(device,topic,payload);
       }
-      return;
-      break;
-    case "sensor":
-      let ref = getFirstWord(topic)
-      updateSensor(device,ref,payload)
-      return;
       break;
     // Optional: default case if needed
     default:
-      // handle other topics or do nothing
       break;
   }
 
+  // check if topic is associated with device
+  let findTopic = topicBck;
+  let set = false;
+  if(topicBck.endsWith("/set")){ // remove set if exists on topic
+    set = true;
+    findTopic = $.parser.getWordBeforeLastSlash(topicBck);
+  }
+
+  const dbTopic = await $.db_device.getMqttTopic(device.id,findTopic)
+  if(dbTopic != null){
+    self.handleMqttTopic(device,dbTopic,payload,set);
+  }
+
+  // check if topic is a sensor
+  if(!set)
+    self.updateSensor(device,topicBck,payload);
+  
   if(_project[project_name]){
     _project[project_name]?.module?.parseMessage(client,project_name,device,`${word}/${topic}`,payload,retain,()=>{});
   }
 }
 
-async function updateLocalSettings(device, topic, payload){
+async function synchMqttTopic(device,dbTopic){
+  
+  const project = await $.db_project.getById(device.project_id);
+  let mqtt_prefix = `${project.name}/${device.uid}`;
+  let topic = `${mqtt_prefix}/${dbTopic.topic}/set`
+  let payload = "";
+  if(dbTopic?.remoteData && typeof dbTopic?.remoteData === 'object'){
+    try{
+      payload = JSON.stringify(dbTopic?.remoteData);
+    }catch(err){
+      console.error(err);
+      return;
+    }
+  }else{
+    payload = dbTopic?.remoteData;
+  }
+  $.mqtt_client.publish(topic,payload,{qos:1,retain:false});
+}
+
+async function updateLocalSettings(device,topic,payload){
 
   $.db_device.addLog(device.id,"local_settings",JSON.stringify(payload));
 
@@ -469,45 +609,42 @@ async function updateRemoteSettings(device,topic,payload){
 
   try {
     await $.db_device.updateRemoteSettings(JSON.stringify(settings), device.id);
+    if(device?.synch)
+      synchSettings(device,route[0]);
   } catch (err) {
     console.error("Failed to update local settings:", err);
   }
 }
 
-async function updateSensor(device,ref,payload){
-  let res = await $.db_device.getSensorByRef(device.id,ref)
-  if(res == null)
-    res = await $.db_model.getSensorByRef(device.model_id,ref)
-  if(res == null)
-    return;
-
-  object = payload;
-  value = null;
-  error = null;
-  timestamp = null;
+async function synchSettings(device,key){
 
-  if (typeof object === 'object' && object !== null) {
-    value = object?.value || object?.v;
-    error = object?.error || object?.e;
-    timestamp = object?.timestamp || objects?.ts;
-  }else{
-    value = payload;
-  }
-
-  if(value || error)
-    object = null;
-
-  const data = {
-    object,
-    value,
-    error,
-    timestamp
+  console.log(`check topic ${key} from ${device.uid}`);
+  // Retrieve existing settings
+  let localSettings = await $.db_device.getLocalSettings(device.id);
+  console.log(localSettings)
+  let remoteSettings = await $.db_device.getRemoteSettings(device.id);
+  console.log(remoteSettings)
+
+  //let keys = await checkSettings(localSettings,remoteSettings);
+  if(localSettings?.hasOwnProperty(key) && remoteSettings?.hasOwnProperty(key)){
+    if(!isDeepStrictEqual(localSettings?.[key], remoteSettings?.[key])){
+      const project = await $.db_project.getById(device.project_id);
+      let mqtt_prefix = `${project.name}/${device.uid}`;
+      let topic = `${mqtt_prefix}/settings/${key}/set`
+      try{
+        const payload = JSON.stringify(localSettings[key]);
+        console.log(`updating key: ${key} with data:`);
+        console.log(payload);
+        $.mqtt_client.publish(topic,payload,{qos:1,retain:false});
+      }catch(err){
+        console.error(localSettings[key])
+        console.error(err);
+      }
+    }
   }
-
-  $.db_sensor.insert(logs_table,device.id,res.id,data);
-  return;
 }
 
+
 function handleFotaSuccess (deviceId){
   let object = {
     "nAttempts" : 0,
@@ -527,29 +664,4 @@ function handleFotaError (deviceId, error){
   $.db_fota.updateLog(deviceId,object);
 }
 
-function getFirstWord(str){
-  const slashIndex = str.indexOf('/');
-  if (slashIndex === -1) {
-    // No slash found, return the original string
-    return str;
-  }
-  return str.substring(0,slashIndex);
-}
-
-function getWordAfterSlash(str){
-  const slashIndex = str.indexOf('/');
-  if (slashIndex === -1) {
-    // No slash found, return empty string
-    return "";
-  }
-  return str.substring(slashIndex + 1);
-}
 
-function getWordAfterLastSlash(str){
-  const lastSlashIndex = str.lastIndexOf('/');
-  if (lastSlashIndex === -1) {
-    // No slash found, return the original string
-    return str;
-  }
-  return str.substring(lastSlashIndex + 1);
-}

package/src/kafka/consumer.js

@@ -58,9 +58,17 @@ var self = module.exports = {
 
         kafka = new Kafka(kafkaConfig);
 
+        let kafkaGroupId = "";
+        if(config.env === 'development'){
+          const n = Math.floor(Math.random() * 1000) + 1; // 1..1000
+          kafkaGroupId = `${config.kafka.groupId}-${String(n)}`;
+        }else{
+          kafkaGroupId = config.kafka.groupId;
+        }
+
         // Create consumer with shared subscription support
         consumer = kafka.consumer({
-          groupId: config.kafka.groupId,
+          groupId: kafkaGroupId,
           sessionTimeout: 30000,
           rebalanceTimeout: 60000,
           heartbeatInterval: 3000,
@@ -69,7 +77,7 @@ var self = module.exports = {
           maxBytes: 10485760,
           maxWaitTimeInMs: 5000,
           retry: {
-            initialRetryTime: 100,
+            initialRetryTime: 300,
             retries: 8
           }
         });