mqtt-devices-parser 1.0.22 → 1.0.24

package/Changelog.md

@@ -1,5 +1,34 @@
 # Changelog
 
+## 1.0.24
+	src/device/device: fix log
+	src/kafka/consumer: increase initialRetryTime from 100 to 300ms
+	src/device/device: parseMqttMessage: get remote configs if not known
+	synch configs if they mismatch
+	models/devices.models: new column synch
+	synchs mqtt topics
+		Explanation
+		**
+		When a message is received, it checks if mqtt topic is associated to the device without "set" if present.
+		If topic exists and ends with set, updates localData column. If topic exists and doesn't end with set, updates remoteData column.
+		If localData was updated sets topic as unsynched
+		If remoteData was received and differs from localData and synch is enabled, tries to synch topic
+		**
+		adds synch and synched columns to mqtt table
+		Adds methods getMqttTopic, updateRemoteTopic, setSynchedTopic
+	kafka: adds random number to kafka when program is in dev mode.
+		Avoids to read all queue messages since last session
+	mqtt templates: preparing mqtt topics synch
+		new column synch and localData for mqttTemplates table
+		new column mqttTemplate_id for mqtt table
+	add property "property" to table sensors
+	src/device/device: decode json payload and get property associated
+	models/devices: add synched column (for future implementations)
+
+## 1.0.23
+	src/device/device: fix tech, version and app_version db update
+	npm warn audit Updating websocket-stream to 5.3.0, which is a SemVer major change.
+
 ## 1.0.22
 	Adds SSL and SASL support for kafka
 		mqtt.parseMessages removed

package/index.js

@@ -174,7 +174,7 @@ function mqtt_connect(){
         $.mqtt_client.subscribe(project+"/#", (err) => {
           if(err){
             console.log("[MQTT] error");
-            console.err(err);
+            console.error(err);
           }
           else
             console.log("[MQTT] subscribed to project:",project);
@@ -230,6 +230,6 @@ function mqtt_connect(){
 
   $.mqtt_client.on("error",(error)=>{
     console.log("[MQTT] error");
-    console.err(error)
+    console.error(error)
   })
 }

package/models/devices.models.js

@@ -71,6 +71,16 @@ module.exports = (sequelize,DataTypes)=>{
 			type: DataTypes.STRING,
 			allowNull: true
 		},
+		synch: { // set to 1 to enable auto synch
+			type: DataTypes.INTEGER,
+			default: 0,
+			allowNull: true
+		},
+		synched: { // handle internally
+			type: DataTypes.INTEGER,
+			default: 0,
+			allowNull: true
+		},
 	},
 	{
 		tableName: 'devices',

package/models/mqtt.models.js

@@ -32,6 +32,20 @@ module.exports = (sequelize,DataTypes)=>{
 		template_id: {
 			type: DataTypes.INTEGER,
 			allowNull: true,
+		},
+		mqttTemplate_id: {
+			type: DataTypes.INTEGER,
+			allowNull: true,
+		},
+		synch: {
+			type: DataTypes.INTEGER,
+			default: 0,
+			allowNull: true,
+		},
+		synched: {
+			type: DataTypes.INTEGER,
+			default: 0,
+			allowNull: true,
 		}
 	},
 	{

package/models/mqttTemplate.models.js

@@ -13,10 +13,18 @@ module.exports = (sequelize,DataTypes)=>{
 			type: DataTypes.JSON,
 			allowNull: true	
 		},
+		localData: {
+			type: DataTypes.JSON,
+			allowNull: true	
+		},
 		readInterval: {
 			type: DataTypes.INTEGER,
 			allowNull: true
 		},
+		synch: {
+			type: DataTypes.INTEGER,
+			allowNull: true
+		},
 		template_id: {
 			type: DataTypes.INTEGER,
 			allowNull: false

package/models/sensors.models.js

@@ -26,6 +26,10 @@ module.exports = (sequelize,DataTypes)=>{
 			type: DataTypes.STRING,
 			allowNull: false,
 		},
+		property: {
+			type: DataTypes.STRING,
+			allowNull: false,
+		},
 		graph: {
 			type: DataTypes.JSON,
 			allowNull: true,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mqtt-devices-parser",
-  "version": "1.0.22",
+  "version": "1.0.24",
   "description": "",
   "main": "index.js",
   "scripts": {

package/src/db/device.js

@@ -194,7 +194,6 @@ var self = module.exports = {
 		});
 	},
 
-
 	addLog : async(id,column,value)=>{
 		return new Promise((resolve,reject) => {
 
@@ -255,26 +254,26 @@ var self = module.exports = {
 
 	getLocalSettings : async(deviceId)=>{
 
-  		return new Promise((resolve,reject) => {
+		return new Promise((resolve,reject) => {
 
-		    let query = "SELECT local_settings FROM ?? where id = ?";
-		    let table = ["devices", deviceId];
-		    query = mysql.format(query,table);
+	    let query = "SELECT local_settings FROM ?? where id = ?";
+	    let table = ["devices", deviceId];
+	    query = mysql.format(query,table);
 
-		    $.db.queryRow(query)
-		    .then( rows => {
-		    	if(rows?.length > 0)
-			  		return resolve(rows[0]?.local_settings);
-			  	else
-			  		return resolve(null);
-		    })
-		    .catch( err => {
-		      return reject(err);
-		    });
+	    $.db.queryRow(query)
+	    .then( rows => {
+	    	if(rows?.length > 0)
+		  		return resolve(rows[0]?.local_settings);
+		  	else
+		  		return resolve(null);
+	    })
+	    .catch( err => {
+	      return reject(err);
+	    });
 		});
-  	},
+	},
 
-  updateLocalSettings : async(settings, deviceId)=>{
+	updateLocalSettings : async(settings, deviceId)=>{
 
 		return new Promise((resolve,reject) => {
 			let obj = {
@@ -299,7 +298,7 @@ var self = module.exports = {
 		});
 	},
 
-  getRemoteSettings : async(deviceId)=>{
+	getRemoteSettings : async(deviceId)=>{
 
 		return new Promise((resolve,reject) => {
 
@@ -408,6 +407,103 @@ var self = module.exports = {
 	      return resolve(null);
 	    });
 	  });
-	}
+	},
+
+	getMqttTopic : async(deviceId, topic)=>{
+		return new Promise((resolve,reject) => {
+
+	    let query = "SELECT * FROM ?? where device_id = ? and topic = ?";
+	    let args = ["mqtt",deviceId,topic];
+	    query = mysql.format(query,args);
+
+	    $.db.queryRow(query)
+	    .then( rows => {
+	      if(rows.length > 0)
+	        return resolve(rows[0]);
+	      else
+	        return resolve(null);
+	    })
+	    .catch( err => {
+	      console.log(err);
+	      return resolve(null);
+	    });
+	  });
+	},
 
+	updateLocalTopic : async(id, data)=>{
+
+		let jsonData = {
+			value : data
+		};
+		
+		return new Promise((resolve,reject) => {
+			let obj = {
+				localData : JSON.stringify(jsonData),
+				updatedAt : moment().utc().format('YYYY-MM-DD HH:mm:ss')
+			}
+
+			let filter = {
+				id
+			};
+
+		    $.db.update("mqtt",obj,filter)
+		    .then (rows => {
+		      return resolve(rows[0]);
+		    })
+		    .catch(error => {
+		      return reject(error);
+		    });
+		    
+		});
+	},
+
+	updateRemoteTopic : async(id, data)=>{
+
+		let jsonData = {
+			value : data
+		};
+
+		return new Promise((resolve,reject) => {
+			let obj = {
+				remoteData : JSON.stringify(jsonData),
+				updatedAt : moment().utc().format('YYYY-MM-DD HH:mm:ss')
+			}
+
+			let filter = {
+				id
+			};
+			
+		    $.db.update("mqtt",obj,filter)
+		    .then (rows => {
+		      return resolve(rows[0]);
+		    })
+		    .catch(error => {
+		      return reject(error);
+		    });
+		    
+		});
+	},
+
+	setSynchedTopic : async(id, data)=>{
+		
+		return new Promise((resolve,reject) => {
+			let obj = {
+				synched : data ? 1 : 0,
+				updatedAt : moment().utc().format('YYYY-MM-DD HH:mm:ss')
+			}
+
+			let filter = {
+				id,
+			};
+
+		    $.db.update("mqtt",obj,filter)
+		    .then (rows => {
+		      return resolve(rows[0]);
+		    })
+		    .catch(error => {
+		      return reject(error);
+		    });
+		    
+		});
+	},
 }

package/src/device/device.js

@@ -1,4 +1,5 @@
 const moment = require('moment');
+const { isDeepStrictEqual } = require('node:util');
 
 var _project = [];
 const logs_table = "logs_sensor";
@@ -281,6 +282,7 @@ async function parseMqttMessage(client, project_name, device, topic, payload, re
   if(topic.endsWith("/get"))
     return;
 
+  const topicBck = topic;
   //console.log("[MQTT] parse topic: ",topic);
 
   try{
@@ -296,6 +298,31 @@ async function parseMqttMessage(client, project_name, device, topic, payload, re
         $.db_device.update(device.id, "status", payload);
         $.db_device.addLog(device.id,"status",payload);
       }
+
+      if (payload === "online"){
+        let mqtt_prefix = `${project_name}/${device.uid}`;
+        // check if remote settings are known, if not require it
+        if(!device.remote_settings?.log){
+          let topic = `${mqtt_prefix}/settings/log/get`
+          $.mqtt_client.publish(topic,"",{qos:1,retain:false});
+        }
+        if(!device.remote_settings?.keepalive){
+          let topic = `${mqtt_prefix}/settings/keepalive/get`
+          $.mqtt_client.publish(topic,"",{qos:1,retain:false});
+        }
+        if(!device.remote_settings?.modem){
+          let topic = `${mqtt_prefix}/settings/modem/get`
+          $.mqtt_client.publish(topic,"",{qos:1,retain:false});
+        }
+        if(!device.remote_settings?.wifi){
+          let topic = `${mqtt_prefix}/settings/wifi/get`
+          $.mqtt_client.publish(topic,"",{qos:1,retain:false});
+        }
+        if(!device.remote_settings?.mqtt){
+          let topic = `${mqtt_prefix}/settings/mqtt/get`
+          $.mqtt_client.publish(topic,"",{qos:1,retain:false});
+        }
+      }
       return;
       break;
     case "model":
@@ -308,14 +335,14 @@ async function parseMqttMessage(client, project_name, device, topic, payload, re
       return;
       break;
     case "tech":
-      if (device?.tech && payload != device.tech) {
+      if (payload != null && payload != device?.tech) {
         $.db_device.update(device.id, "tech", payload);
         $.db_device.addLog(device.id,"tech",payload);
       }
       return;
       break;
     case "version":
-      if (device?.version && payload != device.version) {
+      if (payload != null && payload != device?.version) {
         $.db_device.addLog(device.id,"version",payload);
         $.db_device.update(device.id, "version", payload);
         handleFotaSuccess(device.id);
@@ -323,7 +350,7 @@ async function parseMqttMessage(client, project_name, device, topic, payload, re
       return;
       break;
     case "app_version":
-      if (device?.app_version && payload != device.app_version) {
+      if (payload != null && payload != device?.app_version) {
         $.db_device.addLog(device.id,"app_version",payload);
         $.db_device.update(device.id, "app_version", payload);
         handleFotaSuccess(device.id);
@@ -370,6 +397,45 @@ async function parseMqttMessage(client, project_name, device, topic, payload, re
     // Optional: default case if needed
     default:
       // handle other topics or do nothing
+      // check if topic is associated with device
+      let findTopic = topicBck;
+      if(topicBck.endsWith("/set")) // remove set if exists on topic
+        findTopic = getWordBeforeLastSlash(topicBck);
+
+      const dbTopic = await $.db_device.getMqttTopic(device.id,findTopic)
+      if(dbTopic != null){
+        if(topicBck.endsWith("/set")){
+          // update local topic
+          $.db_device.updateLocalTopic(dbTopic.id,payload)
+          .then(()=>{
+            // set as not synched
+            $.db_device.setSynchedTopic(dbTopic.id,false);
+          })
+          .catch((error)=>{
+            console.error(error);
+          })
+        }else{
+          // update remote topic
+          $.db_device.updateRemoteTopic(dbTopic.id,payload)
+          .then(async (res)=>{
+            // check if topics mismatch
+            if(isDeepStrictEqual(dbTopic?.localData, dbTopic?.remoteData)){
+              $.db_device.setSynchedTopic(dbTopic.id,true);
+            }else{
+              // set as not synched
+              await $.db_device.setSynchedTopic(dbTopic.id,false);
+              if(dbTopic?.synch){ // check if sync is enabled
+                // synch topic
+                synchMqttTopic(device,dbTopic,findTopic);
+              }
+            }
+          })
+          .catch((error)=>{
+            console.error(error);
+          })
+
+        }
+      }
       break;
   }
 
@@ -378,7 +444,26 @@ async function parseMqttMessage(client, project_name, device, topic, payload, re
   }
 }
 
-async function updateLocalSettings(device, topic, payload){
+async function synchMqttTopic(device,dbTopic,topicBck){
+  
+  const project = await $.db_project.getById(device.project_id);
+  let mqtt_prefix = `${project.name}/${device.uid}`;
+  let topic = `${mqtt_prefix}/${topicBck}/set`
+  let payload = "";
+  if(dbTopic?.remoteData && typeof dbTopic?.remoteData === 'object'){
+    try{
+      payload = JSON.stringify(dbTopic?.remoteData);
+    }catch(err){
+      console.error(err);
+      return;
+    }
+  }else{
+    payload = dbTopic?.remoteData;
+  }
+  $.mqtt_client.publish(topic,payload,{qos:1,retain:false});
+}
+
+async function updateLocalSettings(device,topic,payload){
 
   $.db_device.addLog(device.id,"local_settings",JSON.stringify(payload));
 
@@ -469,11 +554,41 @@ async function updateRemoteSettings(device,topic,payload){
 
   try {
     await $.db_device.updateRemoteSettings(JSON.stringify(settings), device.id);
+    if(device?.synch)
+      synchSettings(device,route[0]);
   } catch (err) {
     console.error("Failed to update local settings:", err);
   }
 }
 
+async function synchSettings(device,key){
+
+  console.log(`check topic ${key} from ${device.uid}`);
+  // Retrieve existing settings
+  let localSettings = await $.db_device.getLocalSettings(device.id);
+  console.log(localSettings)
+  let remoteSettings = await $.db_device.getRemoteSettings(device.id);
+  console.log(remoteSettings)
+
+  //let keys = await checkSettings(localSettings,remoteSettings);
+  if(localSettings?.hasOwnProperty(key) && remoteSettings?.hasOwnProperty(key)){
+    if(!isDeepStrictEqual(localSettings?.[key], remoteSettings?.[key])){
+      const project = await $.db_project.getById(device.project_id);
+      let mqtt_prefix = `${project.name}/${device.uid}`;
+      let topic = `${mqtt_prefix}/settings/${key}/set`
+      try{
+        const payload = JSON.stringify(localSettings[key]);
+        console.log(`updating key: ${key} with data:`);
+        console.log(payload);
+        $.mqtt_client.publish(topic,payload,{qos:1,retain:false});
+      }catch(err){
+        console.error(localSettings[key])
+        console.error(err);
+      }
+    }
+  }
+}
+
 async function updateSensor(device,ref,payload){
   let res = await $.db_device.getSensorByRef(device.id,ref)
   if(res == null)
@@ -486,6 +601,14 @@ async function updateSensor(device,ref,payload){
   error = null;
   timestamp = null;
 
+  if(res?.type.toLowerCase() === "json"){
+    if(res?.property && payload.hasOwnProperty(res?.property)){
+      object = payload[res.property];
+    }else{
+      return;
+    }
+  }
+
   if (typeof object === 'object' && object !== null) {
     value = object?.value || object?.v;
     error = object?.error || object?.e;
@@ -553,3 +676,12 @@ function getWordAfterLastSlash(str){
   }
   return str.substring(lastSlashIndex + 1);
 }
+
+function getWordBeforeLastSlash(str){
+  const lastSlashIndex = str.lastIndexOf('/');
+  if (lastSlashIndex === -1) {
+    // No slash found, return the original string
+    return str;
+  }
+  return str.substring(0,lastSlashIndex);
+}

package/src/kafka/consumer.js

@@ -58,9 +58,17 @@ var self = module.exports = {
 
         kafka = new Kafka(kafkaConfig);
 
+        let kafkaGroupId = "";
+        if(config.env === 'development'){
+          const n = Math.floor(Math.random() * 1000) + 1; // 1..1000
+          kafkaGroupId = `${config.kafka.groupId}-${String(n)}`;
+        }else{
+          kafkaGroupId = config.kafka.groupId;
+        }
+
         // Create consumer with shared subscription support
         consumer = kafka.consumer({
-          groupId: config.kafka.groupId,
+          groupId: kafkaGroupId,
           sessionTimeout: 30000,
           rebalanceTimeout: 60000,
           heartbeatInterval: 3000,
@@ -69,7 +77,7 @@ var self = module.exports = {
           maxBytes: 10485760,
           maxWaitTimeInMs: 5000,
           retry: {
-            initialRetryTime: 100,
+            initialRetryTime: 300,
             retries: 8
           }
         });