mqtt-devices-parser 1.0.28 → 1.0.30

package/Changelog.md

@@ -1,12 +1,23 @@
 # Changelog
 
+## 1.0.30
+	fix: tests, recover "settings" parsing, deal with new call addMqttMsgLog
+	doc: update Release
+	fix: handle ER_DUP_ENTRY race condition in fota.update() (#14)
+	perf(src/device/device): clean and parse mqtt payload
+
+## 1.0.29
+	fix(models/sensors): allow null model_id
+	fix: synching mqtt topics
+	feat: stores mqtt messages
+
 ## 1.0.28
 	test: Add comprehensive Jest test suite for device.js module (#2)
 	feat: scope FOTA checks to device variant_id (#11)
 	doc(device): recover readme add test.md file
 	fix(models/variants): remove unique true from column name
 	ci: fix vulnerabilities
-	
+
 ## 1.0.27
 	feat(db): adds variants table
 	

package/Release.md

@@ -2,8 +2,9 @@
 
 ## Before release
 
+  run npm test
   increases version on package.json
-  run npm install
+  run npm install (check vulnerabilities)
   edit changelog
   commit and push to repo
 

package/models/logs_mqtt.models.js

@@ -0,0 +1,29 @@
+
+module.exports = (sequelize,DataTypes)=>{
+	return sequelize.define("logs_mqtt", {
+		mqtt_id: {
+			type: DataTypes.INTEGER,
+			allowNull: false,
+		},
+		device_id: {
+			type: DataTypes.INTEGER,
+			allowNull: false,
+		},
+		source: {
+			type: DataTypes.STRING,
+			allowNull: false
+		},
+		action: {
+			type: DataTypes.STRING,
+			allowNull: false
+		},
+		payload: {
+			type: DataTypes.STRING,
+			allowNull: true
+		},
+	},
+	{
+		tableName: 'logs_mqtt',
+		freezeTableName: true
+	})
+}

package/models/logs_mqtt_msgs.models.js

@@ -0,0 +1,29 @@
+
+module.exports = (sequelize,DataTypes)=>{
+	return sequelize.define("logs_mqtt_msgs", {
+		device_id: {
+			type: DataTypes.INTEGER,
+			allowNull: false,
+		},
+		topic: {
+			type: DataTypes.STRING,
+			allowNull: false
+		},
+		payload: {
+			type: DataTypes.STRING,
+			allowNull: true
+		},
+		qos: {
+			type: DataTypes.INTEGER,
+			allowNull: false
+		},
+		retain: {
+			type: DataTypes.INTEGER,
+			allowNull: false
+		},
+	},
+	{
+		tableName: 'logs_mqtt_msgs',
+		freezeTableName: true
+	})
+}

package/models/sensors.models.js

@@ -3,7 +3,7 @@ module.exports = (sequelize,DataTypes)=>{
 	return sequelize.define("sensors", {
 		model_id: { // reference a model
 			type: DataTypes.INTEGER,
-			allowNull: false,
+			allowNull: true,
 			references: {
 				model: 'models',
 				key: 'id'

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mqtt-devices-parser",
-  "version": "1.0.28",
+  "version": "1.0.30",
   "description": "",
   "main": "index.js",
   "scripts": {

package/src/db/device.js

@@ -216,6 +216,56 @@ var self = module.exports = {
 		});
 	},
 
+	addMqttLog : async(deviceId,dbTopicId,from,action,payload)=>{
+		return new Promise((resolve,reject) => {
+
+			const timestamp = moment().utc().format('YYYY-MM-DD HH:mm:ss')
+			let obj = {
+				createdAt : timestamp,
+				updatedAt : timestamp
+			};
+
+			obj['device_id'] = deviceId;
+			obj['mqtt_id'] = dbTopicId;
+			obj['source'] = from;
+			obj['action'] = action;
+			obj['payload'] = payload;
+
+			$.db.insert("logs_mqtt",obj)
+			.then (rows => {
+				return resolve(rows);
+			})
+			.catch(error => {
+				return reject(error);
+			});
+		});
+	},
+
+	addMqttMsgLog : async(deviceId,topic,payload,retain,qos)=>{
+		return new Promise((resolve,reject) => {
+
+			const timestamp = moment().utc().format('YYYY-MM-DD HH:mm:ss')
+			let obj = {
+				createdAt : timestamp,
+				updatedAt : timestamp
+			};
+
+			obj['device_id'] = deviceId;
+			obj['topic'] = topic;
+			obj['payload'] = payload;
+			obj['retain'] = retain;
+			obj['qos'] = qos;
+
+			$.db.insert("logs_mqtt_msgs",obj)
+			.then (rows => {
+				return resolve(rows);
+			})
+			.catch(error => {
+				return reject(error);
+			});
+		});
+	},
+
 	addLogIfChanged : async(id,column,value)=>{
 		return new Promise(async (resolve,reject) => {
 

package/src/db/fota.js

@@ -96,7 +96,14 @@ var self = module.exports = {
 			      return resolve(rows[0]);
 			    })
 			    .catch(error => {
-			      return reject(error);
+			      if(error.code === 'ER_DUP_ENTRY'){
+			        let filter = { device_id : deviceId };
+			        $.db.update("fota",obj,filter)
+			        .then(rows => { return resolve(rows[0]); })
+			        .catch(err => { return reject(err); });
+			      }else{
+			        return reject(error);
+			      }
 			    });
 
 			}else{

package/src/device/device.js

@@ -33,7 +33,7 @@ var self = module.exports = {
     });
   },
 
-  parseMessage : async (client,topic,payload,retain)=>{
+  parseMessage : async (client,topic,payload,retain,qos=0)=>{
 
     let topic_bck = topic;
     let project_name = null;
@@ -77,6 +77,7 @@ var self = module.exports = {
     if(device.protocol.toLowerCase() === "lwm2m"){
       parseLwm2mMessage(client, project_name, device, topic, payload, action);
     }else if(device.protocol.toLowerCase() === "mqtt"){
+      $.db_device.addMqttMsgLog(device.id,topic,payload,retain,qos);
       parseMqttMessage(client, project_name, device, topic, payload, retain);
     }
   },
@@ -266,7 +267,6 @@ var self = module.exports = {
     }
   },
 
-
   updateSensor : async (device,ref,payload)=>{
 
     let sensors = await $.db_device.getSensorsByRef(device.id,ref)
@@ -319,17 +319,36 @@ var self = module.exports = {
 
   handleMqttTopic : async(device, dbTopic, payload, set)=>{
     if(set){
+      if(payload === "" || payload == null){
+        // acknowledgment
+        $.db_device.setSynchedTopic(dbTopic.id,true);
+        $.db_device.updateRemoteTopic(dbTopic.id,dbTopic?.localData?.value ? dbTopic.localData.value : dbTopic?.localData);
+        let from = "device";
+        let action = "ack"
+        $.db_device.addMqttLog(device.id,dbTopic.id,from,action,null);
+        return;
+      }
       // update local topic
+      let from = "server";
+      let action = "write";
+      $.db_device.addMqttLog(device.id,dbTopic.id,from,action,JSON.stringify(payload));
+
       $.db_device.updateLocalTopic(dbTopic.id,payload)
       .then(()=>{
-        // set as not synched
-        $.db_device.setSynchedTopic(dbTopic.id,false);
+        if(isDeepStrictEqual(payload, dbTopic?.remoteData))
+          $.db_device.setSynchedTopic(dbTopic.id,true);
+        else
+          $.db_device.setSynchedTopic(dbTopic.id,false);
       })
       .catch((error)=>{
         console.error(error);
       })
     }else{
       // update remote topic
+      let from = "device";
+      let action = "update";
+      $.db_device.addMqttLog(device.id,dbTopic.id,from,action,JSON.stringify(payload));
+
       $.db_device.updateRemoteTopic(dbTopic.id,payload)
       .then(async (res)=>{
         // check if topics mismatch
@@ -383,14 +402,24 @@ async function parseLwm2mMessage(client, project_name, device, topic, payload, a
 
 async function parseMqttMessage(client, project_name, device, topic, payload, retain){
 
-  if(topic.endsWith("/get"))
+  if(topic.endsWith("/get")){
+    let findTopic = $.parser.getWordBeforeLastSlash(topic);
+    const dbTopic = await $.db_device.getMqttTopic(device.id,findTopic)
+    if(dbTopic != null){
+      let from = "server";
+      let action = "request";
+      let data = null;
+      $.db_device.addMqttLog(device.id,dbTopic.id,from,action,data);
+    }
     return;
+  }
 
   const topicBck = topic;
   //console.log("[MQTT] parse topic: ",topic);
 
   try{
     payload = JSON.parse(payload);
+    payload = cleanAndParse(payload);
   }catch(error){}
 
   let word = $.parser.getFirstWord(topic);
@@ -485,7 +514,7 @@ async function parseMqttMessage(client, project_name, device, topic, payload, re
         }
       }
       break;
-    case "settings":
+    case "settings": // deprecated
       if(topic.endsWith("/set")){
         updateLocalSettings(device,topic,payload);
       }else{
@@ -525,19 +554,25 @@ async function synchMqttTopic(device,dbTopic){
   let mqtt_prefix = `${project.name}/${device.uid}`;
   let topic = `${mqtt_prefix}/${dbTopic.topic}/set`
   let payload = "";
-  if(dbTopic?.remoteData && typeof dbTopic?.remoteData === 'object'){
+  if(dbTopic?.localData && typeof dbTopic?.localData === 'object'){
     try{
-      payload = JSON.stringify(dbTopic?.remoteData);
+      if(dbTopic?.localData?.value)
+        payload = JSON.stringify(dbTopic?.localData?.value);
+      else
+        payload = JSON.stringify(dbTopic?.localData);
     }catch(err){
       console.error(err);
       return;
     }
   }else{
-    payload = dbTopic?.remoteData;
+    payload = dbTopic?.localData;
   }
   $.mqtt_client.publish(topic,payload,{qos:1,retain:false});
 }
 
+// keep a struct with all settings configured on the device
+// only topics <project>/<uid>/settings are stored on this struct
+// easier way to return all settings at once
 async function updateLocalSettings(device,topic,payload){
 
   $.db_device.addLog(device.id,"local_settings",JSON.stringify(payload));
@@ -586,6 +621,9 @@ async function updateLocalSettings(device,topic,payload){
   }
 }
 
+// keep a struct with all settings configured on the device
+// only topics <project>/<uid>/settings are stored on this struct
+// easier way to return all settings at once
 async function updateRemoteSettings(device,topic,payload){
 
   $.db_device.addLog(device.id,"remote_settings",JSON.stringify(payload));
@@ -629,13 +667,15 @@ async function updateRemoteSettings(device,topic,payload){
 
   try {
     await $.db_device.updateRemoteSettings(JSON.stringify(settings), device.id);
-    if(device?.synch)
+    // deprecated, only defined topics with synch enabled can be synched
+    if(device?.synch && false)
       synchSettings(device,route[0]);
   } catch (err) {
     console.error("Failed to update local settings:", err);
   }
 }
 
+// deprecated - only defined mqtt topics can be synched
 async function synchSettings(device,key){
 
   console.log(`check topic ${key} from ${device.uid}`);
@@ -646,6 +686,7 @@ async function synchSettings(device,key){
   console.log(remoteSettings)
 
   //let keys = await checkSettings(localSettings,remoteSettings);
+  /*
   if(localSettings?.hasOwnProperty(key) && remoteSettings?.hasOwnProperty(key)){
     if(!isDeepStrictEqual(localSettings?.[key], remoteSettings?.[key])){
       const project = await $.db_project.getById(device.project_id);
@@ -662,9 +703,9 @@ async function synchSettings(device,key){
       }
     }
   }
+  */
 }
 
-
 function handleFotaSuccess (deviceId){
   let object = {
     "nAttempts" : 0,
@@ -684,4 +725,18 @@ function handleFotaError (deviceId, error){
   $.db_fota.updateLog(deviceId,object);
 }
 
-
+function cleanAndParse(payload) {
+  if (typeof payload === 'string') {
+    let cleaned = payload.trim();
+    if (cleaned.startsWith('"') && cleaned.endsWith('"')) {
+      cleaned = cleaned.slice(1, -1);
+    }
+    cleaned = cleaned.replace(/\\"/g, '"');
+    try {
+      return JSON.parse(cleaned);
+    } catch (e) {
+      return payload; // fallback: return original if still invalid
+    }
+  }
+  return payload;
+}

package/src/device/device.test.js

@@ -29,7 +29,8 @@ global.$ = {
     updateLocalTopic: jest.fn(),
     setSynchedTopic: jest.fn(),
     updateRemoteTopic: jest.fn(),
-    getAssociatedDevice: jest.fn()
+    getAssociatedDevice: jest.fn(),
+    addMqttMsgLog: jest.fn()
   },
   db_model: {
     getByName: jest.fn(),

package/src/kafka/consumer.js

@@ -140,6 +140,7 @@ var self = module.exports = {
 
             try{
               payload = JSON.parse(payload);
+              //console.log(`[KAFKA] payload: ${payload.mqtt.payload}`);
             }catch(error){
               console.log(`[KAFKA] error: ${payload}`);
               console.log(error)