mqtt-devices-parser 1.0.8 → 1.0.10

package/Changelog.md

@@ -1,7 +1,18 @@
 # Changelog
 
+## 1.0.10
+	src/db/data: handles json data
+	- Adds updateJson call
+	- Adds addJsonLog
+	src/db/project: deals with settings
+	src/device/device: fixes fota fw link, stores settings on project table (twin model)
+	
+## 1.0.9
+	- Adds method to delete logs older than 1 week
+
 ## 1.0.8
 	- removes port from link to update fw devices
+	- fixes fw_version comparison
 
 ## 1.0.7
 	- config/index: changes local IP

package/index.js

@@ -41,6 +41,11 @@ var self = module.exports = {
       console.log("mqtt connecting..")
       mqtt_connect();
 
+      setInterval(async ()=>{
+        await $.device.deleteLogs();
+      },60*60*1000);
+      
+
       return resolve();
     })
   },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mqtt-devices-parser",
-  "version": "1.0.8",
+  "version": "1.0.10",
   "description": "",
   "main": "index.js",
   "scripts": {

package/src/db/data.js

@@ -5,24 +5,24 @@ var self = module.exports = {
 
 	checkEntry : async (table,deviceId)=>{
 
-	  return new Promise((resolve,reject) => {
-
-	    let query = "SELECT * FROM ?? where device_id = ?";
-	    let args = [table,deviceId];
-	    query = mysql.format(query,args);
-
-	    $.db.queryRow(query)
-	    .then( rows => {
-	      if(rows.length > 0)
-	        return resolve(rows[0]);
-	      else
-	        return resolve(null);
-	    })
-	    .catch( err => {
-	      console.log(err);
-	      return resolve(null);
-	    });
-	  });
+		return new Promise((resolve,reject) => {
+
+		let query = "SELECT * FROM ?? where device_id = ?";
+		let args = [table,deviceId];
+		query = mysql.format(query,args);
+
+		$.db.queryRow(query)
+		.then( rows => {
+			if(rows.length > 0)
+				return resolve(rows[0]);
+			else
+				return resolve(null);
+		})
+		.catch( err => {
+			console.log(err);
+			return resolve(null);
+			});
+		});
 	},
 
 	update : async (table, deviceId, topic, payload)=>{
@@ -54,41 +54,41 @@ var self = module.exports = {
 				topic = topic.substring(index+1);
 
 				if(db_columns.hasOwnProperty(key)){ // check if db has a column for this key
-				  // create JSON struct with the remaining topic
-				  // table is the table
-				  // key is the column
-
-				  // build object with the remaining topic and respective data
-				  if(typeof data === "object")
-				    obj[key] = JSON.stringify(parser.pathIntoObject(topic,JSON.stringify(data)))
-				  else
-				    obj[key] = JSON.stringify(parser.pathIntoObject(topic,data));
-
-				  let exists = await self.checkEntry(table,deviceId);
-				  if(!exists){
-				    obj['device_id'] = deviceId;
-				    obj['createdAt'] = moment().utc().format('YYYY-MM-DD HH:mm:ss');
-				    $.db.insert(table,obj)
-				    .then((rows)=>{
-				    	return resolve(rows);
-				    })
-				    .catch((err)=>{
-				    	console.log(err);
-				    	return resolve();
-				    })
-				  }else{
-				    let filter = {
-				      device_id : deviceId
-				    }
-				    $.db.update(table,obj,filter)
-				    .then((rows)=>{
-				    	return resolve(rows);
-				    })
-				    .catch((err)=>{
-				    	console.log(err);
-				    	return resolve();
-				    })
-				  }
+					// create JSON struct with the remaining topic
+					// table is the table
+					// key is the column
+
+					// build object with the remaining topic and respective data
+					if(typeof data === "object")
+						obj[key] = JSON.stringify(parser.pathIntoObject(topic,JSON.stringify(data)))
+					else
+						obj[key] = JSON.stringify(parser.pathIntoObject(topic,data));
+
+					let exists = await self.checkEntry(table,deviceId);
+					if(!exists){
+						obj['device_id'] = deviceId;
+						obj['createdAt'] = moment().utc().format('YYYY-MM-DD HH:mm:ss');
+						$.db.insert(table,obj)
+						.then((rows)=>{
+							return resolve(rows);
+						})
+						.catch((err)=>{
+							console.log(err);
+							return resolve();
+						})
+					}else{
+						let filter = {
+							device_id : deviceId
+						}
+						$.db.update(table,obj,filter)
+						.then((rows)=>{
+							return resolve(rows);
+						})
+						.catch((err)=>{
+							console.log(err);
+							return resolve();
+						})
+					}
 
 				}else return resolve();
 			}else{
@@ -115,7 +115,7 @@ var self = module.exports = {
 						})
 					}else{
 						let filter = {
-						  device_id : deviceId
+							device_id : deviceId
 						}
 
 						$.db.update(table,obj,filter)
@@ -132,28 +132,195 @@ var self = module.exports = {
 		})
 	},
 
+	updateJson : async (table, deviceId, payload)=>{
+
+		return new Promise( async (resolve, reject) => {
+		
+			if (!payload || typeof payload !== 'object') 
+				return resolve();
+
+			let obj = {
+				updatedAt: moment().utc().format('YYYY-MM-DD HH:mm:ss'),
+				device_id: deviceId,
+			};
+
+			// Get columns info
+			const db_columns = $.models.get(table);
+			if (db_columns == null) 
+				return resolve();
+
+			// Prepare data for insertion
+			for (let key in payload) {
+				if (db_columns.hasOwnProperty(key)) {
+					let value = payload[key];
+
+					// Convert object to JSON string if needed
+					if (typeof value === 'object') {
+						value = JSON.stringify(value);
+					}
+
+					obj[key] = value;
+				}
+			}
+
+			let exists = await self.checkEntry(table,deviceId);
+			if(!exists){
+				obj['createdAt'] = moment().utc().format('YYYY-MM-DD HH:mm:ss');
+				$.db.insert(table,obj)
+				.then((rows)=>{
+					return resolve(rows);
+				})
+				.catch((err)=>{
+					console.log(err);
+					return resolve();
+				})
+			}else{
+				let filter = {
+					device_id : deviceId
+				}
+				$.db.update(table,obj,filter)
+				.then((rows)=>{
+					return resolve(rows);
+				})
+				.catch((err)=>{
+					console.log(err);
+					return resolve();
+				})
+			}
+
+			return resolve();
+		});
+	},
+
 	addLog : async (table, deviceId, topic, payload)=>{
 
 	  return new Promise((resolve,reject) => {
 
 	    if(!payload) return resolve();
 
-		let data = null;
-		// check if data is in JSON format
-		try{
-			data = JSON.parse(payload);
-		}catch(e){
-			data = payload;
-		}
+			let data = null;
+			// check if data is in JSON format
+			try{
+				data = JSON.parse(payload);
+			}catch(e){
+				data = payload;
+			}
 
-		let obj = {
-			updatedAt : moment().utc().format('YYYY-MM-DD HH:mm:ss')
-		}
+			let obj = {
+				updatedAt : moment().utc().format('YYYY-MM-DD HH:mm:ss')
+			}
+
+			let db_columns = $.models.get(table);
+			if(db_columns == null)
+				return resolve();
+
+			let index = topic.indexOf("/");
+
+			if(index > -1){ // check if topic has subtopics
+				let key = topic.substring(0,index);
+				topic = topic.substring(index+1);
+
+				if(db_columns.hasOwnProperty(key)){ // check if db has a column for this key
+					// create JSON struct with the remaining topic
+					// key is the column
+
+					// build object with the remaining topic and respective data
+					if(typeof data === "object")
+						obj[key] = JSON.stringify(parser.pathIntoObject(topic,JSON.stringify(data)))
+					else
+						obj[key] = JSON.stringify(parser.pathIntoObject(topic,data));
+
+					obj['device_id'] = deviceId;
+					obj['createdAt'] = moment().utc().format('YYYY-MM-DD HH:mm:ss');
+					$.db.insert(table,obj)
+					.then((rows)=>{
+						return resolve(rows);
+					})
+					.catch((err)=>{
+						console.log(err);
+						return resolve();
+					})
+
+				}else
+					return resolve();
+			}else{
+
+				let key = topic;
+				if(db_columns.hasOwnProperty(key)){ // check if db has a column for this key
+					// build object with respective data
+					if(typeof data === "object")
+						obj[key] = JSON.stringify(data);
+					else
+						obj[key] = data;
+
+					obj['device_id'] = deviceId;
+					obj['createdAt'] = moment().utc().format('YYYY-MM-DD HH:mm:ss');
+					$.db.insert(table,obj)
+					.then((rows)=>{
+						return resolve(rows);
+					})
+					.catch((err)=>{
+						console.log(err);
+						return resolve();
+					})
+				}else
+					return resolve();
+			}
+
+	  });
+	},
+
+	addJsonLog: async (table, deviceId, dataObject) => {
+		return new Promise((resolve, reject) => {
+			if (!dataObject || typeof dataObject !== 'object') 
+				return resolve();
+
+			let obj = {
+				updatedAt: moment().utc().format('YYYY-MM-DD HH:mm:ss'),
+				device_id: deviceId,
+				createdAt: moment().utc().format('YYYY-MM-DD HH:mm:ss')
+			};
+
+			// Get columns info
+			const db_columns = $.models.get(table);
+			if (db_columns == null) 
+				return resolve();
+
+			// Prepare data for insertion
+			for (let key in dataObject) {
+				if (db_columns.hasOwnProperty(key)) {
+					let value = dataObject[key];
+
+					// Convert object to JSON string if needed
+					if (typeof value === 'object') {
+						value = JSON.stringify(value);
+					}
+
+					obj[key] = value;
+				}
+			}
+
+			//console.log("updateJSONlog")
+			$.db.insert(table, obj)
+			.then((rows)=>{
+				return resolve(rows);
+			})
+			.catch((err)=>{
+				console.log(err);
+				return resolve();
+			})
+		});
+	},
+
+	deleteLogs : async (table, deviceId, topic, payload)=>{
+
+	  return new Promise((resolve,reject) => {
 
 		let db_columns = $.models.get(table);
 		if(db_columns == null)
 			return resolve();
 
+		$.db.delete(table,{id:0})
 		let index = topic.indexOf("/");
 
 		if(index > -1){ // check if topic has subtopics
@@ -173,7 +340,13 @@ var self = module.exports = {
 			  obj['device_id'] = deviceId;
 			  obj['createdAt'] = moment().utc().format('YYYY-MM-DD HH:mm:ss');
 			  $.db.insert(table,obj)
-
+			  .then((rows)=>{
+				return resolve(rows);
+			  })
+			.catch((err)=>{
+				console.log(err);
+				return resolve();
+			})
 			}
 		}else{
 
@@ -187,7 +360,7 @@ var self = module.exports = {
 
 			  obj['device_id'] = deviceId;
 			  obj['createdAt'] = moment().utc().format('YYYY-MM-DD HH:mm:ss');
-			  $.db.insert(table,obj);
+			  $.db.insert(table,obj)
 			}
 		}
 

package/src/db/db.js

@@ -264,6 +264,59 @@ var self = module.exports = {
     });
   },
 
+  getTables : async()=>{
+
+    return new Promise((resolve,reject) => {
+
+      self.getConnection((err,conn)=>{
+        if(err) return reject(err);
+
+        let query = "SHOW TABLES;"
+
+        conn.query(query,function(err,rows){
+          self.close_db_connection(conn);
+          if(err) return reject(err)
+          else return resolve(rows);
+        });
+      });
+    });
+  },
+
+  deleteOldEntries : async(table,filter)=>{
+
+    return new Promise((resolve,reject) => {
+
+      self.getConnection((err,conn)=>{
+        if(err) return reject(err);
+
+        let query = "";
+        if(typeof filter === "object"){
+          let values = [];
+          query = `DELETE FROM ?? WHERE `;
+          values.push(table);
+          for (let key in filter){
+            if(key == "createdAt"){
+              query += "createdAt < ?";
+            }else if(key = "id")
+              query += "id < ?"
+            else 
+              return reject("key not supported");
+            values.push(filter[key]);
+          }
+          query = mysql.format(query,values);
+        }else{
+          return reject("filter passed is not an object");
+        }
+
+        conn.query(query,function(err,rows){
+          self.close_db_connection(conn);
+          if(err) return reject(err)
+          else return resolve(rows);
+        });
+      });
+    });
+  },
+
   getFieldFromDeviceId : async (table,deviceId,field)=>{
 
     return new Promise((resolve,reject) => {

package/src/db/project.js

@@ -20,8 +20,7 @@ var self = module.exports = {
 	        return resolve(null);
 	    })
 	    .catch( err => {
-	      console.log(err);
-	      return resolve(null);
+	      return reject(err);
 	    });
 	  });
 	},
@@ -42,8 +41,7 @@ var self = module.exports = {
 	        return resolve(null);
 	    })
 	    .catch( err => {
-	      console.log(err);
-	      return resolve(null);
+	      return reject(err);
 	    });
 	  });
 	},
@@ -61,8 +59,7 @@ var self = module.exports = {
 		  return resolve(rows);
 	    })
 	    .catch( err => {
-	      console.log(err);
-	      return resolve(null);
+	      return reject(null);
 	    });
 	  });
 	},
@@ -88,4 +85,49 @@ var self = module.exports = {
 		});
   	},
 
+  	getSettings : async(project,deviceId)=>{
+
+  		return new Promise((resolve,reject) => {
+
+		    let query = "SELECT settings FROM ?? where device_id = ?";
+		    let table = [project, deviceId];
+		    query = mysql.format(query,table);
+
+		    $.db.queryRow(query)
+		    .then( rows => {
+		    	if(rows?.length > 0)
+			  		return resolve(rows[0].settings);
+			  	else
+			  		return resolve(null);
+		    })
+		    .catch( err => {
+		      return reject(err);
+		    });
+		});
+  	},
+
+  	updateSettings : async(project, settings, deviceId)=>{
+
+		return new Promise((resolve,reject) => {
+			let obj = {
+				settings : settings,
+				updatedAt : moment().utc().format('YYYY-MM-DD HH:mm:ss')
+			}
+
+			let filter = {
+				device_id : deviceId
+			};
+
+			resolve(null)
+			
+		    $.db.update(project,obj,filter)
+		    .then (rows => {
+		      return resolve(rows[0]);
+		    })
+		    .catch(error => {
+		      return reject(error);
+		    });
+		    
+		});
+  	},
 }

package/src/device/device.js

@@ -1,3 +1,4 @@
+const moment = require('moment');
 
 var project = [];
 const logs_table = "sensor_logs";
@@ -31,6 +32,25 @@ var self = module.exports = {
 
   },
 
+  deleteLogs : async()=>{
+
+    let tables = await $.db.getTables();
+    // Define one week in milliseconds
+    // Loop through each table
+    for (const tableObj of tables) {
+      const tableName = tableObj['Tables_in_mqtt-aedes']
+      const oneWeekAgo = moment().utc().subtract(7, 'days').format('YYYY-MM-DD HH:mm:ss');
+      if(tableName.startsWith("logs_")){
+        const uptimeSeconds = Math.floor(process.uptime());
+        console.log(`deleting logs of table ${tableName} older than ${oneWeekAgo}"`);
+        await $.db.deleteOldEntries(tableName, { createdAt: oneWeekAgo });
+        time = Math.floor(process.uptime()) - uptimeSeconds;
+        console.log(`Logs of table ${tableName} deleted in ${time}s`);
+      }
+    }
+
+  },
+
   parseMessage : async (client,topic,payload,retain)=>{
 
     let topic_bck = topic;
@@ -110,15 +130,17 @@ var self = module.exports = {
           return;
 
         let mqtt_prefix = `${project_name}/${uid}`;
-        let link = `${$.config.web.protocol}${$.config.web.domain}${$.config.web.fw_path}${new_firmware.filename}/download?token=${new_firmware.token}`;
 
-        if(new_firmware != null && semver.lt(fw_version, new_firmware.fw_version)) {
-          console.log(`updating firmware of ${uid} to ${new_firmware.fw_version}`);
+        if(new_app != null && semver.lt(app_version, new_app.app_version)) {
+          console.log(new_app)
+          console.log(`updating firmware of ${uid} to minor version ${new_app.app_version}`);
+          let link = `${$.config.web.protocol}${$.config.web.domain}${$.config.web.fw_path}${new_app.filename}/download?token=${new_app.token}`;
           client.publish(mqtt_prefix+"/fw/fota/update/set",`{"url":"${link}"}`,{qos:2,retain:false});
           await $.db_data.update(project_name,device.id,"fota_tries",++fota_tries);
         }else{
-          if(new_app != null && semver.lt(app_version, new_app.app_version)) {
-            console.log(`updating firmware of ${uid} to ${new_app.app_version}`);
+          if(new_firmware != null && semver.lt(fw_version, new_firmware.fw_version)) {
+            console.log(`updating firmware of ${uid} to major version ${new_firmware.fw_version}`);
+            let link = `${$.config.web.protocol}${$.config.web.domain}${$.config.web.fw_path}${new_firmware.filename}/download?token=${new_firmware.token}`;
             client.publish(mqtt_prefix+"/fw/fota/update/set",`{"url":"${link}"}`,{qos:2,retain:false});
             await $.db_data.update(project_name,device.id,"fota_tries",++fota_tries);
           }
@@ -133,6 +155,55 @@ var self = module.exports = {
         }
       }
 
+      // store remote device settings on project table - twin model
+      if(topic.endsWith("/set") && payload != "" ){
+
+        let route = topic.split("/");
+
+        if(route == null){
+          console.log("topic invalid:",topic);
+          return;
+        }
+        // get settings
+        let settings = await $.db_project.getSettings(project_name,device.id);
+        let obj = settings;
+
+        if(obj == null){
+          console.log("no settings available for deviceId:",device.id)
+          obj = {};
+        }
+        // Traverse through all route parts except the last one
+        route.slice(0, -1).forEach(property => {
+          if (!obj.hasOwnProperty(property) || 
+            ( obj.hasOwnProperty(property) && typeof obj[property] !== 'object')) 
+          {
+            obj[property] = {}; // create nested object if missing or not an object
+          }
+          obj = obj[property]; // go deeper
+        })
+        
+        let data = {};
+        try{
+          data = JSON.parse(payload)
+        }catch(error){}
+        
+        if (true || (data && typeof payload === 'object' && !Array.isArray(payload) ) ) {
+          // Assuming payload is an object with properties to process
+          for (const [key, value] of Object.entries(data)) {
+            obj[key] = value; // go deeper
+          };
+        }else{
+          obj = payload
+        }
+
+        // update on device fw settings
+        try{
+          await $.db_project.updateSettings(project_name,JSON.stringify(settings),device.id);
+        }catch(error){
+          console.log(error);
+        }
+      }
+
       if(project[project_name]){
         project[project_name].module.parseMessage(client,project_name,device,topic,payload,retain,()=>{});
       }
@@ -141,3 +212,6 @@ var self = module.exports = {
   },
 
 }
+
+
+