mqtt-devices-parser 1.0.12 → 1.0.14

package/Changelog.md

@@ -1,7 +1,17 @@
 # Changelog
 
+## 1.0.14
+	Fixes check FOTA mechanism and response
+	
+## 1.0.13
+	adds 2 new tables to db:
+	Adds 5 new columns to devices
+	Registers device logs, records device settings..
+	fixes: ws affected by a DoS when handling a request with many HTTP headers !breaking change
+	Updates npm
+
 ## 1.0.12
-	Adds periodically call to check and update devices
+	Adds module fota
 
 ## 1.0.11
 	index: MQTT

package/Release.md

@@ -8,4 +8,7 @@
 
 ## Releasing
 
+	launch release on git:
+		>> gh release create ${tag}
+
   npm publish

package/models/devices.models.js

@@ -43,6 +43,27 @@ module.exports = (sequelize,DataTypes)=>{
 			type: DataTypes.STRING,
 			allowNull: true
 		},
+		remote_settings: { // device settings
+			type: DataTypes.JSON,
+			allowNull: true
+		},
+		local_settings: { // server settings
+			type: DataTypes.JSON,
+			allowNull: true
+		},
+		settings_ref: { // settings to be used, can be a uid, deviceId, default..
+			type: DataTypes.STRING,
+			allowNull: true
+		},
+		associatedDevice: {
+			type: DataTypes.INTEGER,
+			allowNull: true
+		},
+		endpoint: { // info about protocol communication
+			type: DataTypes.JSON,
+			allowNull: true
+		},
+
 	},
 	{
 		tableName: 'devices',

package/models/fota.models.js

@@ -4,6 +4,9 @@ module.exports = (sequelize,DataTypes)=>{
 			type: DataTypes.INTEGER,
 			unique: true,
 		},
+		model_id: {
+			type: DataTypes.INTEGER,
+		},
 		target_version: {
 			type: DataTypes.STRING,
 		},

package/models/logs_actions.models.js

@@ -0,0 +1,17 @@
+
+module.exports = (sequelize,DataTypes)=>{
+	return sequelize.define("logs_actions", {
+		client_id: {
+			type: DataTypes.INTEGER,
+			allowNull: false
+		},
+		action: {
+			type: DataTypes.STRING,
+			allowNull: true
+		},
+	},
+	{
+		tableName: 'logs_actions',
+		freezeTableName: true
+	})
+}

package/models/logs_device.models.js

@@ -0,0 +1,62 @@
+
+module.exports = (sequelize,DataTypes)=>{
+	return sequelize.define("logs_devices", {
+		device_id: {
+			type: DataTypes.INTEGER,
+		},
+		status: {
+			type: DataTypes.STRING,
+			allowNull: true
+		},
+		project_id: {
+			type: DataTypes.INTEGER,
+			references: {
+				model: 'projects',
+				key: 'id'
+			}
+		},
+		model_id: {
+			type: DataTypes.INTEGER,
+			references: {
+				model: 'models',
+				key: 'id'
+			}
+		},
+		version: {
+			type: DataTypes.STRING,
+			allowNull: true
+		},
+		app_version: {
+			type: DataTypes.STRING,
+			allowNull: true
+		},
+		tech: {
+			type: DataTypes.STRING,
+			allowNull: true
+		},
+		associatedDevice: {
+			type: DataTypes.INTEGER,
+			allowNull: true
+		},
+		local_settings: { // server settings
+			type: DataTypes.JSON,
+			allowNull: true
+		},
+		remote_settings: { // server settings
+			type: DataTypes.JSON,
+			allowNull: true
+		},
+		settings_ref: { // settings to be used, can be a uid, deviceId, default..
+			type: DataTypes.STRING,
+			allowNull: true
+		},
+		endpoint: { // info about protocol communication
+			type: DataTypes.JSON,
+			allowNull: true
+		},
+	},
+	{
+		tableName: 'logs_devices',
+		freezeTableName: true
+	})
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mqtt-devices-parser",
-  "version": "1.0.12",
+  "version": "1.0.14",
   "description": "",
   "main": "index.js",
   "scripts": {

package/src/db/data.js

@@ -278,7 +278,7 @@ var self = module.exports = {
 	addJsonLog: async (table, deviceId, dataObject) => {
 		return new Promise((resolve, reject) => {
 			if (!dataObject || typeof dataObject !== 'object') 
-				return resolve();
+				return reject("Not an object");
 
 			let obj = {
 				updatedAt: moment().utc().format('YYYY-MM-DD HH:mm:ss'),
@@ -289,7 +289,7 @@ var self = module.exports = {
 			// Get columns info
 			const db_columns = $.models.get(table);
 			if (db_columns == null) 
-				return resolve();
+				return reject(`No columns for table: ${table}`);
 
 			// Prepare data for insertion
 			for (let key in dataObject) {

package/src/db/device.js

@@ -194,4 +194,199 @@ var self = module.exports = {
 		});
 	},
 
+
+	addLog : async(id,column,value)=>{
+		return new Promise((resolve,reject) => {
+
+			const timestamp = moment().utc().format('YYYY-MM-DD HH:mm:ss')
+			let obj = {
+				createdAt : timestamp,
+				updatedAt : timestamp
+			};
+
+			obj['device_id'] = id;
+			obj[column] = value;
+
+			$.db.insert("logs_devices",obj)
+			.then (rows => {
+				return resolve(rows);
+			})
+			.catch(error => {
+				return reject(error);
+			});
+		});
+	},
+
+	addLogIfChanged : async(id,column,value)=>{
+		return new Promise(async (resolve,reject) => {
+
+			let query = "SELECT ?? FROM ?? where id = ? and ?? is NOT NULL ORDER BY updatedAt DESC LIMIT 1";
+			let table = [column,"logs_devices",id,column];
+			query = mysql.format(query,table);
+
+			try{
+				let rows = await $.db.queryRow(query)
+				
+				if(rows.length > 0){
+					let lastValue = rows[0]?.column
+					if(lastValue == value)
+						resolve();
+				}
+
+				const timestamp = moment().utc().format('YYYY-MM-DD HH:mm:ss')
+				let obj = {
+					createdAt : timestamp,
+					updatedAt : timestamp
+				};
+
+				obj['device_id'] = id;
+				obj[column] = value;
+
+				await $.db.insert("logs_devices",obj)
+
+				resolve();
+
+			}catch (err) {
+				return reject(err);
+			};
+
+		});
+	},
+
+	getLocalSettings : async(deviceId)=>{
+
+  		return new Promise((resolve,reject) => {
+
+		    let query = "SELECT local_settings FROM ?? where id = ?";
+		    let table = ["devices", deviceId];
+		    query = mysql.format(query,table);
+
+		    $.db.queryRow(query)
+		    .then( rows => {
+		    	if(rows?.length > 0)
+			  		return resolve(rows[0]?.local_settings);
+			  	else
+			  		return resolve(null);
+		    })
+		    .catch( err => {
+		      return reject(err);
+		    });
+		});
+  	},
+
+  updateLocalSettings : async(settings, deviceId)=>{
+
+		return new Promise((resolve,reject) => {
+			let obj = {
+				local_settings : settings,
+				updatedAt : moment().utc().format('YYYY-MM-DD HH:mm:ss')
+			}
+
+			let filter = {
+				id : deviceId
+			};
+
+			resolve(null)
+			
+	    $.db.update("devices",obj,filter)
+	    .then (rows => {
+	      return resolve(rows[0]);
+	    })
+	    .catch(error => {
+	      return reject(error);
+	    });
+		    
+		});
+	},
+
+  getRemoteSettings : async(deviceId)=>{
+
+		return new Promise((resolve,reject) => {
+
+	    let query = "SELECT remote_settings FROM ?? where id = ?";
+	    let table = ["devices", deviceId];
+	    query = mysql.format(query,table);
+
+	    $.db.queryRow(query)
+	    .then( rows => {
+	    	if(rows?.length > 0)
+		  		return resolve(rows[0]?.remote_settings);
+		  	else
+		  		return resolve(null);
+	    })
+	    .catch( err => {
+	      return reject(err);
+	    });
+		});
+	},
+
+	updateRemoteSettings : async(settings, deviceId)=>{
+
+		return new Promise((resolve,reject) => {
+			let obj = {
+				remote_settings : settings,
+				updatedAt : moment().utc().format('YYYY-MM-DD HH:mm:ss')
+			}
+
+			let filter = {
+				id : deviceId
+			};
+
+			resolve(null)
+			
+		    $.db.update("devices",obj,filter)
+		    .then (rows => {
+		      return resolve(rows[0]);
+		    })
+		    .catch(error => {
+		      return reject(error);
+		    });
+		    
+		});
+	},
+
+	getAssociatedDevice : async (deviceId)=>{
+
+		return new Promise((resolve,reject) => {
+
+			let query = "SELECT associatedDevice FROM ?? where id = ?";
+			let args = ["devices",deviceId];
+			query = mysql.format(query,args);
+
+			$.db.queryRow(query)
+			.then( rows => {
+				if(rows.length > 0)
+					return resolve(rows[0]?.associatedDevice);
+				else
+					return resolve(null);
+			})
+			.catch( err => {
+				return reject(err);
+			});
+
+		});
+	},
+
+	updateAssociatedDevice : async(deviceId,associatedDeviceId)=>{
+		return new Promise((resolve,reject) => {
+			let obj = {
+				associatedDevice : associatedDeviceId,
+				updatedAt : moment().utc().format('YYYY-MM-DD HH:mm:ss')
+			}
+
+			let filter = {
+				id : deviceId
+			};
+			
+	    $.db.update("devices",obj,filter)
+	    .then (rows => {
+	      return resolve(rows[0]);
+	    })
+	    .catch(error => {
+	      return reject(error);
+	    });
+		    
+		});
+	}
+
 }

package/src/db/fota.js

@@ -7,7 +7,7 @@ var self = module.exports = {
 
 	  return new Promise((resolve,reject) => {
 
-	    let query = `SELECT COUNT(*) FROM ?? where 
+	    let query = `SELECT * FROM ?? where 
 	    	device_id = ? and
 	    	target_version = ? and
 	    	target_app_version = ? and
@@ -91,13 +91,13 @@ var self = module.exports = {
 				obj['createdAt'] = moment().utc().format('YYYY-MM-DD HH:mm:ss');
 				obj['device_id'] = deviceId;
 
-		    $.db.insert("fota",obj)
-		    .then (rows => {
-		      return resolve(rows[0]);
-		    })
-		    .catch(error => {
-		      return reject(error);
-		    });
+			    $.db.insert("fota",obj)
+			    .then (rows => {
+			      return resolve(rows[0]);
+			    })
+			    .catch(error => {
+			      return reject(error);
+			    });
 
 			}else{
 
@@ -106,18 +106,18 @@ var self = module.exports = {
 				};
 
 				$.db.update("fota",obj,filter)
-		    .then (rows => {
-		      return resolve(rows[0]);
-		    })
-		    .catch(error => {
-		      return reject(error);
-		    });
+			    .then (rows => {
+			      return resolve(rows[0]);
+			    })
+			    .catch(error => {
+			      return reject(error);
+			    });
 			}
 
 		});
 	},
 
-	getDeviceLastLog : async(deviceId)=>{
+	getFotaLastLog : async(deviceId)=>{
 
 		return new Promise((resolve,reject) => {
 
@@ -127,7 +127,7 @@ var self = module.exports = {
 	    	LIMIT 1`;
 
 	    let table = [
-	    	"fota",
+	    	"logs_fota",
 	    	deviceId,
     	];
 	    query = mysql.format(query,table);
@@ -195,7 +195,7 @@ var self = module.exports = {
 
 		return new Promise(async (resolve,reject) => {
 
-			const log = await getDeviceLastLog(deviceId);
+			const log = await self.getFotaLastLog(deviceId);
 
 			if(log?.createdAt != log?.updatedAt)
 

package/src/device/device.js

@@ -1,6 +1,6 @@
 const moment = require('moment');
 
-var project = [];
+var _project = [];
 const logs_table = "sensor_logs";
 const semver = require('semver');
 
@@ -14,11 +14,11 @@ var self = module.exports = {
 
         projects.map( async (name,counter)=>{
           console.log("project:",name)
-          project[name] = {
+          _project[name] = {
             module : null
           };
-          project[name].module = require(`${BASE_DIR}/projects/${name}/${name}.js`)
-          project[name].module?.init();
+          _project[name].module = require(`${BASE_DIR}/projects/${name}/${name}.js`)
+          _project[name].module?.init();
           let row = await $.db_project.getByName(name);
           if(row == null)
             $.db_project.insert(name,name,"logs_"+name);
@@ -78,33 +78,42 @@ var self = module.exports = {
       }
 
       // update project id if needed
-      if(project_id != null && project_id != device?.project_id) 
+      if(project_id != null && project_id != device?.project_id){
         $.db_device.update(device.id,"project_id",project_id);
+        $.db_device.addLog(device.id,"project_id",project_id);
+      } 
 
       switch (topic) {
         case "status":
           if (payload === "online" || payload === "offline") {
             await $.db_device.update(device.id, "status", payload);
+            $.db_device.addLog(device.id,"status",payload);
           }
           break;
         case "model":
           let res = await $.db_model.getByName(payload);
           let model_id = res?.id;
-          if (model_id != null) {
+          if (model_id != null && device?.tech != payload) {
             await $.db_device.update(device.id, "model_id", model_id);
+            $.db_device.addLog(device.id,"model_id",model_id);
           }
           break;
         case "tech":
-          await $.db_device.update(device.id, "tech", payload);
+          if (device?.tech && payload != device.tech) {
+            await $.db_device.update(device.id, "tech", payload);
+            $.db_device.addLog(device.id,"tech",payload);
+          }
           break;
         case "version":
           if (device?.version && payload != device.version) {
+            $.db_device.addLog(device.id,"version",payload);
             await $.db_device.update(device.id, "version", payload);
             await self.handleFotaSuccess(device.id);
           }
           break;
         case "app_version":
           if (device?.app_version && payload != device.app_version) {
+            $.db_device.addLog(device.id,"app_version",payload);
             await $.db_device.update(device.id, "app_version", payload);
             await self.handleFotaSuccess(device.id);
           }
@@ -118,8 +127,38 @@ var self = module.exports = {
           break;
       }
 
-      if(topic.startsWith("sensor/")){
-
+      if(topic.startsWith("settings/") && topic.endsWith("/set") && payload != "" ){
+        // update local settings
+        let index = topic.indexOf("settings/");
+        topic = topic.substring(index+9);
+        self.updateLocalSettings(device,topic,payload);
+      }else if(topic.startsWith("settings/") && !topic.endsWith("/set") && payload != "" ){
+        // store remote device settings
+        let index = topic.indexOf("settings/");
+        topic = topic.substring(index+9);
+        self.updateRemoteSettings(device,topic,payload);
+      }else if(topic == "fw"){
+        try{
+          payload = JSON.parse(payload);
+        }catch(error){}
+        if(typeof payload === 'object' && payload !== null){
+          try{
+            $.db_data.updateJson("fw",device.id,payload);
+            $.db_data.addJsonLog("logs_fw",device.id,payload);
+          }catch(error){
+            console.error(error)
+          } 
+        }else if(typeof payload !== 'object' && payload !== null){
+          // change it to topic.startsWith("fw")
+          try{
+            const column = getWordAfterLastSlash(topic);
+            $.db_data.update("fw",device.id,column,payload);
+            $.db_data.addLog("logs_fw",device.id,column,payload);
+          }catch(error){
+            console.error(error)
+          } 
+        }
+      }else if(topic.startsWith("sensor/")){
         updateSensor(device,topic,paylaod)
         index = topic.indexOf("/");
         if(index == -1)
@@ -132,13 +171,8 @@ var self = module.exports = {
         }
       }
 
-      // store remote device settings on project table - twin model
-      if(topic.endsWith("/set") && payload != "" ){
-        self.updateSettings(project_name,device,topic,payload);
-      }
-
-      if(project[project_name]){
-        project[project_name].module.parseMessage(client,project_name,device,topic,payload,retain,()=>{});
+      if(_project[project_name]){
+        _project[project_name]?.module?.parseMessage(client,project_name,device,topic,payload,retain,()=>{});
       }
     }
 
@@ -162,49 +196,115 @@ var self = module.exports = {
     }
   },
 
-  updateSettings : async (project_name,device,topic,payload)=>{
+  updateLocalSettings: async (device, topic, payload) => {
+
+    $.db_device.addLog(device.id,"local_settings",JSON.stringify(payload));
+
     let route = topic.split("/");
 
-    if(route == null){
-      console.log("topic invalid:",topic);
+    if (route == null || route.length == 0) {
+      console.warn("updateLocalSettings: topic invalid:", topic);
       return;
     }
-    // get settings
-    let settings = await $.db_project.getSettings(project_name,device.id);
-    let obj = settings;
 
-    if(obj == null){
-      obj = {};
+    // Retrieve existing settings
+    let settings = await $.db_device.getLocalSettings(device.id);
+
+    if (!settings || typeof settings !== 'object') {
+      settings = {};
     }
-    // Traverse through all route parts except the last one
+
+    let obj = settings;
+
+    // Traverse route parts to reach the target object
     route.slice(0, -1).forEach(property => {
-      if (!obj.hasOwnProperty(property) || 
-        ( obj.hasOwnProperty(property) && typeof obj[property] !== 'object')) 
-      {
+      if (
+        !Object.prototype.hasOwnProperty.call(obj, property) ||
+        (obj.hasOwnProperty(property) && typeof obj[property] !== 'object') ||
+        obj[property] === null
+      ) {
+        obj[property] = {}; // Create nested object if missing or not an object
+      }
+      obj = obj[property];
+    });
+
+    // Parse the payload JSON
+    let data = {};
+    try {
+      data = JSON.parse(payload);
+    } catch (error) {
+      console.error("Failed to parse payload JSON:", error);
+      return;
+    }
+
+    // Check if the data is a plain object, then merge
+    if (data && typeof data === 'object' && !Array.isArray(data)) {
+      Object.assign(obj, data);
+    } else {
+      console.warn("Payload is not a valid object:", payload);
+      return;
+    }
+
+    try {
+      // Update the settings in the database
+      await $.db_device.updateLocalSettings(JSON.stringify(settings), device.id);
+    } catch (err) {
+      console.error("Failed to update local settings:", err);
+    }
+  },
+
+  updateRemoteSettings : async (device,topic,payload)=>{
+
+    $.db_device.addLog(device.id,"remote_settings",JSON.stringify(payload));
+
+    let route = topic.split("/");
+
+    if(route == null){
+      console.warn("updateRemoteSettings: topic invalid:",topic);
+      return;
+    }
+    // Parse existing settings
+    let settings = await $.db_device.getRemoteSettings(device.id);
+
+    if (!settings || typeof settings !== 'object') {
+      settings = {};
+    }
+
+    let obj = settings;
+
+    // Traverse route parts
+    route.slice(0, route.length).forEach(property => {
+      if (
+        !Object.prototype.hasOwnProperty.call(obj, property) ||
+        (obj.hasOwnProperty(property) && typeof obj[property] !== 'object') ||
+        obj[property] === null
+      ) {
         obj[property] = {}; // create nested object if missing or not an object
       }
-      obj = obj[property]; // go deeper
-    })
-    
+      obj = obj[property];
+    });
+
+    // Merge payload into nested object
     let data = {};
-    try{
-      data = JSON.parse(payload)
-    }catch(error){}
-    
-    if ((data && typeof payload === 'object' && !Array.isArray(payload) ) ) {
-      // Assuming payload is an object with properties to process
+    try {
+      data = JSON.parse(payload);
+    } catch(error) {
+    }
+
+    if (data && typeof data === 'object' && !Array.isArray(data)) {
+      // Merge data properties
       for (const [key, value] of Object.entries(data)) {
-        obj[key] = value; // go deeper
-      };
-    }else{
-      obj = payload
+        obj[key] = value;
+      }
+    } else {
+      // payload isn't an object, replace the nested object
+      obj = payload;
     }
 
-    // update on device fw settings
-    try{
-      await $.db_project.updateSettings(project_name,JSON.stringify(settings),device.id);
-    }catch(error){
-      console.log(error);
+    try {
+      await $.db_device.updateRemoteSettings(JSON.stringify(settings), device.id);
+    } catch (err) {
+      console.error("Failed to update local settings:", err);
     }
   },
 
@@ -214,12 +314,15 @@ var self = module.exports = {
 
     for (const model of models) {
       
+      console.log("model:",model?.name);
       if(!model?.id)
         continue;
 
       try{
         const latestVersion = await $.db_firmware.getLatestVersion(model.id,release);
         const latestAppVersion = await $.db_firmware.getLatestAppVersion(model.id,release);
+        console.log("latestVersion:",latestVersion?.version);
+        console.log("latestAppVersion:",latestAppVersion?.app_version);
 
         const devices = await $.db_device.listByModel(model.id);
 
@@ -256,23 +359,31 @@ var self = module.exports = {
             };
           }
           if(obj != null){
-            try{
-              let res = await $.db_fota.getEntry(device.id,obj);
-              if(res == null)
-                $.db_fota.update(device.id,obj);
-            }catch(error){
-              console.log(error)
+            try {
+
+              let res = await $.db_fota.getEntry(device.id, obj);
+
+              if (res == null) {
+                try {
+                  console.log("add fota entry for device:", device.uid);
+                  await $.db_fota.update(device.id, obj);
+                } catch (err) {
+                  console.error("Error updating FOTA entry:", err);
+                }
+              }
+              
+            } catch (error) {
+              console.error("Error getting or processing FOTA entry:", error);
             }
           }
         }
       }catch(error){
-        console.log(error);
+        console.error(error);
         continue;
       }
     }
   },
 
-  
   triggerFota : async (release = "dev")=>{
 
     const devices = await $.db_fota.getUpdatable(release);
@@ -290,21 +401,27 @@ var self = module.exports = {
           const model_name = model?.name;
           let topic = "";
           if(model_name == "sniffer"){
+            resolve();
+            // !! This cannot be handled on this way
+
             // get sniffer info associated to device
-            const sniffer = await $.db_data.getGwAssociatedToDevice("sniffer",device.uid);
+            const sniffer = await $.db_data.getAssociatedToDevice("sniffer",device.id);
             if(sniffer == null) return;
             // get device_uid from devices table
             const gw = await $.db_device.getById(sniffer?.id)
             if(gw == null) return;
             let mqtt_prefix = `${project_name}/${gw.uid}`;
-            topic = mqtt_prefix+"/app/sniffer/fota/update/set";
+            if (semver.gt(sniffer.version, "2.0.1"))
+              topic = mqtt_prefix+`/app/sniffer/${sniffer.uid}/fota/update/set`;
+            else
+              topic = mqtt_prefix+`/app/sniffer/fota/update/set`;
           }
           else{
             let mqtt_prefix = `${project_name}/${device.uid}`;
             topic = mqtt_prefix+"/fw/fota/update/set";
           }
           let link = `${$.config.web.protocol}${$.config.web.domain}${$.config.web.fw_path}${firmware?.filename}/download?token=${firmware?.token}`;
-          console.log(`Requesting firmware update of ${device.uid} to ${firmware?.filename}`);
+          console.log(`Requesting firmware update for ${device.uid} to ${firmware?.filename}`);
           $.mqtt_client.publish(topic,`{"url":"${link}"}`,{qos:1,retain:false});
 
           let obj = {
@@ -343,14 +460,14 @@ var self = module.exports = {
     object = {
       success : 1,
     }
-    await $.db_fota.updateLog(deviceId,obj);
+    await $.db_fota.updateLog(deviceId,object);
   },
 
   handleFotaError : async (deviceId, error) => {
     let object = {
       error : error,
     }
-    await $.db_fota.updateLog(deviceId,obj);
+    await $.db_fota.updateLog(deviceId,object);
   }
 }