mqtt-devices-parser 1.0.9 → 1.0.11

package/Changelog.md

@@ -1,5 +1,22 @@
 # Changelog
 
+## 1.0.11
+	index: MQTT
+		- makes mqtt client global,
+		- adds periodically call for FOTA update
+	models/devices.models: adds version fields to devices table
+	models/firmwares.models: adds new fields for firmware management
+	src/db/device: optimizes calls to manage devices table
+	src/device/device: Fota changes, db calls updated to match previous changes
+	Adds method to check FOTA updates. Doing 10 at time at each minute for now. Non configurable
+
+## 1.0.10
+	src/db/data: handles json data
+	- Adds updateJson call
+	- Adds addJsonLog
+	src/db/project: deals with settings
+	src/device/device: fixes fota fw link, stores settings on project table (twin model)
+	
 ## 1.0.9
 	- Adds method to delete logs older than 1 week
 

package/index.js

@@ -14,6 +14,7 @@ $.db_device = require('./src/db/device');
 $.db_model = require('./src/db/model');
 $.db_firmware = require('./src/db/firmware');
 $.db_sensor = require('./src/db/sensor');
+$.mqtt_client = null;
 
 const packageJson = require(__dirname+'/package.json');
 const packageVersion = packageJson.version;
@@ -44,7 +45,6 @@ var self = module.exports = {
       setInterval(async ()=>{
         await $.device.deleteLogs();
       },60*60*1000);
-      
 
       return resolve();
     })
@@ -117,7 +117,8 @@ var self = module.exports = {
 function mqtt_connect(){
 
   const mqtt_prefix = $.config.mqtt.logs_path+"/"+$.config.mqtt.client;
-  const client = mqtt.connect({
+  let checkFota = null;
+  $.mqtt_client = mqtt.connect({
     host : $.config.mqtt.host,
     port : $.config.mqtt.port,
     username : $.config.mqtt.user,
@@ -132,50 +133,58 @@ function mqtt_connect(){
     }
   });
 
-  client.on("connect", () => {
+  $.mqtt_client.on("connect", () => {
 
-    client.publish(mqtt_prefix+"/status","online",{qos:2,retain:true});
-    client.publish(mqtt_prefix+"/version",packageVersion,{qos:2,retain:true});
+    $.mqtt_client.publish(mqtt_prefix+"/status","online",{qos:2,retain:true});
+    $.mqtt_client.publish(mqtt_prefix+"/version",packageVersion,{qos:2,retain:true});
 
     projects.map( (project)=>{
-      client.subscribe(project+"/#", (err) => {});
+      $.mqtt_client.subscribe(project+"/#", (err) => {});
 
       for(let project in $.config.projects){
         if($.config.projects[project])
-          client.publish(mqtt_prefix+"/"+project,"active",{qos:2,retain:true});
+          $.mqtt_client.publish(mqtt_prefix+"/"+project,"active",{qos:2,retain:true});
         else
-          client.publish(mqtt_prefix+"/"+project,"deactive",{qos:2,retain:true});
+          $.mqtt_client.publish(mqtt_prefix+"/"+project,"deactive",{qos:2,retain:true});
       };
     })
     console.log(`MQTT connected to: ${$.config.mqtt.host}:${$.config.mqtt.port}`);
     projects.map( project=>{
       console.log("subscribing project:",project);
-      client.subscribe(project+"/#")
+      $.mqtt_client.subscribe(project+"/#")
     })
+
+    checkFota = setInterval(async ()=>{
+      await $.device.checkFota();
+    },60*1000);
+
   });
 
-  client.on("message", (topic, payload, packet) => {
+  $.mqtt_client.on("message", (topic, payload, packet) => {
     // payload is Buffer
-    $.device.parseMessage(client,topic.toString(),payload.toString(),packet.retain);
+    $.device.parseMessage($.mqtt_client,topic.toString(),payload.toString(),packet.retain);
   });
 
-  client.on("reconnect",()=>{
+  $.mqtt_client.on("reconnect",()=>{
     console.log("reconnect")
   });
 
-  client.on("close",()=>{
+  $.mqtt_client.on("close",()=>{
     console.log("close")
+    checkFota = null;
   });
 
-  client.on("offline",()=>{
+  $.mqtt_client.on("offline",()=>{
     console.log("offline")
+    checkFota = null;
   });
 
-  client.on("disconnect",(packet)=>{
+  $.mqtt_client.on("disconnect",(packet)=>{
     console.log(packet)
+    checkFota = null;
   })
 
-  client.on("error",(error)=>{
+  $.mqtt_client.on("error",(error)=>{
     console.log(error)
   })
 }

package/models/devices.models.js

@@ -9,10 +9,6 @@ module.exports = (sequelize,DataTypes)=>{
 			type: DataTypes.STRING,
 			allowNull: true
 		},
-		project: { // deprecated
-			type: DataTypes.STRING,
-			allowNull: true
-		},
 		status: {
 			type: DataTypes.STRING,
 			allowNull: true
@@ -31,6 +27,22 @@ module.exports = (sequelize,DataTypes)=>{
 				key: 'id'
 			}
 		},
+		version: {
+			type: DataTypes.STRING,
+			allowNull: true
+		},
+		app_version: {
+			type: DataTypes.STRING,
+			allowNull: true
+		},
+		release: {
+			type: DataTypes.STRING,
+			allowNull: true
+		},
+		fota_tries: {
+			type: DataTypes.STRING,
+			allowNull: true
+		},
 		tech: {
 			type: DataTypes.STRING,
 			allowNull: true

package/models/firmwares.models.js

@@ -10,7 +10,11 @@ module.exports = (sequelize,DataTypes)=>{
 			allowNull: true,
 			unique: true
 		},
-		fw_version: {
+		fw_version: { // deprecated
+			type: DataTypes.STRING,
+			allowNull: true
+		},
+		version: {
 			type: DataTypes.STRING,
 			allowNull: true
 		},
@@ -18,7 +22,11 @@ module.exports = (sequelize,DataTypes)=>{
 			type: DataTypes.STRING,
 			allowNull: true
 		},
-		fw_release: {
+		fw_release: { // deprecated
+			type: DataTypes.STRING,
+			allowNull: true
+		},
+		release: {
 			type: DataTypes.STRING,
 			allowNull: true
 		},

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mqtt-devices-parser",
-  "version": "1.0.9",
+  "version": "1.0.11",
   "description": "",
   "main": "index.js",
   "scripts": {

package/src/db/data.js

@@ -5,24 +5,24 @@ var self = module.exports = {
 
 	checkEntry : async (table,deviceId)=>{
 
-	  return new Promise((resolve,reject) => {
-
-	    let query = "SELECT * FROM ?? where device_id = ?";
-	    let args = [table,deviceId];
-	    query = mysql.format(query,args);
-
-	    $.db.queryRow(query)
-	    .then( rows => {
-	      if(rows.length > 0)
-	        return resolve(rows[0]);
-	      else
-	        return resolve(null);
-	    })
-	    .catch( err => {
-	      console.log(err);
-	      return resolve(null);
-	    });
-	  });
+		return new Promise((resolve,reject) => {
+
+		let query = "SELECT * FROM ?? where device_id = ?";
+		let args = [table,deviceId];
+		query = mysql.format(query,args);
+
+		$.db.queryRow(query)
+		.then( rows => {
+			if(rows.length > 0)
+				return resolve(rows[0]);
+			else
+				return resolve(null);
+		})
+		.catch( err => {
+			console.log(err);
+			return resolve(null);
+			});
+		});
 	},
 
 	update : async (table, deviceId, topic, payload)=>{
@@ -54,41 +54,41 @@ var self = module.exports = {
 				topic = topic.substring(index+1);
 
 				if(db_columns.hasOwnProperty(key)){ // check if db has a column for this key
-				  // create JSON struct with the remaining topic
-				  // table is the table
-				  // key is the column
-
-				  // build object with the remaining topic and respective data
-				  if(typeof data === "object")
-				    obj[key] = JSON.stringify(parser.pathIntoObject(topic,JSON.stringify(data)))
-				  else
-				    obj[key] = JSON.stringify(parser.pathIntoObject(topic,data));
-
-				  let exists = await self.checkEntry(table,deviceId);
-				  if(!exists){
-				    obj['device_id'] = deviceId;
-				    obj['createdAt'] = moment().utc().format('YYYY-MM-DD HH:mm:ss');
-				    $.db.insert(table,obj)
-				    .then((rows)=>{
-				    	return resolve(rows);
-				    })
-				    .catch((err)=>{
-				    	console.log(err);
-				    	return resolve();
-				    })
-				  }else{
-				    let filter = {
-				      device_id : deviceId
-				    }
-				    $.db.update(table,obj,filter)
-				    .then((rows)=>{
-				    	return resolve(rows);
-				    })
-				    .catch((err)=>{
-				    	console.log(err);
-				    	return resolve();
-				    })
-				  }
+					// create JSON struct with the remaining topic
+					// table is the table
+					// key is the column
+
+					// build object with the remaining topic and respective data
+					if(typeof data === "object")
+						obj[key] = JSON.stringify(parser.pathIntoObject(topic,JSON.stringify(data)))
+					else
+						obj[key] = JSON.stringify(parser.pathIntoObject(topic,data));
+
+					let exists = await self.checkEntry(table,deviceId);
+					if(!exists){
+						obj['device_id'] = deviceId;
+						obj['createdAt'] = moment().utc().format('YYYY-MM-DD HH:mm:ss');
+						$.db.insert(table,obj)
+						.then((rows)=>{
+							return resolve(rows);
+						})
+						.catch((err)=>{
+							console.log(err);
+							return resolve();
+						})
+					}else{
+						let filter = {
+							device_id : deviceId
+						}
+						$.db.update(table,obj,filter)
+						.then((rows)=>{
+							return resolve(rows);
+						})
+						.catch((err)=>{
+							console.log(err);
+							return resolve();
+						})
+					}
 
 				}else return resolve();
 			}else{
@@ -115,7 +115,7 @@ var self = module.exports = {
 						})
 					}else{
 						let filter = {
-						  device_id : deviceId
+							device_id : deviceId
 						}
 
 						$.db.update(table,obj,filter)
@@ -132,67 +132,184 @@ var self = module.exports = {
 		})
 	},
 
+	updateJson : async (table, deviceId, payload)=>{
+
+		return new Promise( async (resolve, reject) => {
+		
+			if (!payload || typeof payload !== 'object') 
+				return resolve();
+
+			let obj = {
+				updatedAt: moment().utc().format('YYYY-MM-DD HH:mm:ss'),
+				device_id: deviceId,
+			};
+
+			// Get columns info
+			const db_columns = $.models.get(table);
+			if (db_columns == null) 
+				return resolve();
+
+			// Prepare data for insertion
+			for (let key in payload) {
+				if (db_columns.hasOwnProperty(key)) {
+					let value = payload[key];
+
+					// Convert object to JSON string if needed
+					if (typeof value === 'object') {
+						value = JSON.stringify(value);
+					}
+
+					obj[key] = value;
+				}
+			}
+
+			let exists = await self.checkEntry(table,deviceId);
+			if(!exists){
+				obj['createdAt'] = moment().utc().format('YYYY-MM-DD HH:mm:ss');
+				$.db.insert(table,obj)
+				.then((rows)=>{
+					return resolve(rows);
+				})
+				.catch((err)=>{
+					console.log(err);
+					return resolve();
+				})
+			}else{
+				let filter = {
+					device_id : deviceId
+				}
+				$.db.update(table,obj,filter)
+				.then((rows)=>{
+					return resolve(rows);
+				})
+				.catch((err)=>{
+					console.log(err);
+					return resolve();
+				})
+			}
+
+			return resolve();
+		});
+	},
+
 	addLog : async (table, deviceId, topic, payload)=>{
 
 	  return new Promise((resolve,reject) => {
 
 	    if(!payload) return resolve();
 
-		let data = null;
-		// check if data is in JSON format
-		try{
-			data = JSON.parse(payload);
-		}catch(e){
-			data = payload;
-		}
+			let data = null;
+			// check if data is in JSON format
+			try{
+				data = JSON.parse(payload);
+			}catch(e){
+				data = payload;
+			}
 
-		let obj = {
-			updatedAt : moment().utc().format('YYYY-MM-DD HH:mm:ss')
-		}
+			let obj = {
+				updatedAt : moment().utc().format('YYYY-MM-DD HH:mm:ss')
+			}
 
-		let db_columns = $.models.get(table);
-		if(db_columns == null)
-			return resolve();
+			let db_columns = $.models.get(table);
+			if(db_columns == null)
+				return resolve();
 
-		let index = topic.indexOf("/");
+			let index = topic.indexOf("/");
 
-		if(index > -1){ // check if topic has subtopics
-			let key = topic.substring(0,index);
-			topic = topic.substring(index+1);
+			if(index > -1){ // check if topic has subtopics
+				let key = topic.substring(0,index);
+				topic = topic.substring(index+1);
 
-			if(db_columns.hasOwnProperty(key)){ // check if db has a column for this key
-			  // create JSON struct with the remaining topic
-			  // key is the column
+				if(db_columns.hasOwnProperty(key)){ // check if db has a column for this key
+					// create JSON struct with the remaining topic
+					// key is the column
 
-			  // build object with the remaining topic and respective data
-			  if(typeof data === "object")
-			    obj[key] = JSON.stringify(parser.pathIntoObject(topic,JSON.stringify(data)))
-			  else
-			    obj[key] = JSON.stringify(parser.pathIntoObject(topic,data));
+					// build object with the remaining topic and respective data
+					if(typeof data === "object")
+						obj[key] = JSON.stringify(parser.pathIntoObject(topic,JSON.stringify(data)))
+					else
+						obj[key] = JSON.stringify(parser.pathIntoObject(topic,data));
+
+					obj['device_id'] = deviceId;
+					obj['createdAt'] = moment().utc().format('YYYY-MM-DD HH:mm:ss');
+					$.db.insert(table,obj)
+					.then((rows)=>{
+						return resolve(rows);
+					})
+					.catch((err)=>{
+						console.log(err);
+						return resolve();
+					})
+
+				}else
+					return resolve();
+			}else{
 
-			  obj['device_id'] = deviceId;
-			  obj['createdAt'] = moment().utc().format('YYYY-MM-DD HH:mm:ss');
-			  $.db.insert(table,obj)
+				let key = topic;
+				if(db_columns.hasOwnProperty(key)){ // check if db has a column for this key
+					// build object with respective data
+					if(typeof data === "object")
+						obj[key] = JSON.stringify(data);
+					else
+						obj[key] = data;
 
+					obj['device_id'] = deviceId;
+					obj['createdAt'] = moment().utc().format('YYYY-MM-DD HH:mm:ss');
+					$.db.insert(table,obj)
+					.then((rows)=>{
+						return resolve(rows);
+					})
+					.catch((err)=>{
+						console.log(err);
+						return resolve();
+					})
+				}else
+					return resolve();
 			}
-		}else{
 
-			let key = topic;
-			if(db_columns.hasOwnProperty(key)){ // check if db has a column for this key
-			  // build object with respective data
-			  if(typeof data === "object")
-			    obj[key] = JSON.stringify(data);
-			  else
-			    obj[key] = data;
+	  });
+	},
 
-			  obj['device_id'] = deviceId;
-			  obj['createdAt'] = moment().utc().format('YYYY-MM-DD HH:mm:ss');
-			  $.db.insert(table,obj);
+	addJsonLog: async (table, deviceId, dataObject) => {
+		return new Promise((resolve, reject) => {
+			if (!dataObject || typeof dataObject !== 'object') 
+				return resolve();
+
+			let obj = {
+				updatedAt: moment().utc().format('YYYY-MM-DD HH:mm:ss'),
+				device_id: deviceId,
+				createdAt: moment().utc().format('YYYY-MM-DD HH:mm:ss')
+			};
+
+			// Get columns info
+			const db_columns = $.models.get(table);
+			if (db_columns == null) 
+				return resolve();
+
+			// Prepare data for insertion
+			for (let key in dataObject) {
+				if (db_columns.hasOwnProperty(key)) {
+					let value = dataObject[key];
+
+					// Convert object to JSON string if needed
+					if (typeof value === 'object') {
+						value = JSON.stringify(value);
+					}
+
+					obj[key] = value;
+				}
 			}
-		}
 
-	    return resolve();
-	  });
+			//console.log("updateJSONlog")
+			$.db.insert(table, obj)
+			.then((rows)=>{
+				return resolve(rows);
+			})
+			.catch((err)=>{
+				console.log(err);
+				return resolve();
+			})
+		});
 	},
 
 	deleteLogs : async (table, deviceId, topic, payload)=>{
@@ -223,7 +340,13 @@ var self = module.exports = {
 			  obj['device_id'] = deviceId;
 			  obj['createdAt'] = moment().utc().format('YYYY-MM-DD HH:mm:ss');
 			  $.db.insert(table,obj)
-
+			  .then((rows)=>{
+				return resolve(rows);
+			  })
+			.catch((err)=>{
+				console.log(err);
+				return resolve();
+			})
 			}
 		}else{
 
@@ -237,7 +360,7 @@ var self = module.exports = {
 
 			  obj['device_id'] = deviceId;
 			  obj['createdAt'] = moment().utc().format('YYYY-MM-DD HH:mm:ss');
-			  $.db.insert(table,obj);
+			  $.db.insert(table,obj)
 			}
 		}
 

package/src/db/device.js

@@ -7,20 +7,62 @@ var self = module.exports = {
 		return new Promise((resolve,reject) => {
 
 			let query = "SELECT id,uid,project_id,model_id,status FROM ?? where uid = ?";
-		    let table = ["devices",uid];
-		    query = mysql.format(query,table);
-
-		    $.db.queryRow(query)
-		    .then( rows => {
-		      if(rows.length > 0)
-		        return resolve(rows[0]);
-		      else
-		        return resolve(null);
-		    })
-		    .catch( err => {
-		      console.log(err);
-		      return resolve(null);
-		    });
+			let table = ["devices",uid];
+			query = mysql.format(query,table);
+
+			$.db.queryRow(query)
+			.then( rows => {
+				if(rows.length > 0)
+					return resolve(rows[0]);
+				else
+					return resolve(null);
+			})
+			.catch( err => {
+				console.log(err);
+				return resolve(null);
+			});
+		});
+	},
+
+	listOnline : async ()=>{
+		return new Promise((resolve,reject) => {
+
+			let query = "SELECT * FROM ?? where status = ?";
+			let table = ["devices","online"];
+			query = mysql.format(query,table);
+
+			$.db.queryRow(query)
+			.then( rows => {
+				if(rows.length > 0)
+					return resolve(rows);
+				else
+					return resolve(null);
+			})
+			.catch( err => {
+				console.log(err);
+				return resolve(null);
+			});
+		});
+	},
+
+	getField : async (id,column)=>{
+		return new Promise((resolve,reject) => {
+
+			let query = "SELECT ?? FROM devices where id = ?";
+			let table = [column,id];
+			query = mysql.format(query,table);
+
+			$.db.queryRow(query)
+			.then( rows => {
+				if(rows.length > 0)
+					return resolve(rows[0]);
+				else
+					return resolve(null);
+			})
+			.catch( err => {
+				console.log(err);
+				return resolve(null);
+			});
 		});
 	},
 
@@ -33,101 +75,37 @@ var self = module.exports = {
 				updatedAt : moment().utc().format('YYYY-MM-DD HH:mm:ss')
 			}
 
-		    $.db.insert("devices",obj)
-		    .then (rows => {
-		      return resolve(rows[0]);
-		    })
-		    .catch(error => {
-		      return reject(error);
-		    });
+			$.db.insert("devices",obj)
+			.then (rows => {
+				return resolve(rows[0]);
+			})
+			.catch(error => {
+				return reject(error);
+			});
 		});
-  	},
+	},
 
-  	updateStatus : async(uid,status)=>{
+	update : async(id,column,value)=>{
 		return new Promise((resolve,reject) => {
 
-		    let obj = {
-		      status : status,
-		      updatedAt : moment().utc().format('YYYY-MM-DD HH:mm:ss')
-		    };
-
-		    let filter = {
-		      uid : uid,
-		    };
-
-		    $.db.update("devices",obj,filter)
-		    .then (rows => {
-		      return resolve(rows);
-		    })
-		    .catch(error => {
-		      return reject(error);
-			});
-	  });
-  	},
+			let obj = {
+				updatedAt : moment().utc().format('YYYY-MM-DD HH:mm:ss')
+			};
 
-  	updateTech : async(uid,tech)=>{
-		return new Promise((resolve,reject) => {
+			obj[column] = value;
 
-		    let obj = {
-		      tech : tech,
-		      updatedAt : moment().utc().format('YYYY-MM-DD HH:mm:ss')
-		    };
-
-		    let filter = {
-		      uid : uid,
-		    };
-
-		    $.db.update("devices",obj,filter)
-		    .then (rows => {
-		      return resolve(rows);
-		    })
-		    .catch(error => {
-		      return reject(error);
-			});
-	  });
-  	},
+			let filter = {
+				id : id,
+			};
 
-  	updateProject : async(uid,project_id)=>{
-		return new Promise((resolve,reject) => {
-
-		    let obj = {
-		      project_id : project_id,
-		      updatedAt : moment().utc().format('YYYY-MM-DD HH:mm:ss')
-		    };
-
-		    let filter = {
-		      uid : uid,
-		    };
-
-		    $.db.update("devices",obj,filter)
-		    .then (rows => {
-		      return resolve(rows);
-		    })
-		    .catch(error => {
-		      return reject(error);
+			$.db.update("devices",obj,filter)
+			.then (rows => {
+				return resolve(rows);
+			})
+			.catch(error => {
+				return reject(error);
 			});
-	  });
-  	},
-
-  	updateModel : async(uid,model_id)=>{
-		return new Promise((resolve,reject) => {
+		});
+	},
 
-		    let obj = {
-		      model_id : model_id,
-		      updatedAt : moment().utc().format('YYYY-MM-DD HH:mm:ss')
-		    };
-
-		    let filter = {
-		      uid : uid,
-		    };
-
-		    $.db.update("devices",obj,filter)
-		    .then (rows => {
-		      return resolve(rows);
-		    })
-		    .catch(error => {
-		      return reject(error);
-			});
-	  });
-  	},
 }

package/src/db/project.js

@@ -20,8 +20,7 @@ var self = module.exports = {
 	        return resolve(null);
 	    })
 	    .catch( err => {
-	      console.log(err);
-	      return resolve(null);
+	      return reject(err);
 	    });
 	  });
 	},
@@ -42,8 +41,7 @@ var self = module.exports = {
 	        return resolve(null);
 	    })
 	    .catch( err => {
-	      console.log(err);
-	      return resolve(null);
+	      return reject(err);
 	    });
 	  });
 	},
@@ -61,8 +59,7 @@ var self = module.exports = {
 		  return resolve(rows);
 	    })
 	    .catch( err => {
-	      console.log(err);
-	      return resolve(null);
+	      return reject(null);
 	    });
 	  });
 	},
@@ -88,4 +85,49 @@ var self = module.exports = {
 		});
   	},
 
+  	getSettings : async(project,deviceId)=>{
+
+  		return new Promise((resolve,reject) => {
+
+		    let query = "SELECT settings FROM ?? where device_id = ?";
+		    let table = [project, deviceId];
+		    query = mysql.format(query,table);
+
+		    $.db.queryRow(query)
+		    .then( rows => {
+		    	if(rows?.length > 0)
+			  		return resolve(rows[0].settings);
+			  	else
+			  		return resolve(null);
+		    })
+		    .catch( err => {
+		      return reject(err);
+		    });
+		});
+  	},
+
+  	updateSettings : async(project, settings, deviceId)=>{
+
+		return new Promise((resolve,reject) => {
+			let obj = {
+				settings : settings,
+				updatedAt : moment().utc().format('YYYY-MM-DD HH:mm:ss')
+			}
+
+			let filter = {
+				device_id : deviceId
+			};
+
+			resolve(null)
+			
+		    $.db.update(project,obj,filter)
+		    .then (rows => {
+		      return resolve(rows[0]);
+		    })
+		    .catch(error => {
+		      return reject(error);
+		    });
+		    
+		});
+  	},
 }

package/src/device/device.js

@@ -51,6 +51,64 @@ var self = module.exports = {
 
   },
 
+  checkFota : async ()=>{
+
+    const devices = await $.db_device.listOnline();
+
+    const batchSize = 10;
+    for (let i = 0; i < devices.length; i += batchSize) {
+      const batch = devices.slice(i, i + batchSize);
+      await Promise.all(batch.map(async (device) => {
+
+        if (device.release == null || device.release === "critical") 
+        {
+          // don't update
+          return;
+        }
+
+        if (device.fota_tries > 3) 
+        {
+          return;
+        }
+
+        let new_firmware = await $.db_firmware.getLatestFWVersion(device.model_id,device.release);
+        let new_app = await $.db_firmware.getLatestAppVersion(device.model_id,device.release);
+        if(new_firmware == null || new_app == null)
+          return;
+
+        let project_name = "";
+        let res = await $.db_project.getById(device.project_id);
+        if(res != null)
+          project_name = res.name;
+        else
+          return;
+
+        let mqtt_prefix = `${project_name}/${device.uid}`;
+        if(new_app != null && semver.lt(device.app_version, new_app.app_version)) 
+        {
+            console.log(`updating firmware of ${device.uid} to minor version ${new_app.app_version}`);
+            let link = `${$.config.web.protocol}${$.config.web.domain}${$.config.web.fw_path}${new_app.filename}/download?token=${new_app.token}`;
+            client.publish(mqtt_prefix+"/fw/fota/update/set",`{"url":"${link}"}`,{qos:2,retain:false});
+            await $.db_data.update(project_name,device.id,"fota_tries",++device.fota_tries);
+        }else{
+          if(new_firmware != null && semver.lt(device.version, new_firmware.fw_version)) 
+          {
+            console.log(`updating firmware of ${device.uid} to major version ${new_firmware.fw_version}`);
+            let link = `${$.config.web.protocol}${$.config.web.domain}${$.config.web.fw_path}${new_firmware.filename}/download?token=${new_firmware.token}`;
+            $.mqtt_client.publish(mqtt_prefix+"/fw/fota/update/set",`{"url":"${link}"}`,{qos:2,retain:false});
+            await $.db_data.update(project_name,device.id,"fota_tries",++device.fota_tries);
+          }
+        }
+      }));
+
+      // Wait for 1 minute after processing each batch
+      if (i + batchSize < devices.length) {
+        await new Promise(resolve => setTimeout(resolve, 60000));
+      }
+    }
+
+  },
+
   parseMessage : async (client,topic,payload,retain)=>{
 
     let topic_bck = topic;
@@ -87,63 +145,31 @@ var self = module.exports = {
       let res = await $.db_project.getByName(project_name);
       let project_id = res?.id;
       if(project_id != null && project_id != device?.project_id) 
-        $.db_device.updateProject(uid,project_id);
+        $.db_device.update(device.id,"project_id",project_id);
 
-      if(topic.endsWith("status")){
+      if(topic === "status"){
         if(payload == "online" || payload == "offline")
-          await $.db_device.updateStatus(uid,payload);
-        console.log(project_name,uid,payload)
-      }else if(topic.endsWith("model")){
+          await $.db_device.update(device.id,"status",payload);
+      }else if(topic === "model"){
         let res = await $.db_model.getByName(payload);
         let model_id = res?.id;
 
         if(model_id != null){
-          let res = await $.db_device.updateModel(uid,model_id);
+          let res = await $.db_device.update(device.id,"model_id",model_id);
         } 
-      }else if(topic.endsWith("tech")){
-        await $.db_device.updateTech(uid,payload);
-      }else if(topic.endsWith("fw_version")){
-        let fw_version = await $.db.getFieldFromDeviceId(project_name,device.id,"fw_version");
-        if(fw_version != payload)
-          await $.db_data.update(project_name,device.id,"fota_tries",1); // ! 0 is not working..
-      }else if(topic.endsWith("app_version")){
-        let app_version = await $.db.getFieldFromDeviceId(project_name,device.id,"app_version");
-        if(app_version != payload)
-          await $.db_data.update(project_name,device.id,"fota_tries",1); // ! 0 is not working..
-      }else if(topic.endsWith("uptime")){ // !! maybe this topic should be change for something ending in fota
-        // check fw version
-        // get current fw and app version
-        let release = await $.db.getFieldFromDeviceId(project_name,device.id,"fw_release");
-        if(release == null || release == "critical"){
-          // don't update
-          return;
+      }else if(topic === "tech"){
+        await $.db_device.update(device.id,"tech",payload);
+      }else if(topic === "version"){
+        // Update version and app_version on device table..
+        if(payload != device.version){
+          await $.db_device.update(device.id,"version",payload);
+          await $.db_device.update(device.id,"fota_tries",1); // ! value 0 doesn't work..
         }
-        let fota_tries = await $.db.getFieldFromDeviceId(project_name,device.id,"fota_tries");
-        if(fota_tries > 3)
-          return;
-
-        let fw_version = await $.db.getFieldFromDeviceId(project_name,device.id,"fw_version");
-        let app_version = await $.db.getFieldFromDeviceId(project_name,device.id,"app_version");
-        let new_firmware = await $.db_firmware.getLatestFWVersion(device.model_id,release);
-        let new_app = await $.db_firmware.getLatestAppVersion(device.model_id,release);
-        if(new_firmware == null || new_app == null)
-          return;
-
-        let mqtt_prefix = `${project_name}/${uid}`;
-        let link = `${$.config.web.protocol}${$.config.web.domain}${$.config.web.fw_path}${new_firmware.filename}/download?token=${new_firmware.token}`;
-
-        if(new_firmware != null && semver.lt(fw_version, new_firmware.fw_version)) {
-          console.log(`updating firmware of ${uid} to ${new_firmware.fw_version}`);
-          client.publish(mqtt_prefix+"/fw/fota/update/set",`{"url":"${link}"}`,{qos:2,retain:false});
-          await $.db_data.update(project_name,device.id,"fota_tries",++fota_tries);
-        }else{
-          if(new_app != null && semver.lt(app_version, new_app.app_version)) {
-            console.log(`updating firmware of ${uid} to ${new_app.app_version}`);
-            client.publish(mqtt_prefix+"/fw/fota/update/set",`{"url":"${link}"}`,{qos:2,retain:false});
-            await $.db_data.update(project_name,device.id,"fota_tries",++fota_tries);
-          }
+      }else if(topic === "app_version"){
+        if(payload != device.app_version){
+          await $.db_device.update(device.id,"app_version",payload);
+          await $.db_device.update(device.id,"fota_tries",1); // ! value 0 doesn't work..
         }
-        
       }
 
       if(device.project_id != null && device.model_id != null){
@@ -153,6 +179,55 @@ var self = module.exports = {
         }
       }
 
+      // store remote device settings on project table - twin model
+      if(topic.endsWith("/set") && payload != "" ){
+
+        let route = topic.split("/");
+
+        if(route == null){
+          console.log("topic invalid:",topic);
+          return;
+        }
+        // get settings
+        let settings = await $.db_project.getSettings(project_name,device.id);
+        let obj = settings;
+
+        if(obj == null){
+          console.log("no settings available for deviceId:",device.id)
+          obj = {};
+        }
+        // Traverse through all route parts except the last one
+        route.slice(0, -1).forEach(property => {
+          if (!obj.hasOwnProperty(property) || 
+            ( obj.hasOwnProperty(property) && typeof obj[property] !== 'object')) 
+          {
+            obj[property] = {}; // create nested object if missing or not an object
+          }
+          obj = obj[property]; // go deeper
+        })
+        
+        let data = {};
+        try{
+          data = JSON.parse(payload)
+        }catch(error){}
+        
+        if (true || (data && typeof payload === 'object' && !Array.isArray(payload) ) ) {
+          // Assuming payload is an object with properties to process
+          for (const [key, value] of Object.entries(data)) {
+            obj[key] = value; // go deeper
+          };
+        }else{
+          obj = payload
+        }
+
+        // update on device fw settings
+        try{
+          await $.db_project.updateSettings(project_name,JSON.stringify(settings),device.id);
+        }catch(error){
+          console.log(error);
+        }
+      }
+
       if(project[project_name]){
         project[project_name].module.parseMessage(client,project_name,device,topic,payload,retain,()=>{});
       }