mqtt-devices-parser 1.0.10 → 1.0.11

package/Changelog.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## 1.0.11
+	index: MQTT
+		- makes mqtt client global,
+		- adds periodically call for FOTA update
+	models/devices.models: adds version fields to devices table
+	models/firmwares.models: adds new fields for firmware management
+	src/db/device: optimizes calls to manage devices table
+	src/device/device: Fota changes, db calls updated to match previous changes
+	Adds method to check FOTA updates. Doing 10 at time at each minute for now. Non configurable
+
 ## 1.0.10
 	src/db/data: handles json data
 	- Adds updateJson call

package/index.js

@@ -14,6 +14,7 @@ $.db_device = require('./src/db/device');
 $.db_model = require('./src/db/model');
 $.db_firmware = require('./src/db/firmware');
 $.db_sensor = require('./src/db/sensor');
+$.mqtt_client = null;
 
 const packageJson = require(__dirname+'/package.json');
 const packageVersion = packageJson.version;
@@ -44,7 +45,6 @@ var self = module.exports = {
       setInterval(async ()=>{
         await $.device.deleteLogs();
       },60*60*1000);
-      
 
       return resolve();
     })
@@ -117,7 +117,8 @@ var self = module.exports = {
 function mqtt_connect(){
 
   const mqtt_prefix = $.config.mqtt.logs_path+"/"+$.config.mqtt.client;
-  const client = mqtt.connect({
+  let checkFota = null;
+  $.mqtt_client = mqtt.connect({
     host : $.config.mqtt.host,
     port : $.config.mqtt.port,
     username : $.config.mqtt.user,
@@ -132,50 +133,58 @@ function mqtt_connect(){
     }
   });
 
-  client.on("connect", () => {
+  $.mqtt_client.on("connect", () => {
 
-    client.publish(mqtt_prefix+"/status","online",{qos:2,retain:true});
-    client.publish(mqtt_prefix+"/version",packageVersion,{qos:2,retain:true});
+    $.mqtt_client.publish(mqtt_prefix+"/status","online",{qos:2,retain:true});
+    $.mqtt_client.publish(mqtt_prefix+"/version",packageVersion,{qos:2,retain:true});
 
     projects.map( (project)=>{
-      client.subscribe(project+"/#", (err) => {});
+      $.mqtt_client.subscribe(project+"/#", (err) => {});
 
       for(let project in $.config.projects){
         if($.config.projects[project])
-          client.publish(mqtt_prefix+"/"+project,"active",{qos:2,retain:true});
+          $.mqtt_client.publish(mqtt_prefix+"/"+project,"active",{qos:2,retain:true});
         else
-          client.publish(mqtt_prefix+"/"+project,"deactive",{qos:2,retain:true});
+          $.mqtt_client.publish(mqtt_prefix+"/"+project,"deactive",{qos:2,retain:true});
       };
     })
     console.log(`MQTT connected to: ${$.config.mqtt.host}:${$.config.mqtt.port}`);
     projects.map( project=>{
       console.log("subscribing project:",project);
-      client.subscribe(project+"/#")
+      $.mqtt_client.subscribe(project+"/#")
     })
+
+    checkFota = setInterval(async ()=>{
+      await $.device.checkFota();
+    },60*1000);
+
   });
 
-  client.on("message", (topic, payload, packet) => {
+  $.mqtt_client.on("message", (topic, payload, packet) => {
     // payload is Buffer
-    $.device.parseMessage(client,topic.toString(),payload.toString(),packet.retain);
+    $.device.parseMessage($.mqtt_client,topic.toString(),payload.toString(),packet.retain);
   });
 
-  client.on("reconnect",()=>{
+  $.mqtt_client.on("reconnect",()=>{
     console.log("reconnect")
   });
 
-  client.on("close",()=>{
+  $.mqtt_client.on("close",()=>{
     console.log("close")
+    checkFota = null;
   });
 
-  client.on("offline",()=>{
+  $.mqtt_client.on("offline",()=>{
     console.log("offline")
+    checkFota = null;
   });
 
-  client.on("disconnect",(packet)=>{
+  $.mqtt_client.on("disconnect",(packet)=>{
     console.log(packet)
+    checkFota = null;
   })
 
-  client.on("error",(error)=>{
+  $.mqtt_client.on("error",(error)=>{
     console.log(error)
   })
 }

package/models/devices.models.js

@@ -9,10 +9,6 @@ module.exports = (sequelize,DataTypes)=>{
 			type: DataTypes.STRING,
 			allowNull: true
 		},
-		project: { // deprecated
-			type: DataTypes.STRING,
-			allowNull: true
-		},
 		status: {
 			type: DataTypes.STRING,
 			allowNull: true
@@ -31,6 +27,22 @@ module.exports = (sequelize,DataTypes)=>{
 				key: 'id'
 			}
 		},
+		version: {
+			type: DataTypes.STRING,
+			allowNull: true
+		},
+		app_version: {
+			type: DataTypes.STRING,
+			allowNull: true
+		},
+		release: {
+			type: DataTypes.STRING,
+			allowNull: true
+		},
+		fota_tries: {
+			type: DataTypes.STRING,
+			allowNull: true
+		},
 		tech: {
 			type: DataTypes.STRING,
 			allowNull: true

package/models/firmwares.models.js

@@ -10,7 +10,11 @@ module.exports = (sequelize,DataTypes)=>{
 			allowNull: true,
 			unique: true
 		},
-		fw_version: {
+		fw_version: { // deprecated
+			type: DataTypes.STRING,
+			allowNull: true
+		},
+		version: {
 			type: DataTypes.STRING,
 			allowNull: true
 		},
@@ -18,7 +22,11 @@ module.exports = (sequelize,DataTypes)=>{
 			type: DataTypes.STRING,
 			allowNull: true
 		},
-		fw_release: {
+		fw_release: { // deprecated
+			type: DataTypes.STRING,
+			allowNull: true
+		},
+		release: {
 			type: DataTypes.STRING,
 			allowNull: true
 		},

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mqtt-devices-parser",
-  "version": "1.0.10",
+  "version": "1.0.11",
   "description": "",
   "main": "index.js",
   "scripts": {

package/src/db/device.js

@@ -7,20 +7,62 @@ var self = module.exports = {
 		return new Promise((resolve,reject) => {
 
 			let query = "SELECT id,uid,project_id,model_id,status FROM ?? where uid = ?";
-		    let table = ["devices",uid];
-		    query = mysql.format(query,table);
-
-		    $.db.queryRow(query)
-		    .then( rows => {
-		      if(rows.length > 0)
-		        return resolve(rows[0]);
-		      else
-		        return resolve(null);
-		    })
-		    .catch( err => {
-		      console.log(err);
-		      return resolve(null);
-		    });
+			let table = ["devices",uid];
+			query = mysql.format(query,table);
+
+			$.db.queryRow(query)
+			.then( rows => {
+				if(rows.length > 0)
+					return resolve(rows[0]);
+				else
+					return resolve(null);
+			})
+			.catch( err => {
+				console.log(err);
+				return resolve(null);
+			});
+		});
+	},
+
+	listOnline : async ()=>{
+		return new Promise((resolve,reject) => {
+
+			let query = "SELECT * FROM ?? where status = ?";
+			let table = ["devices","online"];
+			query = mysql.format(query,table);
+
+			$.db.queryRow(query)
+			.then( rows => {
+				if(rows.length > 0)
+					return resolve(rows);
+				else
+					return resolve(null);
+			})
+			.catch( err => {
+				console.log(err);
+				return resolve(null);
+			});
+		});
+	},
+
+	getField : async (id,column)=>{
+		return new Promise((resolve,reject) => {
+
+			let query = "SELECT ?? FROM devices where id = ?";
+			let table = [column,id];
+			query = mysql.format(query,table);
+
+			$.db.queryRow(query)
+			.then( rows => {
+				if(rows.length > 0)
+					return resolve(rows[0]);
+				else
+					return resolve(null);
+			})
+			.catch( err => {
+				console.log(err);
+				return resolve(null);
+			});
 		});
 	},
 
@@ -33,101 +75,37 @@ var self = module.exports = {
 				updatedAt : moment().utc().format('YYYY-MM-DD HH:mm:ss')
 			}
 
-		    $.db.insert("devices",obj)
-		    .then (rows => {
-		      return resolve(rows[0]);
-		    })
-		    .catch(error => {
-		      return reject(error);
-		    });
+			$.db.insert("devices",obj)
+			.then (rows => {
+				return resolve(rows[0]);
+			})
+			.catch(error => {
+				return reject(error);
+			});
 		});
-  	},
+	},
 
-  	updateStatus : async(uid,status)=>{
+	update : async(id,column,value)=>{
 		return new Promise((resolve,reject) => {
 
-		    let obj = {
-		      status : status,
-		      updatedAt : moment().utc().format('YYYY-MM-DD HH:mm:ss')
-		    };
-
-		    let filter = {
-		      uid : uid,
-		    };
-
-		    $.db.update("devices",obj,filter)
-		    .then (rows => {
-		      return resolve(rows);
-		    })
-		    .catch(error => {
-		      return reject(error);
-			});
-	  });
-  	},
+			let obj = {
+				updatedAt : moment().utc().format('YYYY-MM-DD HH:mm:ss')
+			};
 
-  	updateTech : async(uid,tech)=>{
-		return new Promise((resolve,reject) => {
+			obj[column] = value;
 
-		    let obj = {
-		      tech : tech,
-		      updatedAt : moment().utc().format('YYYY-MM-DD HH:mm:ss')
-		    };
-
-		    let filter = {
-		      uid : uid,
-		    };
-
-		    $.db.update("devices",obj,filter)
-		    .then (rows => {
-		      return resolve(rows);
-		    })
-		    .catch(error => {
-		      return reject(error);
-			});
-	  });
-  	},
+			let filter = {
+				id : id,
+			};
 
-  	updateProject : async(uid,project_id)=>{
-		return new Promise((resolve,reject) => {
-
-		    let obj = {
-		      project_id : project_id,
-		      updatedAt : moment().utc().format('YYYY-MM-DD HH:mm:ss')
-		    };
-
-		    let filter = {
-		      uid : uid,
-		    };
-
-		    $.db.update("devices",obj,filter)
-		    .then (rows => {
-		      return resolve(rows);
-		    })
-		    .catch(error => {
-		      return reject(error);
+			$.db.update("devices",obj,filter)
+			.then (rows => {
+				return resolve(rows);
+			})
+			.catch(error => {
+				return reject(error);
 			});
-	  });
-  	},
-
-  	updateModel : async(uid,model_id)=>{
-		return new Promise((resolve,reject) => {
+		});
+	},
 
-		    let obj = {
-		      model_id : model_id,
-		      updatedAt : moment().utc().format('YYYY-MM-DD HH:mm:ss')
-		    };
-
-		    let filter = {
-		      uid : uid,
-		    };
-
-		    $.db.update("devices",obj,filter)
-		    .then (rows => {
-		      return resolve(rows);
-		    })
-		    .catch(error => {
-		      return reject(error);
-			});
-	  });
-  	},
 }

package/src/device/device.js

@@ -51,6 +51,64 @@ var self = module.exports = {
 
   },
 
+  checkFota : async ()=>{
+
+    const devices = await $.db_device.listOnline();
+
+    const batchSize = 10;
+    for (let i = 0; i < devices.length; i += batchSize) {
+      const batch = devices.slice(i, i + batchSize);
+      await Promise.all(batch.map(async (device) => {
+
+        if (device.release == null || device.release === "critical") 
+        {
+          // don't update
+          return;
+        }
+
+        if (device.fota_tries > 3) 
+        {
+          return;
+        }
+
+        let new_firmware = await $.db_firmware.getLatestFWVersion(device.model_id,device.release);
+        let new_app = await $.db_firmware.getLatestAppVersion(device.model_id,device.release);
+        if(new_firmware == null || new_app == null)
+          return;
+
+        let project_name = "";
+        let res = await $.db_project.getById(device.project_id);
+        if(res != null)
+          project_name = res.name;
+        else
+          return;
+
+        let mqtt_prefix = `${project_name}/${device.uid}`;
+        if(new_app != null && semver.lt(device.app_version, new_app.app_version)) 
+        {
+            console.log(`updating firmware of ${device.uid} to minor version ${new_app.app_version}`);
+            let link = `${$.config.web.protocol}${$.config.web.domain}${$.config.web.fw_path}${new_app.filename}/download?token=${new_app.token}`;
+            client.publish(mqtt_prefix+"/fw/fota/update/set",`{"url":"${link}"}`,{qos:2,retain:false});
+            await $.db_data.update(project_name,device.id,"fota_tries",++device.fota_tries);
+        }else{
+          if(new_firmware != null && semver.lt(device.version, new_firmware.fw_version)) 
+          {
+            console.log(`updating firmware of ${device.uid} to major version ${new_firmware.fw_version}`);
+            let link = `${$.config.web.protocol}${$.config.web.domain}${$.config.web.fw_path}${new_firmware.filename}/download?token=${new_firmware.token}`;
+            $.mqtt_client.publish(mqtt_prefix+"/fw/fota/update/set",`{"url":"${link}"}`,{qos:2,retain:false});
+            await $.db_data.update(project_name,device.id,"fota_tries",++device.fota_tries);
+          }
+        }
+      }));
+
+      // Wait for 1 minute after processing each batch
+      if (i + batchSize < devices.length) {
+        await new Promise(resolve => setTimeout(resolve, 60000));
+      }
+    }
+
+  },
+
   parseMessage : async (client,topic,payload,retain)=>{
 
     let topic_bck = topic;
@@ -87,65 +145,31 @@ var self = module.exports = {
       let res = await $.db_project.getByName(project_name);
       let project_id = res?.id;
       if(project_id != null && project_id != device?.project_id) 
-        $.db_device.updateProject(uid,project_id);
+        $.db_device.update(device.id,"project_id",project_id);
 
-      if(topic.endsWith("status")){
+      if(topic === "status"){
         if(payload == "online" || payload == "offline")
-          await $.db_device.updateStatus(uid,payload);
-        console.log(project_name,uid,payload)
-      }else if(topic.endsWith("model")){
+          await $.db_device.update(device.id,"status",payload);
+      }else if(topic === "model"){
         let res = await $.db_model.getByName(payload);
         let model_id = res?.id;
 
         if(model_id != null){
-          let res = await $.db_device.updateModel(uid,model_id);
+          let res = await $.db_device.update(device.id,"model_id",model_id);
         } 
-      }else if(topic.endsWith("tech")){
-        await $.db_device.updateTech(uid,payload);
-      }else if(topic.endsWith("fw_version")){
-        let fw_version = await $.db.getFieldFromDeviceId(project_name,device.id,"fw_version");
-        if(fw_version != payload)
-          await $.db_data.update(project_name,device.id,"fota_tries",1); // ! 0 is not working..
-      }else if(topic.endsWith("app_version")){
-        let app_version = await $.db.getFieldFromDeviceId(project_name,device.id,"app_version");
-        if(app_version != payload)
-          await $.db_data.update(project_name,device.id,"fota_tries",1); // ! 0 is not working..
-      }else if(topic.endsWith("uptime")){ // !! maybe this topic should be change for something ending in fota
-        // check fw version
-        // get current fw and app version
-        let release = await $.db.getFieldFromDeviceId(project_name,device.id,"fw_release");
-        if(release == null || release == "critical"){
-          // don't update
-          return;
+      }else if(topic === "tech"){
+        await $.db_device.update(device.id,"tech",payload);
+      }else if(topic === "version"){
+        // Update version and app_version on device table..
+        if(payload != device.version){
+          await $.db_device.update(device.id,"version",payload);
+          await $.db_device.update(device.id,"fota_tries",1); // ! value 0 doesn't work..
         }
-        let fota_tries = await $.db.getFieldFromDeviceId(project_name,device.id,"fota_tries");
-        if(fota_tries > 3)
-          return;
-
-        let fw_version = await $.db.getFieldFromDeviceId(project_name,device.id,"fw_version");
-        let app_version = await $.db.getFieldFromDeviceId(project_name,device.id,"app_version");
-        let new_firmware = await $.db_firmware.getLatestFWVersion(device.model_id,release);
-        let new_app = await $.db_firmware.getLatestAppVersion(device.model_id,release);
-        if(new_firmware == null || new_app == null)
-          return;
-
-        let mqtt_prefix = `${project_name}/${uid}`;
-
-        if(new_app != null && semver.lt(app_version, new_app.app_version)) {
-          console.log(new_app)
-          console.log(`updating firmware of ${uid} to minor version ${new_app.app_version}`);
-          let link = `${$.config.web.protocol}${$.config.web.domain}${$.config.web.fw_path}${new_app.filename}/download?token=${new_app.token}`;
-          client.publish(mqtt_prefix+"/fw/fota/update/set",`{"url":"${link}"}`,{qos:2,retain:false});
-          await $.db_data.update(project_name,device.id,"fota_tries",++fota_tries);
-        }else{
-          if(new_firmware != null && semver.lt(fw_version, new_firmware.fw_version)) {
-            console.log(`updating firmware of ${uid} to major version ${new_firmware.fw_version}`);
-            let link = `${$.config.web.protocol}${$.config.web.domain}${$.config.web.fw_path}${new_firmware.filename}/download?token=${new_firmware.token}`;
-            client.publish(mqtt_prefix+"/fw/fota/update/set",`{"url":"${link}"}`,{qos:2,retain:false});
-            await $.db_data.update(project_name,device.id,"fota_tries",++fota_tries);
-          }
+      }else if(topic === "app_version"){
+        if(payload != device.app_version){
+          await $.db_device.update(device.id,"app_version",payload);
+          await $.db_device.update(device.id,"fota_tries",1); // ! value 0 doesn't work..
         }
-        
       }
 
       if(device.project_id != null && device.model_id != null){