mqtt-devices-parser 1.0.16 → 1.0.18

package/Changelog.md

@@ -1,5 +1,29 @@
 # Changelog
 
+## 1.0.18
+	Fix and improve Sensors, checks uid length
+	Adds seedCommonLwM2MObjects and seedCommonLwM2MResources functions
+	fixes sync project tables. Makes load .models more robust
+	Adds templates table
+	add template_id to devices table
+
+## 1.0.17
+	Fix and improve Sensors, checks uid length
+	config/index: When defining a project from here uidPrefix and uidLength must be defined
+	models/logs_sensor.models: removes client_id association
+	models/sensors.models: removes unique and references properties from model_id, adds device_id
+	src/db/db: adds method updateOrInsert
+	src/db/device: adds method getSensorByRef
+	src/db/model: adds method getSensorByRef
+	src/db/project: adds uidPrefix and uidLength properties
+	src/db/sensor: insert: adds fields error and obj
+	src/device/device:
+	 - fixes logs_table
+	 - adds projectOptions
+	 - checks if device.uidLength <= project.uidLength
+	 - looks for sensor ref by device_id first and then by model_id if not found. Insert object, value and error fields
+	 - removes some fota logs
+
 ## 1.0.16
 	changes db tables:
 	 - models/devices.models: removes endpoint, adds protocol and psk.

package/config/index.js

@@ -26,5 +26,9 @@ module.exports = {
   sync_main_tables : process.env.sync_main_tables || true,
   projects: {
     demo : process.env.demo || false,
+  },
+  demo : {
+    uidPrefix: 'uid:',
+    uidLength: 16 // max len for uid
   }
 }

package/index.js

@@ -94,7 +94,7 @@ var self = module.exports = {
 
     //main tables
     if($.config.sync_main_tables){
-
+      
       await $.models.load(__dirname+"/models");
       await $.models.dropTableIndexes();
       await $.models.sync();
@@ -106,7 +106,7 @@ var self = module.exports = {
       await $.models.insertUser("client","client_pwd",3);
       // adds client with credentials admin@admin
       await $.models.insertClient("admin","admin",user_id); // dashboard login
-
+      
       for(name of projects){
         project = $.config[name];
         if (project) {
@@ -119,9 +119,6 @@ var self = module.exports = {
 
     // project tables
     await self.readModelsInsideProjects(projectsPath);
-    await $.models.dropTableIndexes();
-    await $.models.sync();
-
   }
 }
 

package/models/devices.models.js

@@ -20,6 +20,10 @@ module.exports = (sequelize,DataTypes)=>{
 				key: 'id'
 			}
 		},
+		template_id: {
+			type: DataTypes.INTEGER,
+			allowNull: true
+		},
 		model_id: {
 			type: DataTypes.INTEGER,
 			references: {

package/models/logs_sensor.models.js

@@ -22,16 +22,14 @@ module.exports = (sequelize,DataTypes)=>{
 			type: DataTypes.STRING,
 			allowNull: true,
 		},
-		cliend_id: {
-			type: DataTypes.INTEGER,
-			unique: false,
+		error: {
+			type: DataTypes.STRING,
+			allowNull: true,
+		},
+		obj: {
+			type: DataTypes.STRING,
 			allowNull: true,
-			references: {
-				model: 'clients',
-				key: 'id'
-			}
 		},
-
 	},
 	{
 		tableName: 'logs_sensor',

package/models/models.js

@@ -39,22 +39,28 @@ var self = module.exports = {
 
   load: async (modelsPath) => {
     return new Promise(async (resolve, reject) => {
-      var files = await fs.readdir(modelsPath);
-      files = files.filter((file) => {
-        // Exclude models.js and non-js files
-        return (
-          file.indexOf('.') !== 0 &&
-          file !== 'models.js' &&
-          file.slice(-9) === 'models.js'
-        );
-      });
-      files.forEach((file, counter) => {
-        const model = require(path.join(modelsPath, file));
-        model(sequelize, DataTypes);
-        if (counter === files.length - 1) {
-          return resolve();
+      try {
+        let files = await fs.readdir(modelsPath);
+        files = files.filter((file) => {
+          // Exclude models.js and non-js files
+          return (
+            file.indexOf('.') !== 0 &&
+            file !== 'models.js' &&
+            file.slice(-9) === 'models.js'
+          );
+        });
+
+        // Use a for...of loop to load models sequentially
+        for (const file of files) {
+          console.log(`Sync file ${file}`);
+          const model = require(path.join(modelsPath, file));
+          await model(sequelize, DataTypes); // Ensure each model is loaded before moving to the next
         }
-      });
+
+        resolve(); // Resolve the Promise after all files are processed
+      } catch (error) {
+        reject(error); // Reject the Promise on any error
+      }
     });
   },
 
@@ -246,5 +252,153 @@ var self = module.exports = {
         });
     });
   },
-  
+
+  seedCommonLwM2MObjects: async(objects) => {
+    try {
+      const results = await Promise.all(
+        objects.map(async (object) => {
+          const [record, created] = await sequelize.models['lwm2mObjects'].upsert(
+            {
+              objectId: object.objectId,
+              name: object.name,
+              description: object.description,
+            },
+            {
+              where: { objectId: object.objectId }, // Ensure the record exists based on objectId
+            }
+          );
+          return { record, created };
+        })
+      );
+      console.log("Insert/Update completed for LwM2M Objects.");
+      return results;
+    } catch (error) {
+      console.error("Error inserting/updating LwM2M Objects:", error.message);
+      throw error;
+    }
+  },
+  /*
+  seedCommonLwM2MResources: async(resources) => {
+    try {
+      const results = await Promise.all(
+        resources.map(async (resource) => {
+          const [record, created] = await sequelize.models['lwm2mResources'].upsert(
+            {
+              objectId: resource.objectId,
+              resourceId: resource.resourceId,
+              name: resource.name,
+              description: resource.description,
+            },
+            {
+              where: { objectId: object.objectId, resourceId: object.resourceId }, // Ensure the record exists based on objectId and resourceId
+            }
+          );
+          return { record, created };
+        })
+      );
+      console.log("Insert/Update completed for LwM2M Resources.");
+      return results;
+    } catch (error) {
+      console.error("Error inserting/updating LwM2M Resources:", error.message);
+      throw error;
+    }
+  },
+  */
+
+  /**
+ * Seed LwM2M resources into the database.
+ *
+ * Expected DB model: sequelize.models['lwm2mResources']
+ * with a composite unique key on (objectId, resourceId) recommended.
+ *
+ * @param {Array} resources - Either:
+ *   - Flat: [{ objectId, resourceId, name, description }, ...]
+ *   - Nested: [{ objectId, name, resources: [{ resourceId, name, description }, ...] }, ...]
+ * @param {import('sequelize').Sequelize} sequelize - An initialized Sequelize instance.
+ * @returns {Promise<Array<{ objectId:number, resourceId:number, created:boolean, updated:boolean }>>}
+ */
+  seedCommonLwM2MResources: async (resources) => {
+    if (!sequelize || !sequelize.models || !sequelize.models['lwm2mResources']) {
+      throw new Error("sequelize instance with model 'lwm2mResources' is required");
+    }
+    if (!Array.isArray(resources)) {
+      throw new Error('resources must be an array');
+    }
+
+    // Normalize input: flatten nested object.resources into a flat {objectId,resourceId,...} list
+    const flatResources = [];
+    for (const entry of resources) {
+      if (entry && Array.isArray(entry.resources)) {
+        // Nested structure: object -> resources[]
+        for (const res of entry.resources) {
+          flatResources.push({
+            objectId: entry.objectId,
+            resourceId: res.resourceId,
+            name: res.name,
+            description: res.description
+          });
+        }
+      } else if (
+        entry &&
+        typeof entry.objectId !== 'undefined' &&
+        typeof entry.resourceId !== 'undefined'
+      ) {
+        // Already flat
+        flatResources.push({
+          objectId: entry.objectId,
+          resourceId: entry.resourceId,
+          name: entry.name,
+          description: entry.description
+        });
+      } else {
+        // Skip invalid entries but warn
+        // eslint-disable-next-line no-console
+        console.warn('Skipping invalid resource entry:', entry);
+      }
+    }
+
+    const { lwm2mResources } = sequelize.models;
+
+    const results = [];
+    try {
+      await sequelize.transaction(async (t) => {
+        for (const res of flatResources) {
+          const where = { objectId: res.objectId, resourceId: res.resourceId };
+
+          // Try to find existing record
+          const [record, created] = await lwm2mResources.findOrCreate({
+            where,
+            defaults: {
+              name: res.name,
+              description: res.description
+            },
+            transaction: t
+          });
+
+          if (!created) {
+            // Update existing with latest values
+            await record.update(
+              { name: res.name, description: res.description },
+              { transaction: t }
+            );
+          }
+
+          results.push({
+            objectId: res.objectId,
+            resourceId: res.resourceId,
+            created,
+            updated: !created
+          });
+        }
+      });
+
+      // eslint-disable-next-line no-console
+      console.log('Insert/Update completed for LwM2M Resources.');
+      return results;
+    } catch (error) {
+      // eslint-disable-next-line no-console
+      console.error('Error inserting/updating LwM2M Resources:', error.message);
+      throw error;
+    }
+  }
 };

package/models/sensors.models.js

@@ -1,14 +1,13 @@
 
 module.exports = (sequelize,DataTypes)=>{
 	return sequelize.define("sensors", {
-		model_id: {
+		model_id: { // reference a model
 			type: DataTypes.INTEGER,
-			unique: false,
-			allowNull: false,
-			references: {
-				model: 'models',
-				key: 'id'
-			}
+			allowNull: true,
+		},
+		device_id: { // reference a device id
+			type: DataTypes.INTEGER,
+			allowNull: true,
 		},
 		active: {
 			type: DataTypes.BOOLEAN,

package/models/templates.models.js

@@ -0,0 +1,25 @@
+
+module.exports = (sequelize,DataTypes)=>{
+	return sequelize.define("templates", {
+		tag: {
+			type: DataTypes.STRING,
+			allowNull: false
+		},
+		name: {
+			type: DataTypes.STRING,
+			allowNull: false
+		},
+		client_id: {
+			type: DataTypes.INTEGER,
+			allowNull: false
+		},
+		project_id: {
+			type: DataTypes.INTEGER,
+			allowNull: false
+		},
+	},
+	{
+		tableName: 'templates',
+		freezeTableName: true
+	})
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mqtt-devices-parser",
-  "version": "1.0.16",
+  "version": "1.0.18",
   "description": "",
   "main": "index.js",
   "scripts": {

package/src/db/db.js

@@ -144,6 +144,67 @@ var self = module.exports = {
     });
   },
 
+  updateOrInsert: async (table, data, filter) => {
+    return new Promise((resolve, reject) => {
+      self.getConnection((err, conn) => {
+        if (err) return reject(err);
+
+        let updateQuery = "";
+        let updateValues = [];
+
+        if (typeof data === "object") {
+          const keys = Object.keys(data);
+          updateValues = Object.values(data);
+          updateQuery = `UPDATE ?? SET ${keys.map(key => `${key} = ?`).join(', ')}`;
+        } else {
+          self.close_db_connection(conn);
+          return reject("data passed is not an object");
+        }
+
+        if (typeof filter === "object") {
+          const filterKeys = Object.keys(filter);
+          const filterValues = Object.values(filter);
+          updateQuery += ` WHERE ${filterKeys.map(key => `${key} = ?`).join(' AND ')}`;
+          updateValues.push(...filterValues);
+        } else {
+          self.close_db_connection(conn);
+          return reject("filter passed is not an object");
+        }
+
+        updateValues.unshift(table);
+        updateQuery = mysql.format(updateQuery, updateValues);
+
+        conn.query(updateQuery, function (err, rows) {
+          if (err) {
+            self.close_db_connection(conn);
+            return reject(err);
+          }
+          // If affectedRows = 0, do an insert
+          if (rows.affectedRows === 0) {
+            // Combine data and filter for insert
+            const insertObj = { ...filter, ...data }; // filter first, data overrides on conflict
+            const insertKeys = Object.keys(insertObj);
+            const insertValues = Object.values(insertObj);
+
+            const insertQuery = mysql.format(
+              `INSERT INTO ?? (${insertKeys.map(() => '??').join(', ')}) VALUES (${insertKeys.map(() => '?').join(', ')})`,
+              [table, ...insertKeys, ...insertValues]
+            );
+
+            conn.query(insertQuery, function (insertErr, insertRows) {
+              self.close_db_connection(conn);
+              if (insertErr) return reject(insertErr);
+              else return resolve(insertRows);
+            });
+          } else {
+            self.close_db_connection(conn);
+            return resolve(rows);
+          }
+        });
+      });
+    });
+  },
+
   // update column with json data
   updateJSON : async (table, column, data, filter) => {
 

package/src/db/device.js

@@ -387,6 +387,27 @@ var self = module.exports = {
 	    });
 		    
 		});
+	},
+
+	getSensorByRef : async(deviceId, ref)=>{
+		return new Promise((resolve,reject) => {
+
+	    let query = "SELECT * FROM ?? where device_id = ? and ref = ?";
+	    let args = ["sensors",deviceId,ref];
+	    query = mysql.format(query,args);
+
+	    $.db.queryRow(query)
+	    .then( rows => {
+	      if(rows.length > 0)
+	        return resolve(rows[0]);
+	      else
+	        return resolve(null);
+	    })
+	    .catch( err => {
+	      console.log(err);
+	      return resolve(null);
+	    });
+	  });
 	}
 
 }

package/src/db/model.js

@@ -112,4 +112,25 @@ var self = module.exports = {
 		});
   	},
 
+  	getSensorByRef : async(modelId, ref)=>{
+		return new Promise((resolve,reject) => {
+
+	    let query = "SELECT * FROM ?? where model_id = ? and ref = ?";
+	    let args = ["sensors",modelId,ref];
+	    query = mysql.format(query,args);
+
+	    $.db.queryRow(query)
+	    .then( rows => {
+	      if(rows.length > 0)
+	        return resolve(rows[0]);
+	      else
+	        return resolve(null);
+	    })
+	    .catch( err => {
+	      console.log(err);
+	      return resolve(null);
+	    });
+	  });
+	}
+
 }

package/src/db/project.js

@@ -64,13 +64,15 @@ var self = module.exports = {
 	  });
 	},
 
-	insert : async(project, project_table, logs_table)=>{
+	insert : async(project, project_table, logs_table, projectOptions)=>{
 
 		return new Promise((resolve,reject) => {
 			let obj = {
 				name : project,
 				project_table : project_table,
 				logs_table : logs_table,
+				uidPrefix : projectOptions.uidPrefix,
+				uidLength : projectOptions.uidLength,
 				createdAt : moment().utc().format('YYYY-MM-DD HH:mm:ss'),
 				updatedAt : moment().utc().format('YYYY-MM-DD HH:mm:ss')
 			}

package/src/db/sensor.js

@@ -95,7 +95,9 @@ var self = module.exports = {
 			let obj = {
 				device_id : deviceId,
 				sensor_id : sensorId,
-				value : payload,
+				value : payload?.value,
+				error : payload?.error,
+				obj : payload?.object,
 				createdAt : moment().utc().format('YYYY-MM-DD HH:mm:ss'),
 				updatedAt : moment().utc().format('YYYY-MM-DD HH:mm:ss')
 			}

package/src/device/device.js

@@ -1,13 +1,12 @@
 const moment = require('moment');
 
 var _project = [];
-const logs_table = "sensor_logs";
+const logs_table = "logs_sensor";
 const semver = require('semver');
 
 var self = module.exports = {
 
   init : async (config,projects)=>{
-
     return new Promise(async (resolve,reject) => {
       $.db.connect(config,()=>{
         console.log("MYSQL is connected");
@@ -19,10 +18,12 @@ var self = module.exports = {
           };
           _project[name].module = require(`${BASE_DIR}/projects/${name}/${name}.js`)
           _project[name].module?.init();
+          let projectOptions = config[name];
           let row = await $.db_project.getByName(name);
+          
           if(row == null)
-            $.db_project.insert(name,name,"logs_"+name);
-
+            $.db_project.insert(name,name,"logs_"+name,projectOptions);
+          
           if(counter == projects.length-1)
             return resolve();
         })
@@ -60,7 +61,7 @@ var self = module.exports = {
     topic = topic.substring(index+1);
 
     // check if topic corresponds to a device
-    if(uid.startsWith(project.uidPrefix) && uid.length == project?.uidLength){
+    if(uid.startsWith(project.uidPrefix) && uid.length <= project?.uidLength){
 
       let device = await $.db_device.get(uid);
 
@@ -159,15 +160,40 @@ var self = module.exports = {
           } 
         }
       }else if(topic.startsWith("sensor/")){
-        updateSensor(device,topic,paylaod)
+        //updateSensor(device,topic,payload)
         index = topic.indexOf("/");
-        if(index == -1)
-            return;
+        if(index != -1){
+          const ref = topic.substring(index+1);
+          let res = await $.db_device.getSensorByRef(device.id,ref)
+          if(res == null)
+            res = await $.db_model.getSensorByRef(device.model_id,ref)
+          if(res != null){
+            object = null;
+            value = null;
+            error = null;
+            try{
+              object = JSON.parse(payload);
+            }catch(err){
+              console.log(err);
+            }
+            if (typeof object === 'object' && object !== null) {
+              value = object?.value;
+              error = object?.error;
+            }else{
+              value = payload;
+            }
+
+            if(value || error)
+              object = null;
 
-        const name = topic.substring(index+1);
-        let res = await $.db_sensor.getByRef(topic)
-        if(res != null){
-          await $.db_sensor.insert(logs_table,device.id,res.id,payload);
+            const data = {
+              object,
+              value,
+              error
+            }
+
+            await $.db_sensor.insert(logs_table,device.id,res.id,data);
+          }
         }
       }
 
@@ -314,15 +340,15 @@ var self = module.exports = {
 
     for (const model of models) {
       
-      console.log("model:",model?.name);
+      //console.log("model:",model?.name);
       if(!model?.id)
         continue;
 
       try{
         const latestVersion = await $.db_firmware.getLatestVersion(model.id,release);
         const latestAppVersion = await $.db_firmware.getLatestAppVersion(model.id,release);
-        console.log("latestVersion:",latestVersion?.version);
-        console.log("latestAppVersion:",latestAppVersion?.app_version);
+        //console.log("latestVersion:",latestVersion?.version);
+        //console.log("latestAppVersion:",latestAppVersion?.app_version);
 
         const devices = await $.db_device.listByModel(model.id);