mqtt-devices-parser 1.0.24 → 1.0.25

package/Changelog.md

@@ -1,5 +1,35 @@
 # Changelog
 
+## 1.0.25
+
+	Sensors template (#9)
+
+		* new table sensorsTemplate
+
+		* store topics which match a sensor
+		for each mqtt msg received, check if there is a sensor associated to that topic.
+		If there is, update current value and insert new log
+
+		* db: models/sensors
+		changes localUnixTs field to remoteUnixTs
+		adds value, error
+		changes property: allow null
+		adds foreign keys to model_id and device_id
+
+		* handle change db column localUnixTs to remoteUnixTs
+
+		* fix getSensorsByRef
+		ref is not unique so it can have more than 1 row
+
+		* src/db/data: Implements missing function pathIntoObject
+
+		* src/aux/parser: adds functions to handle mqtt topics parsing
+		index: uses global $.parser as a pointer to class src/aux/parser
+		src/device/device: fixes null iterations, sets updateSensor and handleMqttTopic as public functions. Removes local mqtt parser functions. Adapts calls to those functions
+
+	mqtt: always check if topic is registered
+		store values in db
+		
 ## 1.0.24
 	src/device/device: fix log
 	src/kafka/consumer: increase initialRetryTime from 100 to 300ms

package/index.js

@@ -17,6 +17,7 @@ $.db_firmware = require('./src/db/firmware');
 $.db_sensor = require('./src/db/sensor');
 $.db_fota = require('./src/db/fota');
 $.mqtt_client = null;
+$.parser = require('./src/aux/parser')
 
 const packageJson = require(__dirname+'/package.json');
 const packageVersion = packageJson.version;

package/models/logs_sensor.models.js

@@ -30,7 +30,7 @@ module.exports = (sequelize,DataTypes)=>{
 			type: DataTypes.STRING,
 			allowNull: true,
 		},
-		localUnixTs: {
+		remoteUnixTs: {
 			type: DataTypes.BIGINT,
 			allowNull: true,
 		}

package/models/sensors.models.js

@@ -3,11 +3,19 @@ module.exports = (sequelize,DataTypes)=>{
 	return sequelize.define("sensors", {
 		model_id: { // reference a model
 			type: DataTypes.INTEGER,
-			allowNull: true,
+			allowNull: false,
+			references: {
+				model: 'models',
+				key: 'id'
+			}
 		},
 		device_id: { // reference a device id
 			type: DataTypes.INTEGER,
-			allowNull: true,
+			allowNull: false,
+			references: {
+				model: 'devices',
+				key: 'id'
+			}
 		},
 		active: {
 			type: DataTypes.BOOLEAN,
@@ -28,7 +36,19 @@ module.exports = (sequelize,DataTypes)=>{
 		},
 		property: {
 			type: DataTypes.STRING,
-			allowNull: false,
+			allowNull: true
+		},
+		value: {
+			type: DataTypes.STRING,
+			allowNull: true
+		},
+		error: {
+			type: DataTypes.STRING,
+			allowNull: true,
+		},
+		remoteUnixTs: {
+			type: DataTypes.BIGINT,
+			allowNull: true,
 		},
 		graph: {
 			type: DataTypes.JSON,

package/models/sensorsTemplate.models.js

@@ -0,0 +1,43 @@
+
+module.exports = (sequelize,DataTypes)=>{
+	return sequelize.define("sensorsTemplate", {
+		model_id: { // reference a model
+			type: DataTypes.INTEGER,
+			allowNull: false,
+			references: {
+				model: 'models',
+				key: 'id'
+			}
+		},
+		active: {
+			type: DataTypes.BOOLEAN,
+			allowNull: false,
+			defaultValue: true
+		},
+		ref: {
+			type: DataTypes.STRING,
+			allowNull: false,
+		},
+		name: {
+			type: DataTypes.STRING,
+			allowNull: false,
+		},
+		type: {
+			type: DataTypes.STRING,
+			allowNull: false,
+		},
+		property: {
+			type: DataTypes.STRING,
+			allowNull: true
+		},
+		graph: {
+			type: DataTypes.JSON,
+			allowNull: true,
+		}
+	},
+	{
+		tableName: 'sensorsTemplate',
+		freezeTableName: true
+	})
+}
+

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mqtt-devices-parser",
-  "version": "1.0.24",
+  "version": "1.0.25",
   "description": "",
   "main": "index.js",
   "scripts": {

package/src/aux/parser.js

@@ -16,5 +16,41 @@ module.exports = {
 			}
 		});
 		return obj;
+	},
+
+	getFirstWord : (str)=>{
+		const slashIndex = str.indexOf('/');
+		if (slashIndex === -1) {
+			// No slash found, return the original string
+			return str;
+		}
+		return str.substring(0,slashIndex);
+	},
+
+	getTopicAfterSlash : (str)=>{
+		const slashIndex = str.indexOf('/');
+		if (slashIndex === -1) {
+		// No slash found, return empty string
+		return "";
+		}
+		return str.substring(slashIndex + 1);
+	},
+
+	getWordAfterLastSlash : (str)=>{
+		const lastSlashIndex = str.lastIndexOf('/');
+		if (lastSlashIndex === -1) {
+			// No slash found, return the original string
+			return str;
+		}
+		return str.substring(lastSlashIndex + 1);
+	},
+
+	getWordBeforeLastSlash : (str)=>{
+		const lastSlashIndex = str.lastIndexOf('/');
+		if (lastSlashIndex === -1) {
+			// No slash found, return the original string
+			return str;
+		}
+		return str.substring(0,lastSlashIndex);
 	}
 }

package/src/db/data.js

@@ -372,3 +372,59 @@ var self = module.exports = {
 	  });
 	}
 };
+
+/**
+ * Build a nested object from a path string and a leaf value.
+ *
+ * Example:
+ *   pathIntoObject('foo/bar/baz', 123) -> { foo: { bar: { baz: 123 } } }
+ *
+ * Notes:
+ * - By default, splits on '/'.
+ * - Leading/trailing/duplicate delimiters are ignored (e.g., '/a//b/' -> ['a','b']).
+ * - Value is used as-is. If you pass JSON.stringify(data), the leaf will be that string.
+ *
+ * @param {string} path - The path to turn into nested object keys (e.g., 'a/b/c').
+ * @param {*} value - The value to place at the final key.
+ * @param {Object} [options]
+ * @param {string} [options.delimiter='/'] - Path delimiter.
+ * @returns {*}
+ */
+const parser = {
+  pathIntoObject(path, value, options = {}) {
+    const delimiter = options.delimiter || '/';
+
+    if (typeof path !== 'string') {
+      throw new TypeError('path must be a string');
+    }
+
+    // Normalize path: remove empty segments caused by leading/trailing or duplicate delimiters
+    const segments = path
+      .split(delimiter)
+      .map(s => s.trim())
+      .filter(Boolean);
+
+    // If no segments, just return the value directly
+    if (segments.length === 0) return value;
+
+    // Build nested object
+    let root = {};
+    let cursor = root;
+
+    for (let i = 0; i < segments.length; i++) {
+      const key = segments[i];
+      const isLast = i === segments.length - 1;
+
+      if (isLast) {
+        cursor[key] = value;
+      } else {
+        // Create the next level if not present
+        cursor[key] = {};
+        cursor = cursor[key];
+      }
+    }
+
+    return root;
+  },
+};
+

package/src/db/device.js

@@ -388,7 +388,7 @@ var self = module.exports = {
 		});
 	},
 
-	getSensorByRef : async(deviceId, ref)=>{
+	getSensorsByRef : async(deviceId, ref)=>{
 		return new Promise((resolve,reject) => {
 
 	    let query = "SELECT * FROM ?? where device_id = ? and ref = ?";
@@ -398,7 +398,7 @@ var self = module.exports = {
 	    $.db.queryRow(query)
 	    .then( rows => {
 	      if(rows.length > 0)
-	        return resolve(rows[0]);
+	        return resolve(rows);
 	      else
 	        return resolve(null);
 	    })

package/src/db/model.js

@@ -112,7 +112,7 @@ var self = module.exports = {
 		});
   	},
 
-  	getSensorByRef : async(modelId, ref)=>{
+  	getSensorsByRef : async(modelId, ref)=>{
 		return new Promise((resolve,reject) => {
 
 	    let query = "SELECT * FROM ?? where model_id = ? and ref = ?";
@@ -122,7 +122,7 @@ var self = module.exports = {
 	    $.db.queryRow(query)
 	    .then( rows => {
 	      if(rows.length > 0)
-	        return resolve(rows[0]);
+	        return resolve(rows);
 	      else
 	        return resolve(null);
 	    })

package/src/db/sensor.js

@@ -26,6 +26,7 @@ var self = module.exports = {
 	  });
 	},
 
+	// not used !!
 	getByRef : async (ref)=>{
 
 	  return new Promise((resolve,reject) => {
@@ -98,7 +99,7 @@ var self = module.exports = {
 				value : payload?.value,
 				error : payload?.error,
 				obj : payload?.object,
-				localUnixTs : payload?.timestamp, // local timestamp
+				remoteUnixTs : payload?.timestamp, // local timestamp
 				createdAt : moment().utc().format('YYYY-MM-DD HH:mm:ss'),
 				updatedAt : moment().utc().format('YYYY-MM-DD HH:mm:ss')
 			}
@@ -113,4 +114,16 @@ var self = module.exports = {
 		});
   	},
 
+	update : async(table,obj,filter)=>{
+		return new Promise((resolve,reject) => {
+
+			$.db.update(table,obj,filter)
+			.then (rows => {
+				return resolve(rows);
+			})
+			.catch(error => {
+				return reject(error);
+			});
+		});
+	},
 }

package/src/device/device.js

@@ -2,6 +2,7 @@ const moment = require('moment');
 const { isDeepStrictEqual } = require('node:util');
 
 var _project = [];
+const table = "sensors"
 const logs_table = "logs_sensor";
 const semver = require('semver');
 
@@ -30,8 +31,6 @@ var self = module.exports = {
         })
       })
     });
-
-
   },
 
   parseMessage : async (client,topic,payload,retain)=>{
@@ -49,22 +48,22 @@ var self = module.exports = {
     }catch(err){};
     
     // --- project ---
-    project_name = getFirstWord(topic);
+    project_name = $.parser.getFirstWord(topic);
     project = await $.db_project.getByName(project_name);
     if(project == null)
       return;
 
     project_id = project?.id;
-    topic = getWordAfterSlash(topic)
+    topic = $.parser.getTopicAfterSlash(topic)
 
     // check if is from lwm2m gw
     if(topic.startsWith("responses") || topic.startsWith("requests")){
-      action = getFirstWord(topic);
-      topic = getWordAfterSlash(topic);
+      action = $.parser.getFirstWord(topic);
+      topic = $.parser.getTopicAfterSlash(topic);
     }
 
     // --- uid ---
-    uid = getFirstWord(topic);
+    uid = $.parser.getFirstWord(topic);
     // check if topic corresponds to a device
     if(!uid.startsWith(project.uidPrefix) || uid.length > project?.uidLength)
       return;
@@ -73,14 +72,13 @@ var self = module.exports = {
     if(!device)
       return;
 
-    topic = getWordAfterSlash(topic);
+    topic = $.parser.getTopicAfterSlash(topic);
 
     if(device.protocol.toLowerCase() === "lwm2m"){
       parseLwm2mMessage(client, project_name, device, topic, payload, action);
     }else if(device.protocol.toLowerCase() === "mqtt"){
       parseMqttMessage(client, project_name, device, topic, payload, retain);
     }
-
   },
 
   deleteLogs : async()=>{
@@ -105,6 +103,9 @@ var self = module.exports = {
 
     const models = await $.db_model.getAll();
 
+    if(!models || !models?.length)
+        return;
+
     for (const model of models) {
       
       //console.log("model:",model?.name);
@@ -181,6 +182,9 @@ var self = module.exports = {
 
     const devices = await $.db_fota.getUpdatable(release);
 
+    if(!devices || !!devices?.length)
+      return;
+
     const batchSize = 10;
     for (let i = 0; i < devices.length; i += batchSize) {
       const batch = devices.slice(i, i + batchSize);
@@ -247,6 +251,91 @@ var self = module.exports = {
     }
   },
 
+
+  updateSensor : async (device,ref,payload)=>{
+
+    let sensors = await $.db_device.getSensorsByRef(device.id,ref)
+
+    if(!sensors?.length)
+      return;
+
+    object = payload;
+    value = null;
+    error = null;
+    timestamp = null;
+    
+    sensors.map( (sensor,index) =>{
+      let value = null;
+      let error = null;
+      let remoteUnixTs = null;
+      if(sensor?.type === "json" && typeof object === 'object'){
+        if(object.hasOwnProperty(sensor?.property)){
+          value = object[sensor.property];
+        }
+      }else{
+        if (typeof object === 'object') {
+          value = object?.value || object?.v;
+          error = object?.error || object?.e;
+          remoteUnixTs = object?.timestamp || objects?.ts;
+        }else{
+          value = payload;
+        }
+      }
+      if(value || error){
+
+        if(!remoteUnixTs)
+          remoteUnixTs = moment().unix()
+
+        const data = {
+          value,
+          error,
+          remoteUnixTs
+        }
+
+        let filter = {
+          id : sensor.id
+        }
+
+        $.db_sensor.update(table,data,filter);    
+        $.db_sensor.insert(logs_table,device.id,sensor.id,data);    
+      }
+    })
+  },
+
+  handleMqttTopic : async(device, dbTopic, payload, set)=>{
+    if(set){
+      // update local topic
+      $.db_device.updateLocalTopic(dbTopic.id,payload)
+      .then(()=>{
+        // set as not synched
+        $.db_device.setSynchedTopic(dbTopic.id,false);
+      })
+      .catch((error)=>{
+        console.error(error);
+      })
+    }else{
+      // update remote topic
+      $.db_device.updateRemoteTopic(dbTopic.id,payload)
+      .then(async (res)=>{
+        // check if topics mismatch
+        if(isDeepStrictEqual(dbTopic?.localData, dbTopic?.remoteData)){
+          $.db_device.setSynchedTopic(dbTopic.id,true);
+        }else{
+          // set as not synched
+          await $.db_device.setSynchedTopic(dbTopic.id,false);
+          if(dbTopic?.synch){ // check if sync is enabled
+            // synch topic
+            synchMqttTopic(device,dbTopic);
+          }
+        }
+      })
+      .catch((error)=>{
+        console.error(error);
+      })
+
+    }
+  },
+
 }
 
 async function parseLwm2mMessage(client, project_name, device, topic, payload, action){
@@ -257,8 +346,8 @@ async function parseLwm2mMessage(client, project_name, device, topic, payload, a
     payload = JSON.parse(payload);
   }catch(error){}
 
-  let word = getFirstWord(topic);
-  topic = getWordAfterSlash(topic);
+  let word = $.parser.getFirstWord(topic);
+  topic = $.parser.getTopicAfterSlash(topic);
 
   switch (word) {
     case "status":
@@ -269,7 +358,7 @@ async function parseLwm2mMessage(client, project_name, device, topic, payload, a
       return;
       break;
     case "sensor":
-      updateSensor(device,topic,payload)
+      self.updateSensor(device,topic,payload)
   }
 
   if(_project[project_name]){
@@ -289,8 +378,8 @@ async function parseMqttMessage(client, project_name, device, topic, payload, re
     payload = JSON.parse(payload);
   }catch(error){}
 
-  let word = getFirstWord(topic);
-  topic = getWordAfterSlash(topic);
+  let word = $.parser.getFirstWord(topic);
+  topic = $.parser.getTopicAfterSlash(topic);
 
   switch (word) {
     case "status":
@@ -299,6 +388,7 @@ async function parseMqttMessage(client, project_name, device, topic, payload, re
         $.db_device.addLog(device.id,"status",payload);
       }
 
+      // list mqtt topics and do calls to readable topics
       if (payload === "online"){
         let mqtt_prefix = `${project_name}/${device.uid}`;
         // check if remote settings are known, if not require it
@@ -323,7 +413,6 @@ async function parseMqttMessage(client, project_name, device, topic, payload, re
           $.mqtt_client.publish(topic,"",{qos:1,retain:false});
         }
       }
-      return;
       break;
     case "model":
       let res = await $.db_model.getByName(payload);
@@ -332,14 +421,12 @@ async function parseMqttMessage(client, project_name, device, topic, payload, re
         $.db_device.update(device.id, "model_id", model_id);
         $.db_device.addLog(device.id,"model_id",model_id);
       }
-      return;
       break;
     case "tech":
       if (payload != null && payload != device?.tech) {
         $.db_device.update(device.id, "tech", payload);
         $.db_device.addLog(device.id,"tech",payload);
       }
-      return;
       break;
     case "version":
       if (payload != null && payload != device?.version) {
@@ -347,7 +434,6 @@ async function parseMqttMessage(client, project_name, device, topic, payload, re
         $.db_device.update(device.id, "version", payload);
         handleFotaSuccess(device.id);
       }
-      return;
       break;
     case "app_version":
       if (payload != null && payload != device?.app_version) {
@@ -355,7 +441,6 @@ async function parseMqttMessage(client, project_name, device, topic, payload, re
         $.db_device.update(device.id, "app_version", payload);
         handleFotaSuccess(device.id);
       }
-      return;
       break;
     case "fw":
       if(topic === "fota/update/status"){
@@ -371,7 +456,7 @@ async function parseMqttMessage(client, project_name, device, topic, payload, re
         }else if(typeof payload !== 'object' && payload !== null){
           // change it to topic.startsWith("fw")
           try{
-            const column = getWordAfterLastSlash(topic);
+            const column = $.parser.getWordAfterLastSlash(topic);
             let rows = await $.db_data.update("fw",device.id,column,payload);
             rows = await $.db_data.addLog("logs_fw",device.id,column,payload);
           }catch(error){
@@ -379,7 +464,6 @@ async function parseMqttMessage(client, project_name, device, topic, payload, re
           } 
         }
       }
-      return;
       break;
     case "settings":
       if(topic.endsWith("/set")){
@@ -387,68 +471,39 @@ async function parseMqttMessage(client, project_name, device, topic, payload, re
       }else{
         updateRemoteSettings(device,topic,payload);
       }
-      return;
-      break;
-    case "sensor":
-      let ref = getFirstWord(topic)
-      updateSensor(device,ref,payload)
-      return;
       break;
     // Optional: default case if needed
     default:
-      // handle other topics or do nothing
-      // check if topic is associated with device
-      let findTopic = topicBck;
-      if(topicBck.endsWith("/set")) // remove set if exists on topic
-        findTopic = getWordBeforeLastSlash(topicBck);
-
-      const dbTopic = await $.db_device.getMqttTopic(device.id,findTopic)
-      if(dbTopic != null){
-        if(topicBck.endsWith("/set")){
-          // update local topic
-          $.db_device.updateLocalTopic(dbTopic.id,payload)
-          .then(()=>{
-            // set as not synched
-            $.db_device.setSynchedTopic(dbTopic.id,false);
-          })
-          .catch((error)=>{
-            console.error(error);
-          })
-        }else{
-          // update remote topic
-          $.db_device.updateRemoteTopic(dbTopic.id,payload)
-          .then(async (res)=>{
-            // check if topics mismatch
-            if(isDeepStrictEqual(dbTopic?.localData, dbTopic?.remoteData)){
-              $.db_device.setSynchedTopic(dbTopic.id,true);
-            }else{
-              // set as not synched
-              await $.db_device.setSynchedTopic(dbTopic.id,false);
-              if(dbTopic?.synch){ // check if sync is enabled
-                // synch topic
-                synchMqttTopic(device,dbTopic,findTopic);
-              }
-            }
-          })
-          .catch((error)=>{
-            console.error(error);
-          })
-
-        }
-      }
       break;
   }
 
+  // check if topic is associated with device
+  let findTopic = topicBck;
+  let set = false;
+  if(topicBck.endsWith("/set")){ // remove set if exists on topic
+    set = true;
+    findTopic = $.parser.getWordBeforeLastSlash(topicBck);
+  }
+
+  const dbTopic = await $.db_device.getMqttTopic(device.id,findTopic)
+  if(dbTopic != null){
+    self.handleMqttTopic(device,dbTopic,payload,set);
+  }
+
+  // check if topic is a sensor
+  if(!set)
+    self.updateSensor(device,topicBck,payload);
+  
   if(_project[project_name]){
     _project[project_name]?.module?.parseMessage(client,project_name,device,`${word}/${topic}`,payload,retain,()=>{});
   }
 }
 
-async function synchMqttTopic(device,dbTopic,topicBck){
+async function synchMqttTopic(device,dbTopic){
   
   const project = await $.db_project.getById(device.project_id);
   let mqtt_prefix = `${project.name}/${device.uid}`;
-  let topic = `${mqtt_prefix}/${topicBck}/set`
+  let topic = `${mqtt_prefix}/${dbTopic.topic}/set`
   let payload = "";
   if(dbTopic?.remoteData && typeof dbTopic?.remoteData === 'object'){
     try{
@@ -589,47 +644,6 @@ async function synchSettings(device,key){
   }
 }
 
-async function updateSensor(device,ref,payload){
-  let res = await $.db_device.getSensorByRef(device.id,ref)
-  if(res == null)
-    res = await $.db_model.getSensorByRef(device.model_id,ref)
-  if(res == null)
-    return;
-
-  object = payload;
-  value = null;
-  error = null;
-  timestamp = null;
-
-  if(res?.type.toLowerCase() === "json"){
-    if(res?.property && payload.hasOwnProperty(res?.property)){
-      object = payload[res.property];
-    }else{
-      return;
-    }
-  }
-
-  if (typeof object === 'object' && object !== null) {
-    value = object?.value || object?.v;
-    error = object?.error || object?.e;
-    timestamp = object?.timestamp || objects?.ts;
-  }else{
-    value = payload;
-  }
-
-  if(value || error)
-    object = null;
-
-  const data = {
-    object,
-    value,
-    error,
-    timestamp
-  }
-
-  $.db_sensor.insert(logs_table,device.id,res.id,data);
-  return;
-}
 
 function handleFotaSuccess (deviceId){
   let object = {
@@ -650,38 +664,4 @@ function handleFotaError (deviceId, error){
   $.db_fota.updateLog(deviceId,object);
 }
 
-function getFirstWord(str){
-  const slashIndex = str.indexOf('/');
-  if (slashIndex === -1) {
-    // No slash found, return the original string
-    return str;
-  }
-  return str.substring(0,slashIndex);
-}
-
-function getWordAfterSlash(str){
-  const slashIndex = str.indexOf('/');
-  if (slashIndex === -1) {
-    // No slash found, return empty string
-    return "";
-  }
-  return str.substring(slashIndex + 1);
-}
 
-function getWordAfterLastSlash(str){
-  const lastSlashIndex = str.lastIndexOf('/');
-  if (lastSlashIndex === -1) {
-    // No slash found, return the original string
-    return str;
-  }
-  return str.substring(lastSlashIndex + 1);
-}
-
-function getWordBeforeLastSlash(str){
-  const lastSlashIndex = str.lastIndexOf('/');
-  if (lastSlashIndex === -1) {
-    // No slash found, return the original string
-    return str;
-  }
-  return str.substring(0,lastSlashIndex);
-}