mqtt-devices-parser 1.0.0

package/Changelog.md

@@ -0,0 +1,3 @@
+
+# 1.0.0
+	- Module to parse mqtt messages from devices

package/Readme.md

@@ -0,0 +1,77 @@
+
+# mqtt-devices-parser
+
+MQTT parser
+Parses topics received from devices and updates data on dB
+
+## Setup
+
+### Command
+  >> node deploy
+
+  1. build/sync all tables inside models folder
+  2. registers users and clients if not registered already
+
+  3. build/sync all tables inside projects folder
+  4. registers projects and models/apps if not registered already
+
+## Working mode
+
+### Command
+  >> node index
+
+  1. Checks all projects inside src/projects folder and enable it if is activated in config/index.js
+
+  2. Connects to defined MQTT broker
+
+  3. Parse MQTT messages starting with :project/MACRO_UID_PREFIX+uid/
+
+    3.1 register device if not exists
+    3.2 Associate to a project if project is registered
+    3.3 Associate to a model/app if model is registered
+
+  4. Call project parseMessage function
+
+## Instructions
+
+  Projects folders must be kept
+
+  The following functions must exist inside a file with the same name of the project
+
+  ```
+  module.exports = {
+    sync_db : async()=>{
+    },
+    init : async ()=>{
+    },
+    parseMessage : async (client, project, device, topic, payload, retain, cb)=>{
+    },
+  }
+  ```
+
+  Ex: If project name is freeRTOS2, so must exist a file in projects/freeRTOS2/freeRTOS2.js.
+
+  Calling the following methods will write data in a ${table} with the name of the project, and also
+  in other table called "logs_"+${table}. Data will be written if there is a field in the respective table
+  with the same name as the name in topic.
+
+  ```
+  $.db_data.update(project,device?.id,topic,payload);
+  $.db_data.addLog("logs_"+project,device.id,topic,payload);
+  ```
+
+## UnitTest
+
+
+## Dependencies
+
+- mysql8.0
+- MQTT
+
+## Front end
+- Node service running the following [project](https://github.com/zimbora/mgmt-iot-web)
+
+## Configuration
+
+The default configuration uses ./config/index.js file\
+In your program set the same struct and pass it as argument on the module init call.

package/config/index.js

@@ -0,0 +1,29 @@
+module.exports = {
+  version : "0.0.0",
+  dev : process.env.dev || "true",
+  web:{
+    protocol : process.env.HTTP_PROTOCOL  || "http://",
+    domain: process.env.DOMAIN            || '192.168.1.1',
+  },
+  mqtt: {
+    protocol:process.env.MQTT_PROTOCOL || 'MQTT',
+    host:process.env.MQTT_HOST || 'localhost',
+    port:process.env.MQTT_PORT || '1883',
+    user:process.env.MQTT_USER || 'admin',
+    pwd:process.env.MQTT_PWD || 'admin',
+    client:process.env.MQTT_CLIENT || 'mqtt-devices-parser',
+    logs_path: process.env.MQTT_LOGS || 'uServices'
+  },
+  mysqldb: {
+    conn_limit: process.env.DB_CONN_LIMIT || 15,
+    host: process.env.DB_HOST || 'localhost',
+    port: process.env.DB_PORT || 3306,
+    user: process.env.DB_USER || 'user',
+    pwd: process.env.DB_PWD   || 'user_pwd',
+    name: process.env.DB_NAME || 'mqtt-aedes',
+  },
+  sync_main_tables : process.env.sync_main_tables || true,
+  projects: {
+    demo : process.env.demo || false,
+  }
+}

package/deploy.js

@@ -0,0 +1,70 @@
+const fs = require('fs').promises;
+
+$ = {};
+$.config = require('./config');
+$.models = require('./models/models.js');
+var projectsPath = './src/projects/';
+var project = [];
+
+async function readModelsInsideProjects(){
+
+	return new Promise( async (resolve,reject) => {
+
+		// load projects dbs
+		var files = await fs.readdir(projectsPath)
+
+	    files = files.filter((file) => {
+	    	// Exclude any extension file
+	    	return (file.indexOf('.') == -1);
+	    })
+
+	    let counter_help = 0;
+	    files.forEach( async(name,counter) => {
+	    	if($.config.projects[name]){
+		    	project[name] = {
+		        	module : null
+				};
+				project[name].module = require('./src/projects/'+name+'/'+name+".js")
+				await project[name].module.sync_db();
+				counter_help++;
+				if(files.length == counter_help) return resolve();
+	    	}else{
+	    		counter_help++;
+				if(files.length == counter_help) return resolve();
+	    	}
+
+	    });
+
+	});
+
+}
+
+
+async function init(){
+
+	await $.models.init();
+	await $.models.connect();
+
+	//main tables
+	if($.config.sync_main_tables){
+		await $.models.load(__dirname+"/models");
+		await $.models.dropTableIndexes();
+		await $.models.sync();
+
+		let user = await $.models.insertUser("admin","admin",5); // ads user
+		//console.log(user);
+		let user_id = user.dataValues.id;
+		await $.models.insertUser("device","device",3);
+		await $.models.insertUser("client","client_pwd",3);
+		await $.models.insertClient("admin","admin",user_id); // ads client with credentials admin@admin
+	}
+
+	// project tables
+	await readModelsInsideProjects();
+	await $.models.dropTableIndexes();
+    await $.models.sync();
+
+}
+
+
+init();

package/index.js

@@ -0,0 +1,113 @@
+const mqtt = require("mqtt");
+
+$ = {};
+$.md5 = require('md5');
+
+$.config = require('./config');
+$.models = require('./models/models.js');
+$.device = require('./src/device/device.js');
+$.db = require('./src/db/db');
+$.db_data = require('./src/db/data');
+$.db_project = require('./src/db/project');
+$.db_device = require('./src/db/device');
+$.db_model = require('./src/db/model');
+$.db_firmware = require('./src/db/firmware');
+$.db_sensor = require('./src/db/sensor');
+
+const packageJson = require(__dirname+'/package.json');
+const packageVersion = packageJson.version;
+
+var projects = [];
+
+var self = module.exports = {
+
+  init : async(_config, _projects)=>{
+
+    return new Promise(async (resolve,reject) => {
+
+      if(_config != null)
+        $.config = _config;
+      if(_projects != null)
+        projects = _projects;
+
+      await $.models.init();
+      await $.models.connect();
+
+      //auth.init();
+      await $.device.init(projects);
+
+      console.log("mqtt connecting..")
+      mqtt_connect();
+
+      return resolve();
+    })
+  }
+}
+
+function mqtt_connect(){
+
+  const mqtt_prefix = $.config.mqtt.logs_path+"/"+$.config.mqtt.client;
+  const client = mqtt.connect({
+    host : $.config.mqtt.host,
+    port : $.config.mqtt.port,
+    username : $.config.mqtt.user,
+    password : $.config.mqtt.pwd,
+    clientId : $.config.mqtt.client,
+    protocolId : $.config.mqtt.protocol,
+    will : {
+      topic: mqtt_prefix+"/status",
+      payload: "offline",
+      qos: 2,
+      retain: true
+    }
+  });
+
+  client.on("connect", () => {
+
+    client.publish(mqtt_prefix+"/status","online",{qos:2,retain:true});
+    client.publish(mqtt_prefix+"/version",packageVersion,{qos:2,retain:true});
+
+    projects.map( (project)=>{
+      client.subscribe(project+"/#", (err) => {});
+
+      for(let project in $.config.projects){
+        if($.config.projects[project])
+          client.publish(mqtt_prefix+"/"+project,"active");
+        else
+          client.publish(mqtt_prefix+"/"+project,"deactive");
+      };
+    })
+    console.log(`MQTT connected to: ${$.config.mqtt.host}:${$.config.mqtt.port}`);
+    projects.map( project=>{
+      console.log("subscribing project:",project);
+      client.subscribe(project+"/#")
+    })
+  });
+
+  client.on("message", (topic, payload, packet) => {
+    // payload is Buffer
+    $.device.parseMessage(client,topic.toString(),payload.toString(),packet.retain);
+  });
+
+  client.on("reconnect",()=>{
+    console.log("reconnect")
+  });
+
+  client.on("close",()=>{
+    console.log("close")
+  });
+
+  client.on("offline",()=>{
+    console.log("offline")
+  });
+
+  client.on("disconnect",(packet)=>{
+    console.log(packet)
+  })
+
+  client.on("error",(error)=>{
+    console.log(error)
+  })
+}
+
+

package/models/clients.models.js

@@ -0,0 +1,41 @@
+
+module.exports = (sequelize,DataTypes)=>{
+	return sequelize.define("clients", {
+		user_id: {
+			type: DataTypes.INTEGER,
+			references : {
+				model: 'users',
+				key: 'id'
+			}
+		},
+		nick: {
+			type: DataTypes.STRING(64),
+			unique: true
+		},
+		token: {
+			type: DataTypes.STRING,
+			allowNull: true
+		},
+		api_token: {
+			type: DataTypes.STRING,
+			allowNull: true
+		},
+		gmail: {
+			type: DataTypes.STRING(64),
+			//unique: true,
+			allowNull: true
+		},
+		name: {
+			type: DataTypes.STRING,
+			allowNull: true
+		},
+		avatar: {
+			type: DataTypes.STRING,
+			allowNull: true
+		},
+	},
+	{
+		tableName: 'clients',
+		freezeTableName: true
+	})
+}

package/models/devices.models.js

@@ -0,0 +1,39 @@
+
+module.exports = (sequelize,DataTypes)=>{
+	return sequelize.define("devices", {
+		uid: {
+			type: DataTypes.STRING,
+			unique: true
+		},
+		name: {
+			type: DataTypes.STRING,
+			allowNull: true
+		},
+		project: { // deprecated
+			type: DataTypes.STRING,
+			allowNull: true
+		},
+		status: {
+			type: DataTypes.STRING,
+			allowNull: true
+		},
+		project_id: {
+			type: DataTypes.INTEGER,
+			references: {
+				model: 'projects',
+				key: 'id'
+			}
+		},
+		model_id: {
+			type: DataTypes.INTEGER,
+			references: {
+				model: 'models',
+				key: 'id'
+			}
+		},
+	},
+	{
+		tableName: 'devices',
+		freezeTableName: true
+	})
+}

package/models/firmwares.models.js

@@ -0,0 +1,41 @@
+
+module.exports = (sequelize,DataTypes)=>{
+	return sequelize.define("firmwares", {
+		filename: {
+			type: DataTypes.STRING,
+			allowNull: true
+		},
+		originalname: {
+			type: DataTypes.STRING,
+			allowNull: true,
+			unique: true
+		},
+		fw_version: {
+			type: DataTypes.STRING,
+			allowNull: true
+		},
+		app_version: {
+			type: DataTypes.STRING,
+			allowNull: true
+		},
+		fw_release: {
+			type: DataTypes.STRING,
+			allowNull: true
+		},
+		model_id: { // deprecated
+			type: DataTypes.INTEGER,
+			references: {
+				model: 'models',
+				key: 'id'
+			},
+		},
+		token: {
+			type: DataTypes.STRING,
+			allowNull: true
+		},
+	},
+	{
+		tableName: 'firmwares',
+		freezeTableName: true
+	})
+}

package/models/modelPermissions.js

@@ -0,0 +1,23 @@
+
+module.exports = (sequelize,DataTypes)=>{
+	return sequelize.define("modelPermissions", {
+		client_id: {
+			type: DataTypes.INTEGER,
+			references: {
+				model: 'clients',
+				key: 'id'
+			}
+		},
+		model_id: {
+			type: DataTypes.INTEGER,
+			references: {
+				model: 'models',
+				key: 'id'
+			}
+		}
+	},
+	{
+		tableName: 'modelPermissions',
+		freezeTableName: true
+	})
+}

package/models/models.js

@@ -0,0 +1,226 @@
+const fs = require('fs').promises;
+const path = require('path');
+const {Sequelize, DataTypes} = require("sequelize");
+var SqlString = require('sequelize/lib/sql-string')
+var mysql = require('mysql2');
+var sequelize;
+
+//var modelsPath = './models/';
+var config = require('../config');
+
+var self = module.exports = {
+
+   init : async ()=>{
+
+      return new Promise((resolve,reject) => {
+         sequelize = new Sequelize(
+            config.mysqldb.name,
+            config.mysqldb.user,
+            config.mysqldb.pwd,
+            {
+               host: config.mysqldb.host,
+               dialect: 'mysql'
+            },
+         );
+         return resolve(sequelize);
+      })
+   },
+
+   connect : async ()=>{
+
+      return new Promise((resolve,reject) => {
+         sequelize.authenticate().then(() => {
+            console.log('Connection has been established successfully.');
+            return resolve();
+         }).catch((error) => {
+            console.error('Unable to connect to the database: ', error);
+            return reject();
+         });
+      });
+   },
+
+   load : async (modelsPath)=>{
+
+      return new Promise(async(resolve,reject) => {
+         var files = await fs.readdir(modelsPath)
+         files = files.filter((file) => {
+            // Exclude models.js and any non-js files
+            return (file.indexOf('.') !== 0) && (file !== 'models.js') && (file.slice(-9) === 'models.js');
+         })
+         files.forEach((file,counter) => {
+            const model = require(path.join(modelsPath, file));
+            model(sequelize, DataTypes);
+            if(counter == files.length-1){
+               return resolve();
+            }
+         });
+      });
+   },
+
+   sync : async ()=>{
+
+      return new Promise((resolve,reject) => {
+         //sequelize.sync({alter:true,force:true}).then(() => {
+         sequelize.sync({alter:true}).then( async () => {
+            console.log('All tables were synced!');
+            return resolve();
+         }).catch((error) => {
+            console.error('Unable to create table : ', error);
+            return reject();
+         });
+      });
+   },
+
+   dropTableIndexes : async()=>{
+
+      return new Promise((resolve,reject) => {
+         let tables = Object.keys(sequelize.models);
+
+         console.log(tables);
+         if(!tables?.length)
+            return resolve();
+
+         tables.forEach(async(tableName,counter)=>{
+            self.dropAllIndexes(tableName)
+            .then(res =>{
+               if(counter == tables.length-1)
+                  return resolve();
+            })
+            .catch(error => {
+               if(counter == tables.length-1)
+                  return resolve();
+            });
+
+         });
+      });
+   },
+
+   dropAllIndexes : async (tableName)=>{
+
+      return new Promise( async(resolve,reject) => {
+         // Fetch all indexes from the table
+         let query = `SHOW INDEXES FROM ??`
+         query = mysql.format(query,[tableName]);
+
+         var indexes;
+         sequelize.query(query)
+         .then(res=>{
+            indexes = res;
+
+            // Filter out primary key index and unique indexes if you don't want to drop them
+            const filteredIndexes = indexes[0].filter(index => index.Key_name !== 'PRIMARY' && !index.Non_unique);
+
+            if(!filteredIndexes?.length)
+               return resolve();
+
+            // Drop each index
+            filteredIndexes.forEach( async(index,counter)=>{
+               let query = `DROP INDEX ?? ON ??`
+               query = mysql.format(query,[index.Key_name,tableName]);
+               sequelize.query(query)
+               .then(res=>{
+                  console.log(`Dropped index ${index.Key_name}`);
+                  if(counter == filteredIndexes.length-1)
+                     return resolve();
+               })
+               .catch(error=>{
+                  if(counter == filteredIndexes.length-1)
+                     return resolve();
+               })
+            })
+         })
+         .catch(error=>{
+            return resolve();
+         })
+      });
+   },
+
+   get: (modelName)=>{
+
+      if(modelName != ""){
+         if(sequelize.models[modelName]){
+            const model = sequelize.models[modelName];
+            const keys = Object.keys(model.rawAttributes);
+            let obj = {}
+            keys.map((key)=>{
+               obj[key] = null;
+            })
+            return obj;
+         }else return;
+      }else{
+         return models = Object.keys(sequelize.models).map(modelName => {
+            const model = sequelize.models[modelName];
+            const keys = Object.keys(model.rawAttributes);
+            return {
+               modelName,
+               keys
+            };
+         });
+      }
+   },
+
+   getUser : async (type)=>{
+
+      return new Promise( async (resolve,reject) => {
+         const user = await sequelize.models['users'].findOne({
+             where: {
+                 type: type
+             }
+         });
+         return resolve(user);
+      });
+   },
+
+   insertUser: async (type, password,level)=>{
+
+      return new Promise( async (resolve,reject) => {
+         // Insert a record if it doesn't exist
+         const res = await sequelize.models['users'].findOrCreate({
+           where: { type: type },
+           defaults: {
+               password: password,
+               level: level
+            }
+         })
+         .then(([user, created]) => {
+            if (created) {
+               console.log('New user created:', type);
+            } else {
+               console.log('User already exists:', type);
+            }
+            return resolve(user);
+         })
+         .catch(error => {
+           console.error('Error:', error);
+           return reject(error);
+         });
+      });
+   },
+
+   insertClient: async (nick, token, user_id)=>{
+
+      return new Promise( async (resolve,reject) => {
+         // Insert a record if it doesn't exist
+         const res = await sequelize.models['clients'].findOrCreate({
+           where: { nick: nick },
+           defaults: {
+               user_id: user_id,
+               token: token
+            }
+         })
+         .then(([client, created]) => {
+            if (created) {
+               console.log('New client created:', nick);
+            } else {
+               console.log('Client already exists:', nick);
+            }
+            return resolve(client);
+         })
+         .catch(error => {
+           console.error('Error:', error);
+         });
+      });
+   }
+
+}
+

package/models/models.models.js

@@ -0,0 +1,40 @@
+
+module.exports = (sequelize,DataTypes)=>{
+	return sequelize.define("models", {
+		name: {
+			type: DataTypes.STRING,
+			unique: true
+		},
+		description: {
+			type: DataTypes.STRING
+		},
+		model_table: { // deprecated
+			type: DataTypes.STRING,
+			allowNull: true
+		},
+		logs_table: { // deprecated
+			type: DataTypes.STRING,
+			allowNull: true
+		},
+		fw_enabled: {
+			type: DataTypes.BOOLEAN,
+			defaultValue: false
+		},
+		ar_enabled: {
+			type: DataTypes.BOOLEAN,
+			defaultValue: false
+		},
+		alarms_enabled: {
+			type: DataTypes.BOOLEAN,
+			defaultValue: false
+		},
+		js_code_enabled: {
+			type: DataTypes.BOOLEAN,
+			defaultValue: false
+		}
+	},
+	{
+		tableName: 'models',
+		freezeTableName: true
+	})
+}