@frangoteam/fuxa-min 1.1.19

package/runtime/storage/calculator.js

@@ -0,0 +1,227 @@
+/**
+ *  Calculator for DAQ value array (integrale, max, min, average...) 
+ */
+
+'use strict';
+
+const utils = require("../utils");
+
+function getFunctionValues(values, fromts, tots, fnc, interval, format) {
+    if (fnc === ReportFunctionType.min) {
+        return getMin(values, fromts, tots, interval);
+    } else if (fnc === ReportFunctionType.max) {
+        return getMax(values, fromts, tots, interval);
+    } else if (fnc === ReportFunctionType.average) {
+        return getAverage(values, fromts, tots, interval, format);
+    } else if (fnc === ReportFunctionType.sum) {
+        return getSum(values, fromts, tots, interval);
+    }
+}
+
+/**
+ * 
+ * @param {*} values 
+ * @param {*} fnc 
+ * @param {*} intervalType hour / day 
+ * @returns 
+ */
+function getMin(timeserie, fromts, tots, intervalType) {
+    let result = getInterval(fromts, tots, intervalType, Number.MAX_VALUE);
+    // sort to start with the oldest
+    let sorted = timeserie.sort(function (a, b) {
+        return a.dt - b.dt;
+    });
+
+    let addToInterval = (intervals, intervalIndex, value) => {
+        if (!intervals[intervalIndex]) {
+            intervals[intervalIndex] = Number.MAX_VALUE;
+        } else if (intervals[intervalIndex] > value) {
+            intervals[intervalIndex] = value;
+        }
+    }
+    
+    for (let i = 0; i < sorted.length; i++) {
+        let intervalIndex = getIntervalTime(sorted[i].dt, intervalType, false).getTime();
+        addToInterval(result, intervalIndex, utils.parseFloat(sorted[i].value, 5));
+    }
+    return result;
+}
+
+function getMax(timeserie, fromts, tots, intervalType) {
+    let result = getInterval(fromts, tots, intervalType, utils.SMALLEST_NEGATIVE_INTEGER);
+    // sort to start with the oldest
+    let sorted = timeserie.sort(function (a, b) {
+        return a.dt - b.dt;
+    });
+
+    let addToInterval = (intervals, intervalIndex, value) => {
+        if (!intervals[intervalIndex]) {
+            intervals[intervalIndex] = utils.SMALLEST_NEGATIVE_INTEGER;
+        } else if (intervals[intervalIndex] < value) {
+            intervals[intervalIndex] = value;
+        }
+    }
+    
+    for (let i = 0; i < sorted.length; i++) {
+        let intervalIndex = getIntervalTime(sorted[i].dt, intervalType, false).getTime();
+        addToInterval(result, intervalIndex, utils.parseFloat(sorted[i].value, 5));
+    }
+    return result;
+}
+
+function getAverage(timeserie, fromts, tots, intervalType, format) {
+    let result = getInterval(fromts, tots, intervalType, null);
+    let counts = getInterval(fromts, tots, intervalType, 0);
+    // sort to start with the oldest
+    let sorted = timeserie.sort(function (a, b) {
+        return a.dt - b.dt;
+    });
+
+    let addToInterval = (intervals, counters, intervalIndex, value) => {
+        if (utils.isNullOrUndefined(intervals[intervalIndex])) {
+            intervals[intervalIndex] = value;
+        } else {
+            intervals[intervalIndex] += value;
+        }
+        counters[intervalIndex]++;
+    }
+    
+    for (let i = 0; i < sorted.length; i++) {
+        let intervalIndex = getIntervalTime(sorted[i].dt, intervalType, false).getTime();
+        addToInterval(result, counts, intervalIndex, parseFloat(sorted[i].value));
+    }
+    // average
+    Object.keys(result).forEach(k => {
+        if (counts[k]) {
+            result[k] = utils.parseFloat(result[k] / counts[k], format || 5);
+        }
+    });
+    return result; 
+}
+
+function getSum(timeserie, fromts, tots, intervalType) {
+    let result = getInterval(fromts, tots, intervalType, 0);
+    // sort to start with the oldest
+    let sorted = timeserie.sort(function (a, b) {
+        return a.dt - b.dt;
+    });
+
+    let addToInterval = (intervals, intervalIndex, value) => {
+        if (utils.isNullOrUndefined(intervals[intervalIndex])) {
+            intervals[intervalIndex] = 0;
+        } else {
+            intervals[intervalIndex] += value;
+            intervals[intervalIndex] = utils.parseFloat(intervals[intervalIndex], 5)
+        }
+    }
+    
+    for (let i = 0; i < sorted.length; i++) {
+        let intervalIndex = getIntervalTime(sorted[i].dt, intervalType, false).getTime();
+        addToInterval(result, intervalIndex, parseFloat(sorted[i].value));
+    }
+    return result;    
+}
+
+
+function getIntegral(timeserie, fromts, tots, intervalType) {
+    let result = getInterval(fromts, tots, intervalType, Number.MAX_VALUE);
+    // sort to start with the oldest
+    let sorted = timeserie.sort(function (a, b) {
+        return a.dt - b.dt;
+    });
+
+    let addToInterval = (intervals, intervalIndex, value) => {
+        if (!intervals[intervalIndex]) {
+            intervals[intervalIndex] = value;
+        } else if (intervals[intervalIndex] > value) {
+            intervals[intervalIndex] += value;
+        }
+    }
+    
+    let lastRecord = null;// : TimeValue { dt: number, value: number };
+    let lastIntervalIndex = null;
+    for (let i = 0; i < sorted.length; i++) {
+        let intervalIndex = getIntervalTime(sorted[i].dt, intervalType, false).getTime();
+        // check missing value to fill intervalsIndex
+        while (lastIntervalIndex && lastIntervalIndex < intervalIndex) {
+            let nextIntervalIndex = getIntervalTime(lastRecord.dt, intervalType, true).getTime();
+            let delta = nextIntervalIndex - lastRecord.dt;
+            addToInterval(result, nextIntervalIndex, parseFloat(lastRecord.value) * (delta / 1000));
+            lastIntervalIndex = nextIntervalIndex;
+            lastRecord.dt = nextIntervalIndex;
+        }
+        // sum left => skip the first one
+        if (lastRecord) {
+            let delta = sorted[i].dt - lastRecord.dt;
+            addToInterval(result, intervalIndex, parseFloat(sorted[i].value) * (delta / 1000));
+        }
+
+        lastRecord = sorted[i];
+        lastIntervalIndex = intervalIndex;
+    }
+    return result;
+}
+
+function getInterval(fromts, tots, type, defvalue) {
+    let result = {};
+    let dt = getStepDate(fromts, type, 0);
+    while (dt.getTime() < tots) {
+        result[dt.getTime()] = defvalue;
+        dt = getStepDate(dt, type, 1);
+    }
+    return result;
+}
+
+function getIntervalTime(millyDt, _interval, next) {
+    let dt = new Date(millyDt);
+    let toadd = (next) ? 1 : 0;
+    return getStepDate(dt, _interval, toadd);
+}
+
+function getStepDate(ts, type, toadd) {
+    var dtRange;
+    let dt = new Date(ts);
+    if (type === ReportIntervalType.day) {
+        dtRange = new Date(dt.getFullYear(), dt.getMonth(), dt.getDate(), 0, 0, 0);
+        dtRange.setDate(dt.getDate() + toadd);
+    } else if (type === ReportIntervalType.hour) {
+        dtRange = new Date(dt.getFullYear(), dt.getMonth(), dt.getDate(), dt.getHours(), 0, 0);
+        dtRange.setHours(dt.getHours() + toadd);
+    } else if (type === ReportIntervalType.min30) {
+        dtRange = new Date(dt.getFullYear(), dt.getMonth(), dt.getDate(), dt.getHours(), 0, 0);
+        var minutesRange = parseInt(dt.getMinutes() / 30);
+        dtRange.setMinutes((minutesRange + toadd) * 30);
+    } else if (type === ReportIntervalType.min10) {
+        dtRange = new Date(dt.getFullYear(), dt.getMonth(), dt.getDate(), dt.getHours(), 0, 0);
+        var minutesRange = parseInt(dt.getMinutes() / 10);
+        dtRange.setMinutes((minutesRange + toadd) * 10);
+    } else { //if (type === ReportIntervalType.min5) {
+        dtRange = new Date(dt.getFullYear(), dt.getMonth(), dt.getDate(), dt.getHours(), 0, 0);
+        var minutesRange = parseInt(dt.getMinutes() / 5);
+        dtRange.setMinutes((minutesRange + toadd) * 5);
+    }
+    return dtRange;
+}
+
+module.exports = {
+    getFunctionValues: getFunctionValues,
+    getMin: getMin,
+    getMax: getMax,
+    getAverage: getAverage,
+    getSum: this.getSum,
+};
+
+const ReportIntervalType = {
+    min5: 'min5',
+    min10: 'min10',
+    min30: 'min30',
+    hour: 'hour',
+    day: 'day',
+}
+
+const ReportFunctionType = {
+    min: 'min',
+    max: 'max',
+    average: 'average',
+    sum: 'sum',
+}

package/runtime/storage/daqstorage.js

@@ -0,0 +1,188 @@
+/**
+ *  Module to manage the DAQ datastore with daqnode 
+ */
+
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const SqliteDB = require("./sqlite");
+const InfluxDB = require("./influxdb");
+const TDengine  =require("./tdengine");
+const CurrentStorage = require("./sqlite/currentstorage");
+// var DaqNode = require('./daqnode');
+var calculator = require('./calculator');
+var utils = require('../utils');
+
+var daqStoreType;
+
+var settings;
+var logger;
+var daqDB = {};                 // list of daqDB node: SQlite one pro device, influxDB only one
+var currentStorateDB;
+
+function init(_settings, _log) {
+    settings = _settings;
+    logger = _log;
+    logger.info("daqstorage: init successful!", true);
+    currentStorateDB = CurrentStorage.create(_settings, _log);
+}
+
+function reset() {
+    daqStoreType = _getDbType();
+    for (var id in daqDB) {
+        daqDB[id].close();
+    }
+    daqDB = {};
+    logger.info("daqstorage reset!", true);
+}
+
+function addDaqNode(_id, fncgetprop) {
+    var id = _id;
+    const dbType = _getDbType();
+    if (dbType === DaqStoreTypeEnum.influxDB || dbType === DaqStoreTypeEnum.influxDB18 || dbType === DaqStoreTypeEnum.TDengine) {
+        id = dbType;
+    }
+    if (!daqDB[id]) {
+        if (id === DaqStoreTypeEnum.influxDB || id === DaqStoreTypeEnum.influxDB18) {
+            daqDB[id] = InfluxDB.create(settings, logger, currentStorateDB);
+        } else if(id === DaqStoreTypeEnum.TDengine){
+            daqDB[id] = TDengine.create(settings, logger, currentStorateDB);
+        } else {
+            daqDB[id] = SqliteDB.create(settings, logger, id, currentStorateDB);
+        }
+    }
+    return daqDB[id].setCall(fncgetprop);
+    // return daqnodes[id].addDaqValue;
+}
+
+function getNodeValues(tagid, fromts, tots) {
+    return new Promise(function (resolve, reject) {
+        var daqnode = _getDaqNode(tagid);
+        if (daqnode) {
+            resolve(daqnode.getDaqValue(tagid, fromts, tots));
+        } else {
+            resolve([]);
+        }
+    });
+}
+
+/**
+ * Return tags values, 
+ * if with options then return function array [{DD/MM/YYYY mm:HH, ...values}]
+ * else for chart object {tagId} [{Date, value}]
+ * @param {*} tagsid 
+ * @param {*} fromts 
+ * @param {*} tots 
+ * @param {*} options 
+ * @returns 
+ */
+function getNodesValues(tagsid, fromts, tots, options) {
+    return new Promise(async function (resolve, reject) {
+        try {
+            var dbfncs = [];
+            for (let i = 0; i < tagsid.length; i++) {
+                dbfncs.push(await getNodeValues(tagsid[i], fromts, tots));
+            }
+            Promise.all(dbfncs).then(values => {
+                if (!values || values.length < 1) {    // (0)[]
+                    resolve(['', ...tagsid.map(col => '')]);
+                } else if (options) {
+                    let calcValues = [];
+                    for (let idx = 0 ; idx < values.length; idx++) {
+                        if (options.functions[idx]) {
+                            calcValues.push(calculator.getFunctionValues(values[idx], fromts, tots, options.functions[idx], options.interval, options.formats[idx]));
+                        } else {
+                            calcValues.push(calculator.getFunctionValues(values[idx], fromts, tots));
+                        }
+                    }
+                    let keys = Object.keys(calcValues[0]).map(ts => Number(ts));
+                    let mergeValues = Object.keys(calcValues[0]).map(ts => [utils.getFormatDate(new Date(Number(ts))), _getValue(calcValues[0][ts])]);
+                    for (let x = 1; x < calcValues.length; x++) {
+                        let y = 0;
+                        keys.forEach(k => {
+                            mergeValues[y++].push(_getValue(calcValues[x][k]));
+                        });
+                    }
+                    resolve(mergeValues);
+                } else {
+                    var result = {};
+                    for (let i = 0; i < tagsid.length; i++) {
+                        result[tagsid[i]] = values[i].map(v => { return { x: new Date(v.dt), y: v.value} });
+                        result[tagsid[i]].push({ x: new Date(tots), y: null});
+                        result[tagsid[i]].unshift({ x: new Date(fromts), y: null});
+                    }
+                    resolve(result);
+                }
+            }, reason => {
+                reject(reason);
+            });
+        } catch (err) {
+            reject(['ERR', ...tagsid.map(col => 'ERR')]);
+        }
+    });
+}
+
+function checkRetention() {
+    return new Promise(async function (resolve, reject) {
+        if (settings.daqstore && _getDbType() === DaqStoreTypeEnum.SQlite && settings.daqstore.retention !== 'none') {
+            try {
+                SqliteDB.checkRetention(utils.getRetentionLimit(settings.daqstore.retention), settings.dbDir, 
+                (fileDeleted) => {
+                    logger.info(`daqstorage.checkRetention file ${fileDeleted} removed`);
+                },
+                (err) => {
+                    logger.error(`daqstorage.checkRetention remove file failed! ${err}`);
+                });
+            } catch (err) {
+                logger.error(err);
+            }
+        }
+        logger.info(`daqstorage.checkRetention processed`);
+        resolve();
+    });
+}
+
+function getCurrentStorageFnc() {
+    return currentStorateDB.getValuesByDeviceId;
+}
+
+function _getDaqNode(tagid) {
+    var nodes = Object.values(daqDB);
+    for (var i = 0; i < nodes.length; i++) {
+        if (nodes[i].getDaqMap(tagid)[tagid]) {
+            return nodes[i];
+        }
+    }
+}
+
+function _getDbType() {
+    if (settings.daqstore && settings.daqstore.type) {
+        return settings.daqstore.type;
+    }
+    return DaqStoreTypeEnum.SQlite;
+}
+
+var DaqStoreTypeEnum = {
+    SQlite: 'SQlite',
+    influxDB: 'influxDB',
+    influxDB18: 'influxDB18',
+    TDengine: 'TDengine',
+}
+
+function _getValue(value) {
+    if (value == Number.MAX_VALUE || value === utils.SMALLEST_NEGATIVE_INTEGER || value == null) {
+        return '';
+    }
+    return value.toString();
+}
+
+module.exports = {
+    init: init,
+    reset: reset,
+    addDaqNode: addDaqNode,
+    getNodeValues: getNodeValues,
+    getNodesValues: getNodesValues,
+    checkRetention: checkRetention,
+    getCurrentStorageFnc: getCurrentStorageFnc,
+};

package/runtime/storage/influxdb/index.js

@@ -0,0 +1,240 @@
+/**
+ * Module to manage the DAQ with influxdb
+ */
+
+"use strict";
+
+var utils = require('../../utils');
+const influx = require('influx');
+const { URL } = require('url');
+var { InfluxDB, Point, flux } = require('@influxdata/influxdb-client');
+
+const VERSION_18_FLUX = '1.8-flux';
+const VERSION_20 = '2.0';
+
+function Influx(_settings, _log, _currentStorate) {
+
+    var settings = _settings;               // Application settings
+    var logger = _log;                      // Application logger
+    var currentStorage = _currentStorate    // Database to set the last value (current)
+    var status = InfluxDBStatusEnum.CLOSE;
+
+
+    var influxError = { error: null, timestamp: 0 };
+    var influxdbVersion = VERSION_20;
+    var client = null;
+    var clientOptions = null
+    var writeApi = null
+    var queryApi = null
+
+    this.init = function () {
+
+        if (settings.daqstore.type === 'influxDB18') {
+            influxdbVersion = VERSION_18_FLUX;
+        }
+        if (settings.daqstore.credentials && influxdbVersion === VERSION_20) {
+            const token = settings.daqstore.credentials.token;
+            clientOptions = {
+                url: settings.daqstore.url,
+                // rejectUnauthorized: n.rejectUnauthorized,
+                token
+            }
+            try {
+                client = new InfluxDB(clientOptions);
+                writeApi = client.getWriteApi(settings.daqstore.organization, settings.daqstore.bucket, 's');
+                queryApi = client.getQueryApi(settings.daqstore.organization);
+                status = InfluxDBStatusEnum.OPEN;    
+            } catch (error) {
+                logger.error('influxdb-init failed! ' + error.message); 
+            }
+        } else if (influxdbVersion === VERSION_18_FLUX) {
+            try {
+                const parsedUrl = new URL(settings.daqstore.url);
+                const [host] = parsedUrl.host.split(':');
+                const [protocol] = parsedUrl.protocol.split(':');
+                clientOptions = {
+                    host: host,
+                    port: parsedUrl.port || 8086,
+                    protocol: protocol || 'http',
+                    database: settings.daqstore.database,
+                    username: settings.daqstore.credentials.username,
+                    password: settings.daqstore.credentials.password,
+                }
+            } catch (error) {
+                logger.error(`influxdb-init failed! ${error}`);
+                reject(error);
+            }
+            client = new influx.InfluxDB(clientOptions);
+            status = InfluxDBStatusEnum.OPEN;
+        }
+    }
+
+    this.close = function () {
+        try {
+            status = InfluxDBStatusEnum.CLOSE;
+            client.close().then(() => {
+                logger.info('influxdb-close FINISHED');
+            })
+            .catch((e) => {
+                logger.error(`influxdb-close failed! ${e}`);
+            });
+        } catch (error) {
+            logger.error(`influxdb-close failed! ${error}`);
+        }
+    }
+
+    this.setCall = function (_fncGetProp) {
+        fncGetTagProp = _fncGetProp;
+        return this.addDaqValues;
+    }
+    var fncGetTagProp = null;
+
+    this.addDaqValue = function (tagId, tagValue) {
+        logger.error('influxdb-addDaqValue Not supported!');
+    }
+
+    this.addDaqValues = function (tagsValues, deviceName, deviceId) {
+        var dataToWrite = [];
+        var dataToRestore = [];
+        for (var tagid in tagsValues) {
+            let tag = tagsValues[tagid];
+            if (!tag.daq || utils.isNullOrUndefined(tag.value) || Number.isNaN(tag.value)) {
+                if (tag.daq.restored) {
+                    dataToRestore.push({id: tag.id, deviceId: deviceId, value: tag.value});
+                }
+                if (!tag.daq.enabled) {
+                    continue;
+                }
+            }
+            if (influxdbVersion === VERSION_18_FLUX) {
+                const tags = {
+                    id: tag.id,
+                    devicename: deviceName
+                }
+                const fields = {
+                    value: utils.isBoolean(tag.value) ? tag.value * 1 : tag.value
+                }
+                dataToWrite.push({
+                    measurement: tag.id,
+                    tags,
+                    fields,
+                    timestamp: new Date(tag.timestamp || new Date().getTime())
+                });
+            } else {
+                const point = new Point(tag.id)
+                    .tag('id', tag.id)
+                    .tag('name', tag.name || tag.tagref.name)
+                    .tag('type', tag.type)
+                    .timestamp(new Date(tag.timestamp || new Date().getTime()));
+                if (deviceName) {
+                    point.tag('device', deviceName);
+                }
+                if (utils.isBoolean(tag.value)) {
+                    point.booleanField('value', tag.value);
+                } else {
+                    point.floatField('value', tag.value)
+                }
+                dataToWrite.push(point);
+            }
+        }
+        if (dataToWrite.length) {
+            writePoints(dataToWrite);
+        }
+        if (dataToRestore.length && currentStorage) {
+            currentStorage.setValues(dataToRestore);
+        }
+    }
+
+    this.getDaqMap = function (tagid) {
+        var dummy = {};
+        dummy[tagid] = true;
+        return dummy;
+    }
+
+    this.getDaqValue = function (tagid, fromts, tots) {
+        return new Promise(function (resolve, reject) {
+            try {
+                if (influxdbVersion === VERSION_18_FLUX) {
+                    fromts *= 1000000;
+                    tots *= 1000000;
+                    const query = `SELECT * FROM "${tagid}" WHERE time >= ${fromts} AND time <= ${tots}`;
+                    client.query(query).then((result) => {
+                        resolve(result.map(row => { 
+                            return {
+                                dt: new Date(row.time).getTime(),
+                                value: row.value
+                            }
+                        }));
+                    })
+                    .catch((error) => {
+                        logger.error(`influxdb-getDaqValue failed! ${error}`);
+                        reject(error);
+                    });
+                } else {
+                    const query = flux`from(bucket: "${settings.daqstore.bucket}") |> range(start: ${new Date(fromts)}, stop: ${new Date(tots)}) |> filter(fn: (r) => r.id == "${tagid}")`;
+                    var result = [];
+                    queryApi.queryRows(query, {
+                        next(row, tableMeta) {
+                            const o = tableMeta.toObject(row);
+                            result = result.concat({ dt: new Date(o._time).getTime(), value: o._value });
+                        },
+                        error(error) {
+                            logger.error(`influxdb-getDaqValue failed! ${error}`);
+                            reject(error);
+                        },
+                        complete() {
+                            resolve(result);
+                        }
+                    });
+                }
+
+            } catch (error) {
+                logger.error(`influxdb-getDaqValue failed! ${error}`);
+                reject(error);
+            }
+        });
+    }
+
+    function writePoints(points) {
+        try {
+            if (influxdbVersion === VERSION_18_FLUX) {
+                client.writePoints(points)
+                .catch((error) => {
+                    logger.error(`influxdb-writePoints failed! ${error}`);
+                });
+            } else {
+                writeApi.writePoints(points);
+                writeApi.flush(true).then(() => {
+                    // reset last error ;
+                }).catch(err => {
+                    logger.error(`influxdb-writePoints error! ${err}`);
+                    setError(err);
+                });
+            }
+            if (currentStorage) {
+                currentStorage.setValues(points);
+            }
+        } catch (error) {
+            logger.error(`influxdb-writePoints failed! ${error}`);
+        }
+    }
+
+    function setError(error) {
+        influxError.error = error;
+        influxError.timestamp = new Date().getTime();
+    }
+
+    this.init();
+}
+
+
+module.exports = {
+    create: function (data, logger, currentStorate) {
+        return new Influx(data, logger, currentStorate);
+    }
+};
+
+var InfluxDBStatusEnum = {
+    OPEN: 'open',
+    CLOSE: 'close',
+}