iotagent-node-lib 3.1.0 → 3.3.0

package/lib/model/dbConn.js

@@ -106,13 +106,7 @@ function init(host, db, port, options, callback) {
     }
 
     function connectionAttempt(url, options, callback) {
-        logger.info(
-            context,
-            'Attempting to connect to MongoDB instance with url %j and options %j. Attempt %d',
-            url,
-            options,
-            retries
-        );
+        logger.info(context, 'Attempting to connect to MongoDB instance with url %j. Attempt %d', url, retries);
         // FIXME: useNewUrlParser is no longer used in underlying mongodb driver 4.x
         // (see https://github.com/mongodb/node-mongodb-native/blob/HEAD/etc/notes/CHANGES_4.0.0.md)
         // but not sure if current mongoose version is still using mongodb 3.x internally

package/lib/plugins/bidirectionalData.js

@@ -75,9 +75,7 @@ function extractVariables(item) {
 
     function extractFromExpression(value) {
         if (value.expression) {
-            return value.expression.match(/@[a-zA-Z]+/g).map(function (item) {
-                return item.substr(1);
-            });
+            return parser.extractVariables(value.expression);
         }
         return [];
     }
@@ -157,9 +155,7 @@ function getReverseTransformations(list, values, callback) {
     let transformations = [];
 
     function getVariable(expression) {
-        return expression.match(/@[a-zA-Z]+/g).map(function (item) {
-            return item.substr(1);
-        });
+        return parser.extractVariables(expression);
     }
 
     if (list && list.length > 0) {
@@ -191,7 +187,7 @@ function getReverseTransformations(list, values, callback) {
  */
 function processTransformationsNGSIv2(values, typeInformation, transformations, callback) {
     let cleanedExpression;
-    const ctx = parser.extractContext(values, typeInformation);
+    const ctx = parser.extractContext(values);
     let resultTransformations = [];
 
     for (let j = 0; j < 2; j++) {
@@ -201,10 +197,7 @@ function processTransformationsNGSIv2(values, typeInformation, transformations,
     }
 
     for (let i = 0; i < resultTransformations.length; i++) {
-        cleanedExpression = resultTransformations[i].expression.substr(
-            2,
-            resultTransformations[i].expression.length - 3
-        );
+        cleanedExpression = resultTransformations[i].expression;
 
         values.push({
             name: resultTransformations[i].object_id,
@@ -252,20 +245,12 @@ function processTransformationsNGSILD(values, typeInformation, transformations,
     // This is the direct equivalent of the NGSI-v2 function
     if (!_.isEmpty(defaultValues)) {
         for (let i = 0; i < resultTransformations.length; i++) {
-            cleanedExpression = resultTransformations[i].expression.substr(
-                2,
-                resultTransformations[i].expression.length - 3
-            );
+            cleanedExpression = resultTransformations[i].expression;
             values.push({
                 name: resultTransformations[i].object_id,
                 type: resultTransformations[i].type,
                 metadata: {},
-                value: parser.parse(
-                    cleanedExpression,
-                    parser.extractContext(defaultValues, typeInformation),
-                    'String',
-                    typeInformation
-                )
+                value: parser.parse(cleanedExpression, parser.extractContext(defaultValues), 'String', typeInformation)
             });
         }
     }
@@ -274,10 +259,7 @@ function processTransformationsNGSILD(values, typeInformation, transformations,
     // There is no equivalent with NGSI-v2
     _.keys(datasetIds).forEach((datasetId) => {
         for (let i = 0; i < resultTransformations.length; i++) {
-            cleanedExpression = resultTransformations[i].expression.substr(
-                2,
-                resultTransformations[i].expression.length - 3
-            );
+            cleanedExpression = resultTransformations[i].expression;
             values.push({
                 name: resultTransformations[i].object_id,
                 type: resultTransformations[i].type,
@@ -285,7 +267,7 @@ function processTransformationsNGSILD(values, typeInformation, transformations,
                 metadata: {},
                 value: parser.parse(
                     cleanedExpression,
-                    parser.extractContext(datasetIds[datasetId], typeInformation),
+                    parser.extractContext(datasetIds[datasetId]),
                     'String',
                     typeInformation
                 )

package/lib/plugins/expressionPlugin.js

@@ -24,9 +24,7 @@
  * Modified by: Federico M. Facca - Martel Innovate
  */
 
-const legacyParser = require('./expressionParser');
 const jexlParser = require('./jexlParser');
-const config = require('../commonConfig');
 /* eslint-disable no-unused-vars */
 const logger = require('logops');
 const errors = require('../errors');
@@ -44,53 +42,23 @@ function setJEXLTransforms(transformationMap) {
 }
 
 function applyExpression(expression, context, typeInformation) {
-    let parser = jexlParser;
-    if (!checkJexl(typeInformation)) {
-        parser = legacyParser;
-    }
-    return parser.applyExpression(expression, context, typeInformation);
+    return jexlParser.applyExpression(expression, context, typeInformation);
 }
 
-function extractContext(attributeList, typeInformation) {
-    let parser = jexlParser;
-    if (!checkJexl(typeInformation)) {
-        parser = legacyParser;
-    }
-    return parser.extractContext(attributeList);
+function extractContext(attributeList) {
+    return jexlParser.extractContext(attributeList);
 }
 
-function parse(expression, context, type, typeInformation) {
-    if (!checkJexl(typeInformation)) {
-        return legacyParser.parse(expression, context, type);
-    }
+function parse(expression, context) {
     return jexlParser.parse(expression, context);
 }
 
 function contextAvailable(expression, context, typeInformation) {
-    let parser = jexlParser;
-    if (!checkJexl(typeInformation)) {
-        parser = legacyParser;
-    }
-    return parser.contextAvailable(expression, context);
+    return jexlParser.contextAvailable(expression, context);
 }
 
-function checkJexl(typeInformation) {
-    if (
-        config.getConfig().defaultExpressionLanguage === 'jexl' &&
-        typeInformation.expressionLanguage &&
-        typeInformation.expressionLanguage !== 'legacy'
-    ) {
-        return true;
-    } else if (config.getConfig().defaultExpressionLanguage === 'jexl' && !typeInformation.expressionLanguage) {
-        return true;
-    } else if (
-        config.getConfig().defaultExpressionLanguage === 'legacy' &&
-        typeInformation.expressionLanguage &&
-        typeInformation.expressionLanguage === 'jexl'
-    ) {
-        return true;
-    }
-    return false;
+function extractVariables(expression) {
+    return jexlParser.extractVariables(expression);
 }
 
 exports.parse = parse;
@@ -98,4 +66,4 @@ exports.setJEXLTransforms = setJEXLTransforms;
 exports.applyExpression = applyExpression;
 exports.extractContext = extractContext;
 exports.contextAvailable = contextAvailable;
-exports.checkJexl = checkJexl;
+exports.extractVariables = extractVariables;

package/lib/plugins/jexlParser.js

@@ -28,6 +28,8 @@
 /* eslint-disable no-unused-vars */
 
 const jexl = require('jexl');
+const grammar = require('jexl/dist/grammar').getGrammar();
+const Lexer = require('jexl/dist/Lexer');
 const errors = require('../errors');
 const logger = require('logops');
 const fillService = require('../services/common/domain').fillService;
@@ -62,6 +64,31 @@ function parse(expression, context, callback) {
     }
 }
 
+function extractVariables(expression) {
+    const inst = new Lexer(grammar);
+    let variables = [];
+
+    try {
+        const tokens = inst.tokenize(expression);
+
+        // Keep only root attributes, removing the dot and sub-attributes. For example, if we have
+        // a.0.b, a.1.b and a.2.b, we will only keep a
+        // Additionaly, it will remove the function calls, since they are also detected as identifiers
+        variables = tokens.filter(function (token, index, array) {
+            return (
+                (token.type === ' ' && array[index - 1].type !== 'dot') ||
+                (token.type === 'identifier' && array[index + 1] && array[index + 1].type !== 'openParen')
+            );
+        });
+
+        // Return only array of values
+        return variables.map((a) => a.value);
+    } catch (e) {
+        logger.warn(logContext, 'Wrong expression found "[%j]" error: "[%s]", it will be ignored', expression, e);
+        return false;
+    }
+}
+
 function extractContext(attributeList) {
     const context = {};
     let value;
@@ -162,6 +189,7 @@ function setTransforms(configMap) {
     logger.info(logContext, message);
 }
 
+exports.extractVariables = extractVariables;
 exports.extractContext = extractContext;
 exports.contextAvailable = contextAvailable;
 exports.applyExpression = applyExpression;

package/lib/services/commands/commandService.js

@@ -58,7 +58,7 @@ function addCommandDevice(service, subservice, device, command, callback) {
         // The context for the JEXL expression should be the ID, TYPE, S, SS
         let attrList = pluginUtils.getIdTypeServSubServiceFromDevice(device);
         attrList = device.staticAttributes ? attrList.concat(device.staticAttributes) : attrList.concat([]);
-        let ctxt = parser.extractContext(attrList, device);
+        let ctxt = parser.extractContext(attrList);
         logger.debug(context, 'attrList [%j] for device %j', attrList, device);
         // expression result will be the full command payload
         let cmdValueRes = null;

package/lib/services/common/iotManagerService.js

@@ -58,7 +58,6 @@ function register(callback) {
             timestamp: service.timestamp,
             autoprovision: service.autoprovision,
             explicitAttrs: service.explicitAttrs,
-            expressionLanguage: service.expressionLanguage,
             defaultEntityNameConjunction: service.defaultEntityNameConjunction,
             ngsiVersion: service.ngsiVersion,
             entityNameExp: service.entityNameExp

package/lib/services/devices/deviceRegistryMongoDB.js

@@ -55,7 +55,6 @@ const attributeList = [
     'polling',
     'timestamp',
     'explicitAttrs',
-    'expressionLanguage',
     'ngsiVersion'
 ];
 
@@ -176,15 +175,15 @@ function listDevices(type, service, subservice, limit, offset, callback) {
         query.skip(parseInt(offset, 10));
     }
 
-    async.series([query.exec.bind(query), Device.model.countDocuments.bind(Device.model, condition)], function (
-        error,
-        results
-    ) {
-        callback(error, {
-            count: results[1],
-            devices: results[0]
-        });
-    });
+    async.series(
+        [query.exec.bind(query), Device.model.countDocuments.bind(Device.model, condition)],
+        function (error, results) {
+            callback(error, {
+                count: results[1],
+                devices: results[0]
+            });
+        }
+    );
 }
 
 function findOneInMongoDB(queryParams, id, callback) {
@@ -303,6 +302,7 @@ function update(device, callback) {
             data.registrationId = device.registrationId;
             data.explicitAttrs = device.explicitAttrs;
             data.ngsiVersion = device.ngsiVersion;
+            data.timestamp = device.timestamp;
 
             /* eslint-disable-next-line  new-cap */
             const deviceObj = new Device.model(data);

package/lib/services/devices/deviceService.js

@@ -72,7 +72,12 @@ function init() {
  * @param {Object} newDevice        Device object that will be stored in the database.
  */
 function createInitialEntity(deviceData, newDevice, callback) {
-    deviceHandler.createInitialEntity(deviceData, newDevice, callback);
+    if (config.getConfig().appendMode === false || config.ngsiVersion() === 'ld' || deviceData.ngsiVersion === 'ld') {
+        deviceHandler.createInitialEntity(deviceData, newDevice, callback);
+    } else {
+        logger.debug(context, 'Skip create initial entity due appendMode is false or ngsiLD.');
+        callback(null, newDevice);
+    }
 }
 
 /**
@@ -178,13 +183,12 @@ function mergeDeviceWithConfiguration(fields, defaults, deviceData, configuratio
     if (configuration && configuration.ngsiVersion) {
         deviceData.ngsiVersion = configuration.ngsiVersion;
     }
-    if (configuration && configuration.expressionLanguage && deviceData.expressionLanguage === undefined) {
-        deviceData.expressionLanguage = configuration.expressionLanguage;
-    }
     if (configuration && configuration.explicitAttrs !== undefined && deviceData.explicitAttrs === undefined) {
         deviceData.explicitAttrs = configuration.explicitAttrs;
     }
-
+    if (configuration && configuration.entityNameExp !== undefined) {
+        deviceData.entityNameExp = configuration.entityNameExp;
+    }
     logger.debug(context, 'deviceData after merge with conf: %j', deviceData);
     callback(null, deviceData);
 }
@@ -265,7 +269,7 @@ function registerDevice(deviceObj, callback) {
     function prepareDeviceData(deviceObj, configuration, callback) {
         const deviceData = _.clone(deviceObj);
         let selectedConfiguration;
-
+        logger.debug(context, 'Prepare device data:\n%s', JSON.stringify(deviceData, null, 4));
         if (!deviceData.type) {
             if (configuration && configuration.type) {
                 deviceData.type = configuration.type;
@@ -294,7 +298,8 @@ function registerDevice(deviceObj, callback) {
                 // Add device ID, TYPE, S, SS to the context for the JEXL expression
                 let attrList = pluginUtils.getIdTypeServSubServiceFromDevice(deviceData);
                 attrList = deviceData.staticAttributes ? attrList.concat(deviceData.staticAttributes) : attrList;
-                let ctxt = expressionPlugin.extractContext(attrList, deviceData);
+                attrList = configuration.staticAttributes ? attrList.concat(configuration.staticAttributes) : attrList;
+                let ctxt = expressionPlugin.extractContext(attrList);
                 try {
                     entityName = expressionPlugin.applyExpression(configuration.entityNameExp, ctxt, deviceData);
                 } catch (e) {
@@ -361,9 +366,9 @@ function registerDevice(deviceObj, callback) {
                     deviceObj.service = deviceData.service;
                     deviceObj.subservice = deviceData.subservice;
                     deviceObj.type = deviceData.type;
-                    deviceObj.staticAttributes = deviceData.staticAttributes;
-                    deviceObj.commands = deviceData.commands;
-                    deviceObj.lazy = deviceData.lazy;
+                    deviceObj.staticAttributes = deviceObj.staticAttributes ? deviceObj.staticAttributes : [];
+                    deviceObj.commands = deviceObj.commands ? deviceObj.commands : [];
+                    deviceObj.lazy = deviceObj.lazy ? deviceObj.lazy : [];
                     if ('timestamp' in deviceData && deviceData.timestamp !== undefined) {
                         deviceObj.timestamp = deviceData.timestamp;
                     }
@@ -373,6 +378,7 @@ function registerDevice(deviceObj, callback) {
                     if ('apikey' in deviceData && deviceData.apikey !== undefined) {
                         deviceObj.apikey = deviceData.apikey;
                     }
+                    logger.debug(context, 'Storing device :\n%s', JSON.stringify(deviceObj, null, 4));
                     config.getRegistry().store(deviceObj, callback);
                 }
             }
@@ -617,19 +623,9 @@ function findOrCreate(deviceId, group, callback) {
             if (config.getConfig().iotManager && config.getConfig().iotManager.protocol) {
                 newDevice.protocol = config.getConfig().iotManager.protocol;
             }
-
-            if ('timestamp' in group && group.timestamp !== undefined) {
-                newDevice.timestamp = group.timestamp;
-            }
             if ('ngsiVersion' in group && group.ngsiVersion !== undefined) {
                 newDevice.ngsiVersion = group.ngsiVersion;
             }
-            if ('explicitAttrs' in group && group.explicitAttrs !== undefined) {
-                newDevice.explicitAttrs = group.explicitAttrs;
-            }
-            if ('expressionLanguage' in group && group.expressionLanguage !== undefined) {
-                newDevice.expressionLanguage = group.expressionLanguage;
-            }
             if (
                 (!('apikey' in newDevice) || newDevice.apikey === undefined) &&
                 'apikey' in group &&

package/lib/services/devices/devices-NGSI-v2.js

@@ -240,7 +240,7 @@ function createInitialEntityNgsi2(deviceData, newDevice, callback) {
         }
     }
 
-    logger.debug(context, 'deviceData: %j', deviceData);
+    logger.debug(context, 'Creating initial entity with deviceData: %j', deviceData);
     if (
         ('timestamp' in deviceData && deviceData.timestamp !== undefined
             ? deviceData.timestamp
@@ -356,11 +356,8 @@ function updateRegisterDeviceNgsi2(deviceObj, entityInfoUpdated, callback) {
             if ('explicitAttrs' in newDevice && newDevice.explicitAttrs !== undefined) {
                 oldDevice.explicitAttrs = newDevice.explicitAttrs;
             }
-            if ('expressionLanguage' in newDevice && newDevice.expressionLanguage !== undefined) {
-                oldDevice.expressionLanguage = newDevice.expressionLanguage;
-            }
             if ('apikey' in newDevice && newDevice.apikey !== undefined) {
-                oldDevice.explicitAttrs = newDevice.apikey;
+                oldDevice.apikey = newDevice.apikey;
             }
             oldDevice.endpoint = newDevice.endpoint || oldDevice.endpoint;
 

package/lib/services/ngsi/entities-NGSI-LD.js

@@ -410,16 +410,6 @@ function addLinkedEntities(typeInformation, json) {
     });
 }
 
-/**
- * Determines if a value is of type float
- *
- * @param      {String}   value       Value to be analyzed
- * @return     {boolean}              True if float, False otherwise.
- */
-function isFloat(value) {
-    return !isNaN(value) && value.toString().indexOf('.') !== -1;
-}
-
 /**
  * Makes an update in the Device's entity in the context broker, with the values given in the 'attributes' array.
  * This array should comply to the NGSI-LD's attribute format.
@@ -449,10 +439,10 @@ function sendUpdateValueNgsiLD(entityName, attributes, typeInformation, token, c
         payload[0].type = typeInformation.type;
     }
 
-    if (config.getConfig().appendMode === true) {
-        payload.actionType = 'append';
-    } else {
+    if (config.getConfig().appendMode === false) {
         payload.actionType = 'update';
+    } else {
+        payload.actionType = 'append';
     }
 
     const options = NGSIUtils.createRequestObject(url, typeInformation, token);
@@ -535,7 +525,7 @@ function sendUpdateValueNgsiLD(entityName, attributes, typeInformation, token, c
                 }
             }
             // This context is just to calculate explicitAttrs when is an expression
-            let ctxt = expressionPlugin.extractContext(attributesCtxt.concat(idTypeSSSList), typeInformation);
+            let ctxt = expressionPlugin.extractContext(attributesCtxt.concat(idTypeSSSList));
             // typeInformation.active attrs with expressions expanded by current ctxt
             if (typeInformation.active) {
                 typeInformation.active.forEach(function (att) {
@@ -547,10 +537,7 @@ function sendUpdateValueNgsiLD(entityName, attributes, typeInformation, token, c
                                 type: att.type
                             };
                             attributesCtxt.push(expandedAttr);
-                            ctxt = expressionPlugin.extractContext(
-                                attributesCtxt.concat(idTypeSSSList),
-                                typeInformation
-                            );
+                            ctxt = expressionPlugin.extractContext(attributesCtxt.concat(idTypeSSSList));
                         }
                     }
                 });
@@ -689,7 +676,7 @@ function sendUpdateValueNgsiLD(entityName, attributes, typeInformation, token, c
         });
     }
     attributesCtxt = attributesCtxt.concat(idTypeSSSList);
-    const ctxt = expressionPlugin.extractContext(attributesCtxt, typeInformation);
+    const ctxt = expressionPlugin.extractContext(attributesCtxt);
     logger.debug(context, 'sendUpdateValueNgsiLD \n initial ctxt %j ', ctxt);
 
     // Sort currentEntity to get first attrs without expressions (checking attrs in typeInformation.active)
@@ -791,47 +778,12 @@ function sendUpdateValueNgsiLD(entityName, attributes, typeInformation, token, c
                     }
                 } catch (e) {
                     logger.error(context, 'sendUpdateValueNgsiLD \n apply expression exception %j \n', e);
-                    if (!expressionPlugin.checkJexl(typeInformation)) {
-                        return callback(e); // just throw error with legacy parser for backward compatiblity
-                    } else {
-                        res = ctxt[attr.name]; // TBD: add reference to test
-                    }
-                }
-                if (!expressionPlugin.checkJexl(typeInformation)) {
-                    // legacy expression plugin: Involves some legacy checks performed by applierExpression
-                    if (
-                        res &&
-                        res !== 'undefined' &&
-                        res !== 'null' &&
-                        (typeof res === 'string' || res instanceof String) &&
-                        !res.includes('NaN')
-                    ) {
-                        newAttr.value = res;
-                        if (newAttr.type === 'Number' && isFloat(newAttr.value)) {
-                            newAttr.value = Number.parseFloat(newAttr.value);
-                        } else if (newAttr.type === 'Number' && !Number.isNaN(Number.parseInt(newAttr.value))) {
-                            newAttr.value = Number.parseInt(newAttr.value);
-                        } else if (newAttr.type === 'Boolean') {
-                            newAttr.value = newAttr.value === 'true' || newAttr.value === '1';
-                        } else if (newAttr.type === 'None') {
-                            newAttr.value = null;
-                        }
-                    } else if (isNaN(res) || res === 'undefined' || res === 'null') {
-                        logger.debug(
-                            context,
-                            'sendUpdateValueNgsiLD \n apply expression result: isNaN || undefined || null'
-                        );
-                        if (res === 'null' && newAttr.type === 'None') {
-                            newAttr.value = null;
-                        } else {
-                            delete payload[0][j]; // remove measure attr
-                            attr = undefined; // stop process attr
-                        }
-                    }
-                } else {
-                    // jexl expression plugin
-                    newAttr.value = res;
+                    res = ctxt[attr.name]; // TBD: add reference to test
                 }
+
+                // jexl expression plugin
+                newAttr.value = res;
+
                 logger.debug(
                     context,
                     'sendUpdateValueNgsiLD \n apply expression result %j \n newAttr %j',
@@ -1048,5 +1000,9 @@ function sendUpdateValueNgsiLD(entityName, attributes, typeInformation, token, c
 
 exports.convertAttrNGSILD = convertAttrNGSILD;
 exports.formatAsNGSILD = formatAsNGSILD;
-exports.sendUpdateValue = sendUpdateValueNgsiLD;
+exports.sendUpdateValue = function (entityName, attributes, typeInformation, token, callback) {
+    NGSIUtils.applyMiddlewares(NGSIUtils.updateMiddleware, attributes, typeInformation, () => {
+        return sendUpdateValueNgsiLD(entityName, attributes, typeInformation, token, callback);
+    });
+};
 exports.sendQueryValue = sendQueryValueNgsiLD;