iotagent-node-lib 2.18.0 → 2.21.0

package/lib/fiware-iotagent-lib.js

@@ -37,6 +37,7 @@ const iotManager = require('./services/common/iotManagerService');
 const contextServer = require('./services/northBound/northboundServer');
 const errors = require('./errors');
 const constants = require('./constants');
+const request = require('./request-shim');
 const logger = require('logops');
 const config = require('./commonConfig');
 const cluster = require('cluster');
@@ -313,6 +314,7 @@ exports.listDevices = deviceService.listDevices;
 exports.getDevice = deviceService.getDevice;
 exports.getDeviceSilently = deviceService.getDeviceSilently;
 exports.getDeviceByName = deviceService.getDeviceByName;
+exports.getDeviceByNameAndType = deviceService.getDeviceByNameAndType;
 exports.getDevicesByAttribute = deviceService.getDevicesByAttribute;
 exports.mergeDeviceWithConfiguration = deviceService.mergeDeviceWithConfiguration;
 exports.retrieveDevice = deviceService.retrieveDevice;
@@ -325,6 +327,7 @@ exports.setDataQueryHandler = contextServer.setQueryHandler;
 exports.setConfigurationHandler = contextServer.setConfigurationHandler;
 exports.setRemoveConfigurationHandler = contextServer.setRemoveConfigurationHandler;
 exports.setProvisioningHandler = contextServer.setProvisioningHandler;
+exports.setUpdatingHandler = contextServer.setUpdatingHandler;
 exports.setRemoveDeviceHandler = contextServer.setRemoveDeviceHandler;
 exports.setNotificationHandler = contextServer.setNotificationHandler;
 exports.addUpdateMiddleware = ngsi.addUpdateMiddleware;
@@ -353,8 +356,6 @@ exports.fillService = domainUtils.fillService;
 
 exports.middlewares = middlewares;
 
-exports.commandLine = require('./command/commandLine');
-
 exports.dataPlugins = {
     compressTimestamp: require('./plugins/compressTimestamp'),
     attributeAlias: require('./plugins/attributeAlias'),
@@ -372,3 +373,4 @@ exports.constants = constants;
 exports.logModule = logger;
 exports.configModule = config;
 exports.startServer = startServer;
+exports.request = request;

package/lib/jexlTranformsMap.js

@@ -57,7 +57,17 @@ const map = {
     bitwisemask: (i, mask, op, shf) =>
         (op === '&' ? parseInt(i) & mask : op === '|' ? parseInt(i) | mask : op === '^' ? parseInt(i) ^ mask : i) >>
         shf,
-    slice: (arr, init, end) => arr.slice(init, end)
+    slice: (arr, init, end) => arr.slice(init, end),
+    addset: (arr, x) => {
+        return Array.from(new Set(arr).add(x));
+    },
+    removeset: (arr, x) => {
+        let s = new Set(arr);
+        s.delete(x);
+        return Array.from(s);
+    },
+    touppercase: (val) => String(val).toUpperCase(),
+    tolowercase: (val) => String(val).toLowerCase()
 };
 
 exports.map = map;

package/lib/model/Device.js

@@ -23,6 +23,9 @@
 
 const mongoose = require('mongoose');
 const Schema = mongoose.Schema;
+const Group = require('./Group');
+
+mongoose.Schema.Types.ExplicitAttrsType = Group.ExplicitAttrsType;
 
 const Device = new Schema({
     id: String,
@@ -49,7 +52,7 @@ const Device = new Schema({
     internalAttributes: Object,
     autoprovision: Boolean,
     expressionLanguage: String,
-    explicitAttrs: Boolean,
+    explicitAttrs: Group.ExplicitAttrsType,
     ngsiVersion: String
 });
 

package/lib/model/Group.js

@@ -24,6 +24,23 @@
 const mongoose = require('mongoose');
 const Schema = mongoose.Schema;
 
+class ExplicitAttrsType extends mongoose.SchemaType {
+    constructor(key, options) {
+        super(key, options, 'ExplicitAttrsType');
+    }
+    // `cast()` takes a parameter that can be anything. You need to
+    // validate the provided `val` and throw a `CastError` if you
+    // can't convert it.
+    cast(val) {
+        if (!(typeof val === 'boolean' || typeof val === 'string')) {
+            throw new Error('ExplicitAttrsType: ' + val + ' is not Boolean or String');
+        }
+        return val;
+    }
+}
+
+mongoose.Schema.Types.ExplicitAttrsType = ExplicitAttrsType;
+
 const Group = new Schema({
     url: String,
     resource: String,
@@ -43,7 +60,7 @@ const Group = new Schema({
     internalAttributes: Array,
     autoprovision: Boolean,
     expressionLanguage: String,
-    explicitAttrs: Boolean,
+    explicitAttrs: ExplicitAttrsType,
     defaultEntityNameConjunction: String,
     ngsiVersion: String
 });
@@ -55,3 +72,4 @@ function load(db) {
 }
 
 module.exports.load = load;
+module.exports.ExplicitAttrsType = ExplicitAttrsType;

package/lib/plugins/expressionParser.js

@@ -28,15 +28,16 @@
 const Parser = require('jison').Parser;
 const errors = require('../errors');
 const logger = require('logops');
+const fillService = require('../services/common/domain').fillService;
 const _ = require('underscore');
-const logContext = {
+let logContext = {
     op: 'IoTAgentNGSI.Expression'
 };
 const grammar = {
     lex: {
         rules: [
             ['\\s+', '/* skip whitespace */'],
-            ['@[a-zA-Z0-9]+\\b', 'return "VARIABLE";'],
+            ['@[a-zA-Z0-9_]+\\b', 'return "VARIABLE";'],
             ['[0-9]+(?:\\.[0-9]+)?\\b', 'return "NUMBER";'],
             ['\\*', 'return "*";'],
             ['\\/', 'return "/";'],
@@ -156,6 +157,7 @@ function processExpression(context) {
 
 /* eslint-disable-next-line no-unused-vars */
 function applyExpression(expression, context, typeInformation) {
+    logContext = fillService(logContext, typeInformation);
     const expressionList = expression.match(/\$\{.*?\}/g) || [];
     const substitutions = expressionList.map(processExpression(context));
     let expressionResult = expression;
@@ -214,14 +216,14 @@ function expressionApplier(context, typeInformation) {
 function contextAvailable(expression, context) {
     let error;
     try {
-        const variablesList = expression.match(/@[a-zA-Z0-9]+/g) || [];
+        const variablesList = expression.match(/@[a-zA-Z0-9_]+/g) || [];
         const variables = variablesList.map(function (item) {
             return item.substr(1);
         });
         const keys = Object.keys(context);
         let validContext = _.difference(variables, keys).length === 0;
         if (!validContext) {
-            logger.warn(
+            logger.info(
                 logContext,
                 'For expression "[%s]" context "[%j]" does not have element to match',
                 expression,

package/lib/plugins/expressionPlugin.js

@@ -30,6 +30,8 @@ const jexlParser = require('./jexlParser');
 const config = require('../commonConfig');
 /* eslint-disable no-unused-vars */
 const logger = require('logops');
+const errors = require('../errors');
+const constants = require('../constants');
 const context = {
     op: 'IoTAgentNGSI.expressionPlugin'
 };
@@ -42,6 +44,22 @@ function setJEXLTransforms(transformationMap) {
     jexlParser.setTransforms(transformationMap);
 }
 
+function applyExpression(expression, context, typeInformation) {
+    let parser = legacyParser;
+    if (checkJexl(typeInformation)) {
+        parser = jexlParser;
+    }
+    return parser.applyExpression(expression, context, typeInformation);
+}
+
+function extractContext(attributeList, typeInformation) {
+    let parser = legacyParser;
+    if (checkJexl(typeInformation)) {
+        parser = jexlParser;
+    }
+    return parser.extractContext(attributeList);
+}
+
 function mergeAttributes(attrList1, attrList2) {
     const finalCollection = _.clone(attrList1);
     const additionalItems = [];
@@ -70,33 +88,39 @@ function mergeAttributes(attrList1, attrList2) {
     return finalCollection.concat(additionalItems);
 }
 
-function update(entity, typeInformation, callback) {
-    function checkJexl(typeInformation) {
-        if (
-            config.getConfig().defaultExpressionLanguage === 'jexl' &&
-            typeInformation.expressionLanguage &&
-            typeInformation.expressionLanguage !== 'legacy'
-        ) {
-            return true;
-        } else if (config.getConfig().defaultExpressionLanguage === 'jexl' && !typeInformation.expressionLanguage) {
-            return true;
-        } else if (
-            config.getConfig().defaultExpressionLanguage === 'legacy' &&
-            typeInformation.expressionLanguage &&
-            typeInformation.expressionLanguage === 'jexl'
-        ) {
-            return true;
-        }
-        return false;
+function checkJexl(typeInformation) {
+    if (
+        config.getConfig().defaultExpressionLanguage === 'jexl' &&
+        typeInformation.expressionLanguage &&
+        typeInformation.expressionLanguage !== 'legacy'
+    ) {
+        return true;
+    } else if (config.getConfig().defaultExpressionLanguage === 'jexl' && !typeInformation.expressionLanguage) {
+        return true;
+    } else if (
+        config.getConfig().defaultExpressionLanguage === 'legacy' &&
+        typeInformation.expressionLanguage &&
+        typeInformation.expressionLanguage === 'jexl'
+    ) {
+        return true;
     }
+    return false;
+}
 
+function update(entity, typeInformation, callback) {
     function processEntityUpdateNgsi2(attributes) {
         let parser = legacyParser;
         if (checkJexl(typeInformation)) {
             parser = jexlParser;
         }
         let expressionAttributes = [];
-        const ctx = parser.extractContext(attributes);
+        let attributesCtxt = [...attributes]; // just copy
+        if (typeInformation.static) {
+            typeInformation.static.forEach(function (att) {
+                attributesCtxt.push(att);
+            });
+        }
+        const ctx = parser.extractContext(attributesCtxt);
 
         if (typeInformation.active) {
             expressionAttributes = parser.processExpressionAttributes(typeInformation, typeInformation.active, ctx);
@@ -107,15 +131,32 @@ function update(entity, typeInformation, callback) {
     }
 
     try {
-        let attsArray = utils.extractAttributesArrayFromNgsi2Entity(entity);
-        attsArray = processEntityUpdateNgsi2(attsArray);
-        entity = utils.createNgsi2Entity(entity.id, entity.type, attsArray, true);
+        logger.debug(context, 'expressionPlugin entity %j', entity);
+        const attsArray = utils.extractAttributesArrayFromNgsi2Entity(entity);
+        // Exclude processing all attr expressions when current attr is of type 'commandStatus' or 'commandResult'
+        const attsArrayFiltered = attsArray.filter((obj) => {
+            return ![constants.COMMAND_STATUS, constants.COMMAND_RESULT].includes(obj.type);
+        });
+        const attsArrayCmd = attsArray.filter((obj) => {
+            // just attr of type 'commandStatus' or 'commandResult'
+            return [constants.COMMAND_STATUS, constants.COMMAND_RESULT].includes(obj.type);
+        });
+        let attsArrayFinal = [];
+        if (attsArrayFiltered.length > 0) {
+            attsArrayFinal = processEntityUpdateNgsi2(attsArrayFiltered);
+        }
+        attsArrayFinal = attsArrayFinal.concat(attsArrayCmd);
+        entity = utils.createNgsi2Entity(entity.id, entity.type, attsArrayFinal, true);
 
         callback(null, entity, typeInformation);
     } catch (e) {
+        logger.error(context, 'expressionPlugin error %j procesing entity %j', e, entity);
         callback(e);
     }
 }
 
 exports.update = update;
 exports.setJEXLTransforms = setJEXLTransforms;
+exports.applyExpression = applyExpression;
+exports.extractContext = extractContext;
+exports.checkJexl = checkJexl;

package/lib/plugins/jexlParser.js

@@ -30,9 +30,10 @@
 const jexl = require('jexl');
 const errors = require('../errors');
 const logger = require('logops');
+const fillService = require('../services/common/domain').fillService;
 const config = require('../commonConfig');
 const baseTranformsMap = require('../jexlTranformsMap.js').map;
-const logContext = {
+let logContext = {
     op: 'IoTAgentNGSI.JEXL'
 };
 
@@ -96,6 +97,7 @@ function extractContext(attributeList) {
 }
 
 function applyExpression(expression, context, typeInformation) {
+    logContext = fillService(logContext, typeInformation);
     const result = parse(expression, context);
     logger.debug(logContext, 'applyExpression "[%j]" over "[%j]" result "[%j]" ', expression, context, result);
     const expressionResult = result !== undefined ? result : expression;

package/lib/plugins/multiEntity.js

@@ -30,7 +30,7 @@ const _ = require('underscore');
 const constants = require('../constants');
 const legacyParser = require('./expressionParser');
 const jexlParser = require('./jexlParser');
-const config = require('../commonConfig');
+const expressionPlugin = require('./expressionPlugin');
 /* eslint-disable-next-line  no-unused-vars */
 const logger = require('logops');
 /* eslint-disable-next-line  no-unused-vars */
@@ -41,25 +41,6 @@ const utils = require('./pluginUtils');
 /* eslint-disable-next-line  no-unused-vars */
 const aliasPlugin = require('./attributeAlias');
 
-function checkJexl(typeInformation) {
-    if (
-        config.getConfig().defaultExpressionLanguage === 'jexl' &&
-        typeInformation.expressionLanguage &&
-        typeInformation.expressionLanguage !== 'legacy'
-    ) {
-        return true;
-    } else if (config.getConfig().defaultExpressionLanguage === 'jexl' && !typeInformation.expressionLanguage) {
-        return true;
-    } else if (
-        config.getConfig().defaultExpressionLanguage === 'legacy' &&
-        typeInformation.expressionLanguage &&
-        typeInformation.expressionLanguage === 'jexl'
-    ) {
-        return true;
-    }
-    return false;
-}
-
 function hasEntityName(item) {
     return item.entity_name;
 }
@@ -226,7 +207,7 @@ function propagateTimestamp(entity, entities) {
 
 function updateAttribute(entity, typeInformation, callback) {
     let parser = legacyParser;
-    if (checkJexl(typeInformation)) {
+    if (expressionPlugin.checkJexl(typeInformation)) {
         parser = jexlParser;
     }
     const attsArray = utils.extractAttributesArrayFromNgsi2Entity(entity);

package/lib/request-shim.js

@@ -0,0 +1,111 @@
+/*
+ * Copyright 2014 Telefonica Investigación y Desarrollo, S.A.U
+ *
+ * This file is part of fiware-iotagent-lib
+ *
+ * fiware-iotagent-lib is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the License,
+ * or (at your option) any later version.
+ *
+ * fiware-iotagent-lib is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public
+ * License along with fiware-iotagent-lib.
+ * If not, see http://www.gnu.org/licenses/.
+ *
+ * For those usages not covered by the GNU Affero General Public License
+ * please contact with::daniel.moranjimenez@telefonica.com
+ */
+
+const got = require('got');
+const logger = require('logops');
+const context = {
+    op: 'IoTAgentNGSI.Request'
+};
+
+/**
+ *  Transform the "request" options into "got" options and add additional "got" defaults
+ *
+ *  The following options are currently exposed:
+ *   - `method` - HTTP Method
+ *   - `searchParams` - query string params
+ *   - `qs` - alias for query string params
+ *   - `headers`
+ *   - `responseType` - either `text` or `json`. `json` is the default
+ *   - `json` - a supplied JSON object as the request body
+ *   - `body` - any ASCII text as  the request body
+ *   - `url` - the request URL
+ *   - `uri` - alternative alias for the request URL.
+ *
+ * @param {Object} options          Original definition of the request using the request library
+ * @return {Object}                 Updated definition of the request using the got library
+ *
+ */
+function getOptions(options) {
+    const httpOptions = {
+        method: options.method,
+        searchParams: options.searchParams || options.qs,
+        headers: options.headers,
+        throwHttpErrors: options.throwHttpErrors || false,
+        retry: options.retry || 0,
+        responseType: options.responseType || 'json'
+    };
+
+    // got library is not properly documented, so it is not clear which takes precedence
+    // among body, json and form (see https://stackoverflow.com/q/70754880/1485926).
+    // Thus, we are enforcing our own precedence with the "else if" chain below.
+    // Behaviour is consistent with the one described at usermanual.md#iotagentlibrequest
+
+    if (options.method === 'GET' || options.method === 'HEAD' || options.method === 'OPTIONS') {
+        // Do nothing - Never add a body
+    } else if (options.body) {
+        // body takes precedence over json or form
+        httpOptions.body = options.body;
+    } else if (options.json) {
+        // json takes precedence over form
+        httpOptions.json = options.json;
+    } else if (options.form) {
+        // Note that we don't consider 'form' part of the function API (check usermanual.md#iotagentlibrequest)
+        // but we are preparing the code anyway as a safe measure
+        httpOptions.form = options.form;
+    }
+
+    return httpOptions;
+}
+
+/*
+ *
+ *  Make a direct HTTP request using the underlying request library
+ *  (currently [got](https://github.com/sindresorhus/got)),
+ *
+ *  This function mimics the interface of the obsolete request library and switches
+ *  back from promises to callbacks to avoid re-writing large chunks of code.
+ *  This centralizes all HTTP requests in a single location and is useful
+ *  when creating agents which use an HTTP transport for their southbound
+ *  commands, and removes the need for the custom IoT Agent to import its own
+ *  additonal request library.
+ *
+ * @param {Object} options            Definition of the request .
+ * @param {Function} callback         The callback function.
+ *
+ */
+
+function request(options, callback) {
+    const httpOptions = getOptions(options);
+    logger.debug(context, 'Options: %s', JSON.stringify(options, null, 4));
+    got(options.url || options.uri, httpOptions)
+        .then((response) => {
+            logger.debug(context, 'Response %s', JSON.stringify(response.body, null, 4));
+            return callback(null, response, response.body);
+        })
+        .catch((error) => {
+            logger.debug(context, 'Error: %s', JSON.stringify(error, null, 4));
+            return callback(error);
+        });
+}
+
+module.exports = request;

package/lib/services/common/domain.js

@@ -47,6 +47,8 @@ function cleanDomain(domainToClean) {
     delete domainToClean.from;
     delete domainToClean.op;
     delete domainToClean.path;
+    delete domainToClean.service;
+    delete domainToClean.subservice;
     domainToClean.exit();
 }
 
@@ -139,8 +141,10 @@ function ensureSouthboundTransaction(context, callback) {
         reqDomain.corr = reqDomain.trans;
     }
 
-    if (context && context.op) {
-        reqDomain.op = context.op;
+    if (context) {
+        if (context.op) {
+            reqDomain.op = context.op;
+        }
 
         if (context.srv) {
             reqDomain.service = context.srv;

package/lib/services/common/genericMiddleware.js

@@ -71,7 +71,7 @@ function traceRequest(req, res, next) {
 
 /* eslint-disable-next-line  no-unused-vars */
 function changeLogLevel(req, res, next) {
-    const levels = ['INFO', 'ERROR', 'FATAL', 'DEBUG', 'WARNING'];
+    const levels = ['INFO', 'ERROR', 'FATAL', 'DEBUG', 'WARN', 'WARNING'];
 
     if (!req.query.level) {
         res.status(400).json({
@@ -82,7 +82,11 @@ function changeLogLevel(req, res, next) {
             error: 'invalid log level'
         });
     } else {
-        logger.setLevel(req.query.level.toUpperCase());
+        let newLevel = req.query.level.toUpperCase();
+        if (newLevel === 'WARNING') {
+            newLevel = 'WARN';
+        }
+        logger.setLevel(newLevel);
         res.status(200).send('');
     }
 }

package/lib/services/common/iotManagerService.js

@@ -23,7 +23,7 @@
  * Modified by: Federico M. Facca - Martel Innovate
  */
 
-const request = require('request');
+const request = require('../../request-shim');
 const async = require('async');
 const errors = require('../../errors');
 const constants = require('../../constants');

package/lib/services/common/securityServiceKeystone.js

@@ -21,7 +21,7 @@
  * please contact with::daniel.moranjimenez@telefonica.com
  */
 
-const request = require('request');
+const request = require('../../request-shim');
 const errors = require('../../errors');
 const config = require('../../commonConfig');
 const intoTrans = require('../common/domain').intoTrans;

package/lib/services/common/securityServiceOAuth2.js

@@ -26,7 +26,7 @@
 
 /* eslint-disable consistent-return */
 
-const request = require('request');
+const request = require('../../request-shim');
 const queryString = require('query-string');
 const errors = require('../../errors');
 const config = require('../../commonConfig');
@@ -64,7 +64,8 @@ function auth(trust, callback) {
             'Content-Length': formData.length,
             'Content-Type': 'application/x-www-form-urlencoded'
         },
-        body: formData
+        body: formData,
+        responseType: 'text'
     };
 
     logger.debug(context, 'Authentication on the OAuth2 provider [%s]', url);

package/lib/services/devices/deviceRegistryMemory.js

@@ -149,13 +149,19 @@ function getDevice(id, service, subservice, callback) {
     }
 }
 
-function getByName(name, service, subservice, callback) {
+function getByNameAndType(name, type, service, subservice, callback) {
     const devices = _.values(registeredDevices[service]);
     let device;
 
     for (let i = 0; i < devices.length; i++) {
         if (devices[i].name === name) {
-            device = devices[i];
+            if (type) {
+                if (devices[i].type === type) {
+                    device = devices[i];
+                }
+            } else {
+                device = devices[i];
+            }
         }
     }
 
@@ -166,6 +172,10 @@ function getByName(name, service, subservice, callback) {
     }
 }
 
+function getByName(name, service, subservice, callback) {
+    getByNameAndType(name, null, service, subservice, callback);
+}
+
 function update(device, callback) {
     registeredDevices[device.service][device.id] = deepClone(device);
     callback(null, device);
@@ -208,4 +218,5 @@ exports.list = listDevices;
 exports.get = getDevice;
 exports.getSilently = getDevice;
 exports.getByName = getByName;
+exports.getByNameAndType = getByNameAndType;
 exports.clear = clear;

package/lib/services/devices/deviceRegistryMongoDB.js

@@ -243,15 +243,18 @@ function getDevice(id, service, subservice, callback) {
     });
 }
 
-function getByName(name, service, servicepath, callback) {
+function getByNameAndType(name, type, service, servicepath, callback) {
     context = fillService(context, { service, subservice: servicepath });
-    logger.debug(context, 'Looking for device with name [%s].', name);
-
-    const query = Device.model.findOne({
-        name,
-        service,
+    let optionsQuery = {
+        name: name,
+        service: service,
         subservice: servicepath
-    });
+    };
+    if (type) {
+        optionsQuery.type = type;
+    }
+    logger.debug(context, 'Looking for device with [%j].', optionsQuery);
+    const query = Device.model.findOne(optionsQuery);
 
     query.select({ __v: 0 });
 
@@ -270,6 +273,10 @@ function getByName(name, service, servicepath, callback) {
     });
 }
 
+function getByName(name, service, servicepath, callback) {
+    getByNameAndType(name, null, service, servicepath, callback);
+}
+
 /**
  * Updates the given device into the database.
  * updated.
@@ -289,6 +296,7 @@ function update(device, callback) {
             data.internalAttributes = device.internalAttributes;
             data.commands = device.commands;
             data.endpoint = device.endpoint;
+            data.polling = device.polling;
             data.name = device.name;
             data.type = device.type;
             data.apikey = device.apikey;
@@ -360,4 +368,5 @@ exports.list = alarmsInt(constants.MONGO_ALARM, listDevices);
 exports.get = alarmsInt(constants.MONGO_ALARM, getDevice);
 exports.getSilently = getDevice;
 exports.getByName = alarmsInt(constants.MONGO_ALARM, getByName);
+exports.getByNameAndType = alarmsInt(constants.MONGO_ALARM, getByNameAndType);
 exports.clear = alarmsInt(constants.MONGO_ALARM, clear);