iotagent-node-lib 4.4.0 → 4.6.0

package/lib/services/ngsi/ngsiService.js

@@ -26,6 +26,7 @@
 
 const async = require('async');
 const apply = async.apply;
+const statsRegistry = require('../stats/statsRegistry');
 const intoTrans = require('../common/domain').intoTrans;
 const fillService = require('./../common/domain').fillService;
 const errors = require('../../errors');
@@ -67,7 +68,8 @@ function init() {
  * @param {String} token            User token to identify against the PEP Proxies (optional).
  */
 function sendUpdateValue(entityName, attributes, typeInformation, token, callback) {
-    entityHandler.sendUpdateValue(entityName, attributes, typeInformation, token, callback);
+    const newCallback = statsRegistry.withStats('updateEntityRequestsOk', 'updateEntityRequestsError', callback);
+    entityHandler.sendUpdateValue(entityName, attributes, typeInformation, token, newCallback);
 }
 
 /**

package/lib/services/ngsi/subscription-NGSI-LD.js

@@ -89,7 +89,7 @@ function createSubscriptionHandlerNgsiLD(device, triggers, store, callback) {
                 triggers
             });
 
-            config.getRegistry().update(device, callback);
+            config.getRegistry().update(device, device, callback);
         } else {
             callback(null, response.headers.location);
         }
@@ -203,7 +203,7 @@ function createUnsubscribeHandlerNgsiLD(device, id, callback) {
             callback(new errors.BadRequest(body.orionError.details));
         } else {
             device.subscriptions.splice(device.subscriptions.indexOf(id), 1);
-            config.getRegistry().update(device, callback);
+            config.getRegistry().update(device, device, callback);
         }
     };
 }

package/lib/services/ngsi/subscription-NGSI-v2.js

@@ -91,7 +91,7 @@ function createSubscriptionHandlerNgsi2(device, triggers, store, callback) {
                 triggers
             });
 
-            config.getRegistry().update(device, callback);
+            config.getRegistry().update(device, device, callback);
         } else {
             callback(null, response.headers.location);
         }
@@ -203,7 +203,7 @@ function createUnsubscribeHandlerNgsi2(device, id, callback) {
             callback(new errors.BadRequest(body.orionError.details));
         } else {
             device.subscriptions.splice(device.subscriptions.indexOf(id), 1);
-            config.getRegistry().update(device, callback);
+            config.getRegistry().update(device, device, callback);
         }
     };
 }

package/lib/services/northBound/deviceGroupAdministrationServer.js

@@ -29,13 +29,19 @@ const restUtils = require('./restUtils');
 const groupService = require('./../groups/groupService');
 const async = require('async');
 const apply = async.apply;
-const templateGroup = require('../../templates/deviceGroup.json');
+const templateGroupService = require('../../templates/deviceGroup.json');
 let configurationHandler;
 let removeConfigurationHandler;
 let configurationMiddleware = [];
 const _ = require('underscore');
 const mandatoryHeaders = ['fiware-service', 'fiware-servicepath'];
 const mandatoryParameters = ['resource', 'apikey'];
+const constants = require('../../constants');
+
+// Create new template for configuration groups replacing services by CONFIGGROUP_TERM
+const templateGroup = JSON.parse(JSON.stringify(templateGroupService));
+templateGroup.properties[constants.CONFIGGROUP_TERM] = templateGroup.properties.services;
+delete templateGroup.properties.services;
 
 const apiToInternal = {
     entity_type: 'type',
@@ -118,6 +124,8 @@ function applyConfigurationMiddlewares(newConfiguration, callback) {
  * @param {Function} next        Invokes the next middleware in the chain.
  */
 function handleCreateDeviceGroup(req, res, next) {
+    req.body = checkAndModifyGroupRecv(req._parsedUrl.pathname, req.body); // #FIXME1649: This line should be removed when /iot/services support is removed
+
     for (let i = 0; i < req.body.services.length; i++) {
         req.body.services[i] = applyMap(apiToInternal, req.body.services[i]);
         req.body.services[i].service = req.headers['fiware-service'];
@@ -161,8 +169,7 @@ function handleListDeviceGroups(req, res, next) {
             } else {
                 translatedGroup = applyMap(internalToApi, group);
             }
-
-            res.status(200).send(translatedGroup);
+            res.status(200).send(checkAndModifyGroupResp(req._parsedUrl.pathname, translatedGroup)); // #FIXME1649: Response should not use checkAndModifyGroupResp function when /iot/services support is removed
         }
     };
 
@@ -263,25 +270,37 @@ function handleDeleteDeviceGroups(req, res, next) {
  *
  * @param {Object} router      Express request router object.
  */
+// #FIXME1649: Routes using /iot/services path should be removed when support for it is removed
 function loadContextRoutes(router) {
     router.post(
         '/iot/services',
         restUtils.checkRequestAttributes('headers', mandatoryHeaders),
+        restUtils.checkBody(templateGroupService),
+        handleCreateDeviceGroup
+    );
+
+    router.post(
+        '/iot/groups',
+        restUtils.checkRequestAttributes('headers', mandatoryHeaders),
         restUtils.checkBody(templateGroup),
         handleCreateDeviceGroup
     );
 
-    router.get('/iot/services', restUtils.checkRequestAttributes('headers', mandatoryHeaders), handleListDeviceGroups);
+    router.get(
+        ['/iot/services', '/iot/groups'],
+        restUtils.checkRequestAttributes('headers', mandatoryHeaders),
+        handleListDeviceGroups
+    );
 
     router.put(
-        '/iot/services',
+        ['/iot/services', '/iot/groups'],
         restUtils.checkRequestAttributes('headers', mandatoryHeaders),
         restUtils.checkRequestAttributes('query', mandatoryParameters),
         handleModifyDeviceGroups
     );
 
     router.delete(
-        '/iot/services',
+        ['/iot/services', '/iot/groups'],
         restUtils.checkRequestAttributes('headers', mandatoryHeaders),
         restUtils.checkRequestAttributes('query', mandatoryParameters),
         handleDeleteDeviceGroups
@@ -306,6 +325,23 @@ function clear(callback) {
     callback();
 }
 
+// #FIXME1649: This function should be removed when /iot/services support is removed
+function checkAndModifyGroupResp(route, payload) {
+    if (route === '/iot/groups') {
+        payload['groups'] = payload['services'];
+        delete payload['services'];
+    }
+    return payload;
+}
+// #FIXME1649: This function should be removed when /iot/services support is removed
+function checkAndModifyGroupRecv(route, payload) {
+    if (route === '/iot/groups') {
+        payload['services'] = payload['groups'];
+        delete payload['groups'];
+    }
+    return payload;
+}
+
 exports.loadContextRoutes = loadContextRoutes;
 exports.setConfigurationHandler = setConfigurationHandler;
 exports.setRemoveConfigurationHandler = setRemoveConfigurationHandler;

package/lib/services/northBound/deviceProvisioningServer.js

@@ -182,7 +182,6 @@ function attributeToProvisioningAPIFormat(attribute) {
         type: attribute.type,
         expression: attribute.expression,
         skipValue: attribute.skipValue,
-        reverse: attribute.reverse,
         entity_name: attribute.entity_name,
         entity_type: attribute.entity_type,
         mqtt: attribute.mqtt,
@@ -398,11 +397,11 @@ function handleRemoveDevices(req, res, next) {
  * This middleware handles updates in the provisioning devices. The only attribute
  */
 function handleUpdateDevice(req, res, next) {
-    function applyUpdatingHandler(device, callback) {
+    function applyUpdatingHandler(newDevice, oldDevice, callback) {
         if (updatingHandler) {
-            updatingHandler(device, callback);
+            updatingHandler(newDevice, oldDevice, callback);
         } else {
-            callback(null, device);
+            callback(null, newDevice);
         }
     }
 
@@ -429,10 +428,11 @@ function handleUpdateDevice(req, res, next) {
                         isTypeOrNameUpdated = true;
                     }
                     async.waterfall(
-                        [apply(applyUpdatingHandler, newDevice)],
+                        [apply(applyUpdatingHandler, newDevice, device)],
                         function handleUpdating(error, newDeviceUpdated) {
                             deviceService.updateRegister(
                                 newDeviceUpdated,
+                                device,
                                 isTypeOrNameUpdated,
                                 function handleDeviceUpdate(error) {
                                     if (error) {

package/lib/services/northBound/northboundServer.js

@@ -33,6 +33,7 @@ const intoTrans = domainUtils.intoTrans;
 const deviceProvisioning = require('./deviceProvisioningServer');
 const deviceUpdating = require('./deviceProvisioningServer');
 const groupProvisioning = require('./deviceGroupAdministrationServer');
+const statsRegistry = require('../stats/statsRegistry');
 const logger = require('logops');
 const context = {
     op: 'IoTAgentNGSI.NorthboundServer'
@@ -56,8 +57,8 @@ function start(config, callback) {
     northboundServer.app.set('port', config.server.port);
     northboundServer.app.set('host', config.server.host || '0.0.0.0');
     northboundServer.app.use(domainUtils.requestDomain);
-    northboundServer.app.use(bodyParser.json());
-    northboundServer.app.use(bodyParser.json({ type: 'application/*+json' }));
+    northboundServer.app.use(bodyParser.json({ limit: config.expressLimit }));
+    northboundServer.app.use(bodyParser.json({ type: 'application/*+json', limit: config.expressLimit }));
 
     if (config.logLevel && config.logLevel === 'DEBUG') {
         northboundServer.app.use(middlewares.traceRequest);
@@ -83,6 +84,7 @@ function start(config, callback) {
     northboundServer.router.get('/version', middlewares.retrieveVersion);
     northboundServer.router.put('/admin/log', middlewares.changeLogLevel);
     northboundServer.router.get('/admin/log', middlewares.getLogLevel);
+    northboundServer.router.get('/metrics', statsRegistry.openmetricsHandler);
 
     northboundServer.app.use(baseRoot, northboundServer.router);
     contextServer.loadContextRoutes(northboundServer.router);

package/lib/services/stats/statsRegistry.js

@@ -23,16 +23,9 @@
 
 /* eslint-disable no-prototype-builtins */
 
-const async = require('async');
 const _ = require('underscore');
-const apply = async.apply;
 const logger = require('logops');
-const config = require('../../commonConfig');
-const dbService = require('../../model/dbConn');
 let globalStats = {};
-let currentStats = {};
-let timerActions = [];
-let timerHandler;
 const statsContext = {
     op: 'IoTAgentNGSI.TimedStats'
 };
@@ -45,30 +38,14 @@ const statsContext = {
  * @param {Number} value        Value to be added to the total.
  */
 function add(key, value, callback) {
-    if (currentStats[key]) {
-        currentStats[key] += value;
-    } else {
-        currentStats[key] = value;
-    }
-
     if (globalStats[key]) {
         globalStats[key] += value;
     } else {
         globalStats[key] = value;
     }
-
     callback(null);
 }
 
-/**
- * Get the current value of a particular stat.
- *
- * @param {String} key          Name of the stat to retrive.
- */
-function getCurrent(key, callback) {
-    callback(null, currentStats[key]);
-}
-
 /**
  * Get the global value of the selected attribute.
  *
@@ -85,112 +62,162 @@ function getAllGlobal(callback) {
     callback(null, globalStats);
 }
 
-/**
- * Get all the current stats currently stored in the repository.
- */
-function getAllCurrent(callback) {
-    callback(null, currentStats);
-}
-
 /**
  * Loads the values passed as parameters into the global statistics repository.
  *
  * @param {Object} values       Key-value map with the values to be load.
  */
 function globalLoad(values, callback) {
-    globalStats = values;
-    currentStats = {};
-
-    for (const i in values) {
-        if (values.hasOwnProperty(i)) {
-            currentStats[i] = 0;
-        }
-    }
-
+    globalStats = _.clone(values);
     callback(null);
 }
 
 /**
- * Reset each of the current stats to value zero.
+ * Chooses the appropiate content type and version based on Accept header
+ * 
+ * @param {String} accepts The accepts header
  */
-function resetCurrent(callback) {
-    for (const i in currentStats) {
-        if (currentStats.hasOwnProperty(i)) {
-            currentStats[i] = 0;
+function matchContentType(accepts) {
+    const requestedType = [];
+    const vlabel = 'version=';
+    const clabel = 'charset=';
+    const qlabel = 'q=';
+    for (const expression of accepts.split(',')) {
+        const parts = expression.split(';').map((part) => part.trim());
+        const mediaType = parts[0];
+        let version = null;
+        let charset = null;
+        let preference = null;
+        for (let part of parts.slice(1)) {
+            if (part.startsWith(vlabel)) {
+                version = part.substring(vlabel.length).trim();
+            } else if (part.startsWith(clabel)) {
+                charset = part.substring(clabel.length).trim();
+            } else if (part.startsWith(qlabel)) {
+                preference = parseFloat(part.substring(qlabel.length).trim());
+            }
         }
+        requestedType.push({
+            mediaType: mediaType,
+            version: version,
+            charset: charset,
+            preference: preference || 1.0
+        });
     }
-
-    callback();
-}
-
-/**
- * Executes all the stored timer actions when a timer click is received.
- */
-function tickHandler() {
-    process.nextTick(apply(async.series, timerActions));
-}
-
-/**
- * Adds a new timer action to the timerActions Array, activating the timer if it was not previously activated.
- *
- * @param {Function} handler        Action to be executed. Should take two statistics objects and a callback.
- */
-function addTimerAction(handler, callback) {
-    if (!timerHandler && config.getConfig().stats.interval) {
-        timerHandler = setInterval(tickHandler, config.getConfig().stats.interval);
+    // If both text/plain and openmetrics are accepted,
+    // prefer openmetrics
+    const mediaTypePref = {
+        'application/openmetrics-text': 1.0,
+        'text/plain': 0.5
     }
-
-    timerActions.push(apply(handler, currentStats, globalStats));
-    callback();
+    // sort requests by priority descending
+    requestedType.sort(function (a, b) {
+        if (a.preference === b.preference) {
+            // same priority, sort by media type.
+            return (mediaTypePref[b.mediaType] || 0) - (mediaTypePref[a.mediaType] || 0);
+        }
+        return b.preference - a.preference;
+    });
+    for (const req of requestedType) {
+        switch(req.mediaType) {
+            case 'application/openmetrics-text':
+                req.version = req.version || '1.0.0';
+                req.charset = req.charset || 'utf-8';
+                if (
+                    (req.version === '1.0.0' || req.version === '0.0.1') &&
+                    (req.charset === 'utf-8')) {
+                    return req;
+                }
+                break;
+            case 'text/plain':
+            case 'text/*':
+            case '*/*':
+                req.version = req.version || '0.0.4';
+                req.charset = req.charset || 'utf-8';
+                if (
+                    (req.version === '0.0.4') &&
+                    (req.charset === 'utf-8')) {
+                    req.mediaType = 'text/plain';
+                    return req;
+                }
+                break;
+        }
+    }
+    return null;
 }
 
 /**
- * Clear the actions array and stop the timers.
+ * Predefined http handler that returns current openmetrics data
  */
-function clearTimers(callback) {
-    if (timerHandler) {
-        clearInterval(timerHandler);
-        timerHandler = undefined;
+/* eslint-disable-next-line  no-unused-vars */
+function openmetricsHandler(req, res) {
+    // Content-Type:
+    // - For openmetrics collectors, it MUST BE 'application/openmetrics-text; version=1.0.0; charset=utf-8'. See:
+    // https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#overall-structure
+    // - For prometheus compatible collectors, it SHOULD BE 'text/plain; version=0.0.4; charset=utf-8'. See:
+    // https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md
+    // - Caveat: Some versions of prometheus have been observed to send multivalued Accept headers such as
+    // Accept: application/openmetrics-text; version=0.0.1,text/plain;version=0.0.4;q=0.5,*/*;q=0.1
+    let reqType = {
+        mediaType: 'application/openmetrics-text',
+        version: '1.0.0',
+        charset: 'utf-8'
     }
-
-    timerActions = [];
-    callback();
+    if (req.headers.accept) {
+        // WORKAROUND: express version 4 does not parse properly the openmetrics Accept header,
+        // it won't match the regular expressions supported by `express.accepts`.
+        // So we must parse these key-value pairs ourselves.
+        reqType = matchContentType(req.headers.accept);
+        if (reqType === null) {
+            logger.error(statsContext, 'Unsupported media type: %s', req.headers.accept);
+            res.status(406).send('Not Acceptable');
+            return;
+        }
+    }
+    const contentType = `${reqType.mediaType};version=${reqType.version};charset=${reqType.charset}`;
+    // The actual payload is the same for all supported content types
+    const metrics = [];
+    for (const key in globalStats) {
+        if (globalStats.hasOwnProperty(key)) {
+            metrics.push('# HELP ' + key + ' global metric for ' + key);
+            metrics.push('# TYPE ' + key + ' counter');
+            metrics.push(key + ' ' + globalStats[key]);
+        }
+    }
+    // Expositions MUST END WITH '#EOF'
+    // See https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md
+    metrics.push('# EOF');
+    res.set('Content-Type', contentType);
+    res.status(200).send(metrics.join('\n'));
 }
 
 /**
- * Predefined stats action that logs the stats to the standard log.
+ * Wraps a callback with stats, incrementing the given counters
+ * depending on the parameters passed to the callback:
  *
- * @param {Object} currentValues        Current stat values.
- * @param {Object} globalValues         Global stat values.
- */
-function logStats(currentValues, globalValues, callback) {
-    logger.info(statsContext, 'Global stat values:\n%s\n', JSON.stringify(globalValues, null, 4));
-    logger.info(statsContext, 'Current stat values:\n%s\n', JSON.stringify(currentValues, null, 4));
-
-    resetCurrent(callback);
-}
-
-/**
- * Predefined action that persists the current value of the stats in the MongoDb instance.
+ * - If the callback receives an error, the errCounter is incremented.
+ * - If the callback receives no error, the okCounter is incremented.
  *
- * @param {Object} currentValues        Current stat values.
- * @param {Object} globalValues         Global stat values.
+ * @param {String} okCounter          Name of the counter to increment on success.
+ * @param {String} errCounter         Name of the counter to increment on error.
+ * @param {Function} callback         Callback to wrap. It must be a function that can
+ *                                    expect any number of parameters, but the first one must
+ *                                    be an indication of the error occured, if any.
  */
-function mongodbPersistence(currentValues, globalValues, callback) {
-    const statStamp = _.clone(globalValues);
-
-    statStamp.timestamp = new Date().toISOString();
-    dbService.db.collection('kpis').insertOne(statStamp, callback);
+function withStats(okCounter, errCounter, callback) {
+    function accounting(...args) {
+        const counter = args.length > 0 && args[0] ? errCounter : okCounter;
+        add(counter, 1, function () {
+            callback(...args);
+        });
+    }
+    return accounting;
 }
 
 exports.add = add;
-exports.getCurrent = getCurrent;
 exports.getGlobal = getGlobal;
 exports.getAllGlobal = getAllGlobal;
-exports.getAllCurrent = getAllCurrent;
 exports.globalLoad = globalLoad;
-exports.resetCurrent = resetCurrent;
-exports.clearTimers = clearTimers;
-exports.addTimerAction = addTimerAction;
-exports.logStats = logStats;
-exports.mongodbPersistence = mongodbPersistence;
+exports.withStats = withStats;
+exports.openmetricsHandler = openmetricsHandler;
+exports.matchContentType = matchContentType;

package/lib/templates/createDevice.json

@@ -119,30 +119,6 @@
                   "type": "string",
                   "pattern": "^([^<>;'=\"]+)+$"
                 },
-                "reverse": {
-                  "description": "Define the attribute as bidirectional",
-                  "type": "array",
-                  "items": {
-                    "type": "object",
-                    "additionalProperties": false,
-                    "properties": {
-                      "object_id": {
-                        "description": "ID of the attribute in the device",
-                        "type": "string",
-                        "pattern": "^([^<>();'=\"]+)+$"
-                      },
-                      "type": {
-                        "description": "Type of the attribute in the target entity",
-                        "type": "string",
-                        "pattern": "^([^<>();'=\"]+)+$"
-                      },
-                      "expression": {
-                        "description": "Optional expression for measurement transformation",
-                        "type": "string"
-                      }
-                    }
-                  }
-                },
                 "metadata": {
                   "description": "Attribute Metadata",
                   "type": "object"

package/lib/templates/createDeviceLax.json

@@ -116,29 +116,6 @@
                   "description": "Optional entity type for multientity updatess",
                   "type": "string",
                   "pattern": "^([^<>;'=\"]+)+$"
-                },
-                "reverse": {
-                  "description": "Define the attribute as bidirectional",
-                  "type": "array",
-                  "items": {
-                    "type": "object",
-                    "additionalProperties": false,
-                    "properties": {
-                      "object_id": {
-                        "description": "ID of the attribute in the device",
-                        "type": "string"                      
-                      },
-                      "type": {
-                        "description": "Type of the attribute in the target entity",
-                        "type": "string",
-                        "pattern": "^([^<>();'=\"]+)+$"
-                      },
-                      "expression": {
-                        "description": "Optional expression for measurement transformation",
-                        "type": "string"
-                      }
-                    }
-                  }
                 }
               }
             }

package/lib/templates/deviceGroup.json

@@ -60,33 +60,9 @@
           "commands": {
             "description": "list of commands of the devices",
             "type": "array"
-          },
-          "reverse": {
-            "description": "Define the attribute as bidirectional",
-            "type": "array",
-            "items": {
-              "type": "object",
-              "additionalProperties": false,
-              "properties": {
-                "object_id": {
-                  "description": "ID of the attribute in the device",
-                  "type": "string",
-                  "pattern": "^([^<>();'=\"]+)+$"
-                },
-                "type": {
-                  "description": "Type of the attribute in the target entity",
-                  "type": "string",
-                  "pattern": "^([^<>();'=\"]+)+$"
-                },
-                "expression": {
-                  "description": "Optional expression for measurement transformation",
-                  "type": "string"
-                }
-              }
-            }
           }
         }
       }
     }
-  }
+  }  
 }

package/lib/templates/updateDevice.json

@@ -105,30 +105,6 @@
             "type": "string",
             "pattern": "^([^<>;'=\"]+)+$"
           },
-          "reverse": {
-            "description": "Define the attribute as bidirectional",
-            "type": "array",
-            "items": {
-              "type": "object",
-              "additionalProperties": false,
-              "properties": {
-                "object_id": {
-                  "description": "ID of the attribute in the device",
-                  "type": "string",
-                  "pattern": "^([^<>();'=\"]+)+$"
-                },
-                "type": {
-                  "description": "Type of the attribute in the target entity",
-                  "type": "string",
-                  "pattern": "^([^<>();'=\"]+)+$"
-                },
-                "expression": {
-                  "description": "Optional expression for measurement transformation",
-                  "type": "string"
-                }
-              }
-            }
-          },
           "metadata": {
               "description": "Attribute Metadata",
               "type": "object"