iotagent-node-lib 3.1.0 → 3.3.0

package/doc/deprecated.md

@@ -21,6 +21,9 @@ A list of deprecated features and the version in which they were deprecated foll
 -   Support to NGSI-LD v1.3 in iotagent-node-lib 2.25.0 (finally removed in 2.26.0)
 -   Support groups (provision) statically defined by configuration
 -   Support to in-memory registry (i.e.`deviceRegistry.type=memory`)
+-   Support to legacy expressions (finally removed in 3.2.0)
+-   Bidirectinal pluging (to be removed in 3.4.0)
+-   appendMode (to be removed in 3.4.0)
 
 The use of Node.js v14 is highly recommended.
 
@@ -48,4 +51,7 @@ The following table provides information about the last iotagent-node-lib versio
 | Support to Node.js v8  | 2.12.0                                                | April 7th, 2020               |
 | Support to Node.js v10 | 2.15.0                                                | February 18th, 2021           |
 | Support to Node.js v12 | 2.24.0                                                | September 2nd, 2022           |
-| Support to NGSI-LD 1.3 | 2.25.0                                                | January 24th, 2023          |
+| Support to NGSI-LD 1.3 | 2.25.0                                                | January 24th, 2023            |
+| Support to Legacy Expressions | 3.1.0                                          | April 25th, 2023              |
+| bidirrectional plugin | 3.3.0                                                  | August 24th, 2023             |
+| appendMode | 3.3.0                                                             | August 24th, 2023             |

package/doc/development.md

@@ -7,6 +7,9 @@
 -   [Continuous testing](#continuous-testing)
 -   [Code Coverage](#code-coverage)
 -   [Clean](#clean)
+-   [Data mapping plugins](#data-mapping-plugins)
+    -   [Development](#development)
+    -   [Provided plugins](#provided-plugins)
 
 ### Contributions
 
@@ -163,3 +166,120 @@ To ensure consistent Markdown formatting run the following:
 # Use git-bash on Windows
 npm run prettier:text
 ```
+
+# DB Models (from API document)
+
+## Service group model
+
+The table below shows the information held in the service group provisioning resource. The table also contains the
+correspondence between the API resource fields and the same fields in the database model.
+
+| Payload Field                  | DB Field                       | Definition                                                                                                                                                                                                                                                                |
+| ------------------------------ | ------------------------------ | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---- |
+| `service`                      | `service`                      | Service of the devices of this type                                                                                                                                                                                                                                       |
+| `subservice`                   | `subservice`                   | Subservice of the devices of this type.                                                                                                                                                                                                                                   |
+| `resource`                     | `resource`                     | string representing the Southbound resource that will be used to assign a type to a device (e.g.: pathname in the southbound port).                                                                                                                                       |
+| `apikey`                       | `apikey`                       | API Key string.                                                                                                                                                                                                                                                           |
+| `timestamp`                    | `timestamp`                    | Optional flag about whether or not to add the `TimeInstant` attribute to the device entity created, as well as a `TimeInstant` metadata to each attribute, with the current timestamp. With NGSI-LD, the Standard `observedAt` property-of-a-property is created instead. | true |
+| `entity_type`                  | `entity_type`                  | name of the Entity `type` to assign to the group.                                                                                                                                                                                                                         |
+| `trust`                        | `trust`                        | trust token to use for secured access to the Context Broker for this type of devices (optional; only needed for secured scenarios).                                                                                                                                       |
+| `cbHost`                       | `cbHost`                       | Context Broker connection information. This options can be used to override the global ones for specific types of devices.                                                                                                                                                |
+| `lazy`                         | `lazy`                         | list of common lazy attributes of the device. For each attribute, its `name` and `type` must be provided.                                                                                                                                                                 |
+| `commands`                     | `commands`                     | list of common commands attributes of the device. For each attribute, its `name` and `type` must be provided, additional `metadata` is optional.                                                                                                                          |
+| `attributes`                   | `attributes`                   | list of common active attributes of the device. For each attribute, its `name` and `type` must be provided, additional `metadata` is optional.                                                                                                                            |
+| `static_attributes`            | `staticAttributes`             | this attributes will be added to all the entities of this group 'as is', additional `metadata` is optional.                                                                                                                                                               |
+| `internal_attributes`          | `internalAttributes`           | optional section with free format, to allow specific IoT Agents to store information along with the devices in the Device Registry.                                                                                                                                       |
+| `expressionLanguage`           | `expresionLanguage`            | optional boolean value, to set expression language used to compute expressions, possible values are: legacy or jexl. When not set or wrongly set, `legacy` is used as default value.                                                                                      |
+| `explicitAttrs`                | `explicitAttrs`                | optional field to support selective ignore of measures so that IOTA doesn’t progress. See details in [specific section](advanced-topics.md#explicitly-defined-attributes-explicitattrs)                                                                                   |
+| `entityNameExp`                | `entityNameExp`                | optional field to allow use expressions to define entity name, instead default `id` and `type`                                                                                                                                                                            |
+| `ngsiVersion`                  | `ngsiVersion`                  | optional string value used in mixed mode to switch between **NGSI-v2** and **NGSI-LD** payloads. Possible values are: `v2` or `ld`. The default is `v2`. When not running in mixed mode, this field is ignored.                                                           |
+| `defaultEntityNameConjunction` | `defaultEntityNameConjunction` | optional string value to set default conjunction string used to compose a default `entity_name` when is not provided at device provisioning time.                                                                                                                         |
+| `autoprovision`                | `autoprovision`                | optional boolean: If `false`, autoprovisioned devices (i.e. devices that are not created with an explicit provision operation but when the first measure arrives) are not allowed in this group. Default (in the case of omitting the field) is `true`.                   |
+
+## Device model
+
+The table below shows the information held in the Device resource. The table also contains the correspondence between
+the API resource fields and the same fields in the database model.
+
+| Payload Field         | DB Field             | Definition                                                                                                                                                                                                                                                                | Example of value                              |
+| --------------------- | -------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | :-------------------------------------------- |
+| `device_id`           | `id`                 | Device ID that will be used to identify the device.                                                                                                                                                                                                                       | UO834IO                                       |
+| `service`             | `service`            | Name of the service the device belongs to (will be used in the fiware-service header).                                                                                                                                                                                    | smartGondor                                   |
+| `service_path`        | `subservice`         | Name of the subservice the device belongs to (used in the fiware-servicepath header).                                                                                                                                                                                     | /gardens                                      |
+| `entity_name`         | `name`               | Name of the entity representing the device in the Context Broker                                                                                                                                                                                                          | ParkLamplight12                               |
+| `entity_type`         | `type`               | Type of the entity in the Context Broker                                                                                                                                                                                                                                  | Lamplights                                    |
+| `timezone`            | `timezone`           | Time zone of the sensor if it has any                                                                                                                                                                                                                                     | America/Santiago                              |
+| `timestamp`           | `timestamp`          | Optional flag about whether or not to add the `TimeInstant` attribute to the device entity created, as well as a `TimeInstant` metadata to each attribute, with the current timestamp. With NGSI-LD, the Standard `observedAt` property-of-a-property is created instead. | true                                          |
+| `apikey`              | `apikey`             | Optional Apikey key string to use instead of group apikey                                                                                                                                                                                                                 | 9n4hb1vpwbjozzmw9f0flf9c2                     |
+| `endpoint`            | `endpoint`           | Endpoint where the device is going to receive commands, if any.                                                                                                                                                                                                           | http://theDeviceUrl:1234/commands             |
+| `protocol`            | `protocol`           | Name of the device protocol, for its use with an IoT Manager.                                                                                                                                                                                                             | IoTA-UL                                       |
+| `transport`           | `transport`          | Name of the device transport protocol, for the IoT Agents with multiple transport protocols.                                                                                                                                                                              | MQTT                                          |
+| `attributes`          | `active`             | List of active attributes of the device                                                                                                                                                                                                                                   | `[ { "name": "attr_name", "type": "Text" } ]` |
+| `lazy`                | `lazy`               | List of lazy attributes of the device                                                                                                                                                                                                                                     | `[ { "name": "attr_name", "type": "Text" } ]` |
+| `commands`            | `commands`           | List of commands of the device                                                                                                                                                                                                                                            | `[ { "name": "attr_name", "type": "Text" } ]` |
+| `internal_attributes` | `internalAttributes` | List of internal attributes with free format for specific IoT Agent configuration                                                                                                                                                                                         | LWM2M mappings from object URIs to attributes |
+| `static_attributes`   | `staticAttributes`   | List of static attributes to append to the entity. All the updateContext requests to the CB will have this set of attributes appended.                                                                                                                                    | `[ { "name": "attr_name", "type": "Text" } ]` |
+| `expressionLanguage`  | `expresionLanguage`  | optional boolean value, to set expression language used to compute expressions, possible values are: legacy or jexl. When not set or wrongly set, legacy is used as default value.                                                                                        |
+| `explicitAttrs`       | `explicitAttrs`      | optional field to support selective ignore of measures so that IOTA doesn’t progress. See details in [specific section](advanced-topics.md#explicitly-defined-attributes-explicitattrs)                                                                                   | (see details in specific section)             |
+| `ngsiVersion`         | `ngsiVersion`        | optional string value used in mixed mode to switch between **NGSI-v2** and **NGSI-LD** payloads. The default is `v2`. When not running in mixed mode, this field is ignored.                                                                                              | `v2/ld`                                       |
+
+## Data mapping plugins
+
+The IoT Agent Library provides a plugin mechanism in order to facilitate reusing code that makes small transformations
+on incoming data (both from the device and from the context consumers). This mechanism is based in the use of
+middlewares, i.e.: small pieces of code that receive and return an `entity`, making as many changes as they need, but
+taking care of returning a valid entity, that can be used as the input for other middlewares; this way, all those pieces
+of code can be chained together in order to make all the needed transformations in the target entity.
+
+There are two kinds of middlewares: updateContext middlewares and queryContext middlewares. The updateContext
+middlewares are applied before the information is sent to the Context Broker, modifiying the entity before it is sent to
+Orion. The queryContext middlewares are applied on the received data, whenever the IoT Agent queries the Context Broker
+for information. I.e.: both middlewares will be automatically applied whenever the `update()` or `query()` functions are
+called in the library.
+
+All the middlewares have the opportunity to break the chain of middleware applications by calling the `callback()` with
+an error object (the usual convention). If any of the updateContext middlewares raise an error, no request will be sent
+to the Context Broker. On the other hand, the queryContext request is always performed, but the call to the `query()`
+function will end up in an error if any of the queryContext middlewares report an error.
+
+### Development
+
+All the middlewares have the same signature:
+
+```javascript
+function middlewareName(entity, typeInformation, callback) {}
+```
+
+The arguments for any middleware are the NGSI data over which it can operate:
+
+-   An updateContext payload in the case of an updateContext middleware and a queryContext payload otherwise;
+-   a typeInformation object containing all the information about the device stored during registration.
+-   and the customary `callback` parameter, with the usual meaning. It's really important for the library user to call
+    this callback, as failing to do so may hang the IoT Agent completely. The callback must be called with the an
+    optional error in the first argument and the same arguments received (potentially modified) as the following.
+
+In order to manage the middlewares to the system, the following functions can be used:
+
+-   `addUpdateMiddleware`: adds an updateContext middleware to the stack of middlewares. All the middlewares will be
+    applied to every call to the `update()` function. The final payload of the updateContext request will be the result
+    of applying all this middlewares in the order they have been defined.
+
+-   `addQueryMiddleware`: adds a queryContext middleware to the stack of middlewares. All the middlewares will be
+    applied to every call to the `query()` function.
+
+-   `resetMiddlewares`: remove all the middlewares from the system.
+
+Usually, the full list of middlewares an IoT Agent will use would be added in the IoTAgent start sequence, so they
+should not change a lot during the IoT lifetime.
+
+### Provided plugins
+
+The library provides some plugins out of the box, in the `dataPlugins` collection. In order to load any of them, just
+use the `addQueryMiddleware` and `addUpdateMiddleware` functions with the selected plugin, as in the example:
+
+```javascript
+var iotaLib = require('iotagent-node-lib');
+
+iotaLib.addUpdateMiddleware(iotaLib.dataPlugins.compressTimestamp.update);
+iotaLib.addQueryMiddleware(iotaLib.dataPlugins.compressTimestamp.query);
+```

package/doc/howto.md

@@ -75,12 +75,12 @@ folder of your project. Remember to change the Context Broker IP to your local C
 Now we can begin with the code of our IoT Agent. The very minimum code we need to start an IoT Agent is the following:
 
 ```javascript
-var iotAgentLib = require("iotagent-node-lib"),
-    config = require("./config");
+var iotAgentLib = require('iotagent-node-lib'),
+    config = require('./config');
 
 iotAgentLib.activate(config, function (error) {
     if (error) {
-        console.log("There was an error activating the IOTA");
+        console.log('There was an error activating the IOTA');
         process.exit(1);
     }
 });
@@ -112,10 +112,10 @@ In order to add the Express dependency to your project, add the following line t
 The require section would end up like this (the standard `http` module is also needed):
 
 ```javascript
-var iotAgentLib = require("iotagent-node-lib"),
-    http = require("http"),
-    express = require("express"),
-    config = require("./config");
+var iotAgentLib = require('iotagent-node-lib'),
+    http = require('http'),
+    express = require('express'),
+    config = require('./config');
 ```
 
 And install the dependencies as usual with `npm install`. You will have to require both `express` and `http` in your
@@ -129,16 +129,16 @@ function initSouthbound(callback) {
     southboundServer = {
         server: null,
         app: express(),
-        router: express.Router(),
+        router: express.Router()
     };
 
-    southboundServer.app.set("port", 8080);
-    southboundServer.app.set("host", "0.0.0.0");
+    southboundServer.app.set('port', 8080);
+    southboundServer.app.set('host', '0.0.0.0');
 
-    southboundServer.router.get("/iot/d", manageULRequest);
+    southboundServer.router.get('/iot/d', manageULRequest);
     southboundServer.server = http.createServer(southboundServer.app);
-    southboundServer.app.use("/", southboundServer.router);
-    southboundServer.server.listen(southboundServer.app.get("port"), southboundServer.app.get("host"), callback);
+    southboundServer.app.use('/', southboundServer.router);
+    southboundServer.server.listen(southboundServer.app.get('port'), southboundServer.app.get('host'), callback);
 }
 ```
 
@@ -154,18 +154,18 @@ function manageULRequest(req, res, next) {
     iotAgentLib.retrieveDevice(req.query.i, req.query.k, function (error, device) {
         if (error) {
             res.status(404).send({
-                message: "Couldn't find the device: " + JSON.stringify(error),
+                message: "Couldn't find the device: " + JSON.stringify(error)
             });
         } else {
             values = parseUl(req.query.d, device);
-            iotAgentLib.update(device.name, device.type, "", values, device, function (error) {
+            iotAgentLib.update(device.name, device.type, '', values, device, function (error) {
                 if (error) {
                     res.status(500).send({
-                        message: "Error updating the device",
+                        message: 'Error updating the device'
                     });
                 } else {
                     res.status(200).send({
-                        message: "Device successfully updated",
+                        message: 'Device successfully updated'
                     });
                 }
             });
@@ -190,17 +190,17 @@ function parseUl(data, device) {
     }
 
     function createAttribute(element) {
-        var pair = element.split("|"),
+        var pair = element.split('|'),
             attribute = {
                 name: pair[0],
                 value: pair[1],
-                type: findType(pair[0]),
+                type: findType(pair[0])
             };
 
         return attribute;
     }
 
-    return data.split(",").map(createAttribute);
+    return data.split(',').map(createAttribute);
 }
 ```
 
@@ -227,14 +227,14 @@ show the modifications in the `activate()` function:
 ```javascript
 iotAgentLib.activate(config, function (error) {
     if (error) {
-        console.log("There was an error activating the IOTA");
+        console.log('There was an error activating the IOTA');
         process.exit(1);
     } else {
         initSouthbound(function (error) {
             if (error) {
-                console.log("Could not initialize South bound API due to the following error: %s", error);
+                console.log('Could not initialize South bound API due to the following error: %s', error);
             } else {
-                console.log("Both APIs started successfully");
+                console.log('Both APIs started successfully');
             }
         });
     }
@@ -286,7 +286,7 @@ A HTTP request library will be needed in order to make those calls. To this exte
 used. In order to do so, add the following require statement to the initialization code:
 
 ```javascript
-request = require("request");
+request = require('request');
 ```
 
 and add the `request` dependency to the `package.json` file:
@@ -303,11 +303,11 @@ and add the `request` dependency to the `package.json` file:
 The require section should now look like this:
 
 ```javascript
-var iotAgentLib = require("iotagent-node-lib"),
-    http = require("http"),
-    express = require("express"),
-    request = require("request"),
-    config = require("./config");
+var iotAgentLib = require('iotagent-node-lib'),
+    http = require('http'),
+    express = require('express'),
+    request = require('request'),
+    config = require('./config');
 ```
 
 ### Implementation
@@ -321,11 +321,11 @@ for the context provisioning requests. At this point, we should provide two hand
 ```javascript
 function queryContextHandler(id, type, service, subservice, attributes, callback) {
     var options = {
-        url: "http://127.0.0.1:9999/iot/d",
-        method: "GET",
+        url: 'http://127.0.0.1:9999/iot/d',
+        method: 'GET',
         qs: {
-            q: attributes.join(),
-        },
+            q: attributes.join()
+        }
     };
 
     request(options, function (error, response, body) {
@@ -350,21 +350,21 @@ attributes). Here is the code for the `createResponse()` function:
 
 ```javascript
 function createResponse(id, type, attributes, body) {
-    var values = body.split(","),
+    var values = body.split(','),
         responses = [];
 
     for (var i = 0; i < attributes.length; i++) {
         responses.push({
             name: attributes[i],
-            type: "string",
-            value: values[i],
+            type: 'string',
+            value: values[i]
         });
     }
 
     return {
         id: id,
         type: type,
-        attributes: responses,
+        attributes: responses
     };
 }
 ```
@@ -374,11 +374,11 @@ function createResponse(id, type, attributes, body) {
 ```javascript
 function updateContextHandler(id, type, service, subservice, attributes, callback) {
     var options = {
-        url: "http://127.0.0.1:9999/iot/d",
-        method: "GET",
+        url: 'http://127.0.0.1:9999/iot/d',
+        method: 'GET',
         qs: {
-            d: createQueryFromAttributes(attributes),
-        },
+            d: createQueryFromAttributes(attributes)
+        }
     };
 
     request(options, function (error, response, body) {
@@ -388,31 +388,31 @@ function updateContextHandler(id, type, service, subservice, attributes, callbac
             callback(null, {
                 id: id,
                 type: type,
-                attributes: attributes,
+                attributes: attributes
             });
         }
     });
 }
 ```
 
-The updateContext handler deals with the modification requests that arrive at the North Port of the IoT Agent via `/v2/op/update`. It is
-invoked once for each entity requested (note that a single request can contain multiple entity updates), with the same
-parameters used in the queryContext handler. The only difference is the value of the attributes array, now containing a
-list of attribute objects, each containing name, type and value. The handler must also make use of the callback to
-return a list of updated attributes.
+The updateContext handler deals with the modification requests that arrive at the North Port of the IoT Agent via
+`/v2/op/update`. It is invoked once for each entity requested (note that a single request can contain multiple entity
+updates), with the same parameters used in the queryContext handler. The only difference is the value of the attributes
+array, now containing a list of attribute objects, each containing name, type and value. The handler must also make use
+of the callback to return a list of updated attributes.
 
 For this handler we have used a helper function called `createQueryFromAttributes()`, that transforms the NGSI
 representation of the attributes to the UL type expected by the device:
 
 ```javascript
 function createQueryFromAttributes(attributes) {
-    var query = "";
+    var query = '';
 
     for (var i in attributes) {
-        query += attributes[i].name + "|" + attributes[i].value;
+        query += attributes[i].name + '|' + attributes[i].value;
 
         if (i != attributes.length - 1) {
-            query += ",";
+            query += ',';
         }
     }
 
@@ -544,10 +544,6 @@ request/seconds that can be manage by the server. It's important to remark that
 included in the IoT Agent Node Lib but it is not mandatory that you activate this functionality. In this example, we
 will see how to use this functionality to deploy an IoT Agent in multi-thread environment.
 
-**WARNING:** it has been observed in Orion-IOTA integration tests some fails in bidirectional plugin usage scenarios in
-multi-thread mode. The fail has not been confirmed yet (it could be a glitch of the testing environment). However, take
-this into account if you use multi-thread in combination with bidirectional plugin.
-
 In order to activate the functionality, you have two options, configure the `config.js` file to add the following line:
 
 ```javascript
@@ -563,9 +559,9 @@ variable and afterward the value of the multiCore in the `config.js` file. The r
 (the standard `http` module is also needed):
 
 ```javascript
-var iotAgent = require("../lib/iotagent-implementation"),
-    iotAgentLib = require("iotagent-node-lib"),
-    config = require("./config");
+var iotAgent = require('../lib/iotagent-implementation'),
+    iotAgentLib = require('iotagent-node-lib'),
+    config = require('./config');
 ```
 
 It is important to mention the purpose of the `iotAgent` variable. It is the proper implementation of the IoT Agent
@@ -581,9 +577,9 @@ about starting the IoTAgent:
 ```javascript
 iotAgentLib.startServer(config, iotAgent, function (error) {
     if (error) {
-        console.log(context, "Error starting IoT Agent: [%s] Exiting process", error);
+        console.log(context, 'Error starting IoT Agent: [%s] Exiting process', error);
     } else {
-        console.log(context, "IoT Agent started");
+        console.log(context, 'IoT Agent started');
     }
 });
 ```
@@ -610,7 +606,7 @@ handlers themselves. Here we can see the definition of the configuration handler
 
 ```javascript
 function configurationHandler(configuration, callback) {
-    console.log("\n\n* REGISTERING A NEW CONFIGURATION:\n%s\n\n", JSON.stringify(configuration, null, 4));
+    console.log('\n\n* REGISTERING A NEW CONFIGURATION:\n%s\n\n', JSON.stringify(configuration, null, 4));
     callback(null, configuration);
 }
 ```
@@ -627,8 +623,8 @@ feature, let's use the provisioning handler to change the value of the type of t
 
 ```javascript
 function provisioningHandler(device, callback) {
-    console.log("\n\n* REGISTERING A NEW DEVICE:\n%s\n\n", JSON.stringify(device, null, 4));
-    device.type = "CertifiedType";
+    console.log('\n\n* REGISTERING A NEW DEVICE:\n%s\n\n', JSON.stringify(device, null, 4));
+    device.type = 'CertifiedType';
     callback(null, device);
 }
 ```

package/doc/installationguide.md

@@ -73,9 +73,9 @@ overriding the group setting.
 ```
 
 When connected to an **NGSI-LD** context broker, an IoT Agent is able to indicate whether it is willing to accept `null`
-values and also whether it is able to process the **NGSI-LD** `datasetId` metadata element. Setting these
-values to `false` will cause the IoT Agent to return a 400 **Bad Request** HTTP status code explaining that the IoT
-Agent does not support nulls or multi-attribute requests if they are encountered.
+values and also whether it is able to process the **NGSI-LD** `datasetId` metadata element. Setting these values to
+`false` will cause the IoT Agent to return a 400 **Bad Request** HTTP status code explaining that the IoT Agent does not
+support nulls or multi-attribute requests if they are encountered.
 
 ```javascript
 {
@@ -249,10 +249,6 @@ used for the same purpose. For instance:
     'http://192.168.56.1:4041'.
 -   **iotaVersion**: indicates the version of the IoTA that will be displayed in the about method (it should be filled
     automatically by each IoTA).
--   **appendMode**: if this flag is activated, the update requests to the Context Broker will be performed always with
-    APPEND type, instead of the default UPDATE. This have implications in the use of attributes with Context Providers,
-    so this flag should be used with care. This flag is overwritten by `autoprovision` flag in group or device
-    provision.
 -   **dieOnUnexpectedError**: if this flag is activated, the IoTAgent will not capture global exception, thus dying upon
     any unexpected error.
 -   **singleConfigurationMode**: enables the Single Configuration mode for backwards compatibility (see description in
@@ -277,8 +273,6 @@ used for the same purpose. For instance:
     the IoTAgent runs in a single thread. For more details about multi-core functionality, please refer to the
     [Cluster](https://nodejs.org/api/cluster.html) module in Node.js and
     [this section](howto.md#iot-agent-in-multi-thread-mode) of the library documentation.
--   **defaultExpressionLanguage**: the default expression language used to compute expressions, possible values are:
-    `legacy` or `jexl`. When not set or wrongly set, `legacy` is used as default value.
 -   **fallbackTenant** - For Linked Data Context Brokers which do not support multi-tenancy, this provides an
     alternative mechanism for supplying the `NGSILD-Tenant` header. Note that NGSILD-Tenant has not yet been included in
     the NGSI-LD standard (it has been proposed for the next update of the standard, but the final decision has yet been
@@ -354,7 +348,6 @@ overrides.
 | IOTA_MONGO_SSL                       | `mongodb.ssl`                   |
 | IOTA_MONGO_EXTRAARGS                 | `mongodb.extraArgs`             |
 | IOTA_SINGLE_MODE                     | `singleConfigurationMode`       |
-| IOTA_APPEND_MODE                     | `appendMode`                    |
 | IOTA_POLLING_EXPIRATION              | `pollingExpiration`             |
 | IOTA_POLLING_DAEMON_FREQ             | `pollingDaemonFrequency`        |
 | IOTA_AUTOCAST                        | `autocast`                      |
@@ -362,7 +355,6 @@ overrides.
 | IOTA_JSON_LD_CONTEXT                 | `jsonLdContext`                 |
 | IOTA_FALLBACK_TENANT                 | `fallbackTenant`                |
 | IOTA_FALLBACK_PATH                   | `fallbackPath`                  |
-| IOTA_DEFAULT_EXPRESSION_LANGUAGE     | `defaultExpressionLanguage`     |
 | IOTA_EXPLICIT_ATTRS                  | `explicitAttrs`                 |
 | IOTA_DEFAULT_ENTITY_NAME_CONJUNCTION | `defaultEntityNameConjunction`  |
 | IOTA_RELAX_TEMPLATE_VALIDATION       | `relaxTemplateValidation`       |

package/doc/requirements.txt

@@ -1,4 +1,4 @@
 mkdocs==1.2.3
-Pygments==2.9.0
+Pygments==2.15.0
 Markdown==3.3.4
 jinja2==3.0.0

package/docker/Mosquitto/Dockerfile

@@ -1,4 +1,4 @@
-ARG  IMAGE_TAG=11.3-slim
+ARG  IMAGE_TAG=12.1-slim
 FROM debian:${IMAGE_TAG}
 
 ARG CLEAN_DEV_TOOLS

package/lib/commonConfig.js

@@ -153,7 +153,6 @@ function processEnvironmentVariables() {
         'IOTA_POLLING_EXPIRATION',
         'IOTA_POLLING_DAEMON_FREQ',
         'IOTA_MULTI_CORE',
-        'IOTA_DEFAULT_EXPRESSION_LANGUAGE',
         'IOTA_RELAX_TEMPLATE_VALIDATION',
         'IOTA_DEFAULT_ENTITY_NAME_CONJUNCTION',
         'IOTA_JSON_LD_CONTEXT',
@@ -245,9 +244,9 @@ function processEnvironmentVariables() {
         config.contextBroker.jsonLdContext = process.env.IOTA_JSON_LD_CONTEXT.split(',').map((ctx) => ctx.trim());
     }
 
-   if (Array.isArray(config.contextBroker.jsonLdContext) && config.contextBroker.jsonLdContext.length === 1){
+    if (Array.isArray(config.contextBroker.jsonLdContext) && config.contextBroker.jsonLdContext.length === 1) {
         config.contextBroker.jsonLdContext = config.contextBroker.jsonLdContext[0];
-   }
+    }
 
     config.contextBroker.fallbackTenant =
         process.env.IOTA_FALLBACK_TENANT || config.contextBroker.service || 'iotagent';
@@ -265,7 +264,7 @@ function processEnvironmentVariables() {
         config.server.port = process.env.IOTA_NORTH_PORT;
     }
 
-    config.server.ldSupport = config.server.ldSupport || {null: true, datasetId: true, merge: false};
+    config.server.ldSupport = config.server.ldSupport || { null: true, datasetId: true, merge: false };
 
     if (process.env.IOTA_LD_SUPPORT_NULL) {
         config.server.ldSupport.null = process.env.IOTA_LD_SUPPORT_NULL === 'true';
@@ -273,7 +272,7 @@ function processEnvironmentVariables() {
     if (process.env.IOTA_LD_SUPPORT_DATASET_ID) {
         config.server.ldSupport.datasetId = process.env.IOTA_LD_SUPPORT_DATASET_ID === 'true';
     }
-     if (process.env.IOTA_LD_SUPPORT_MERGE) {
+    if (process.env.IOTA_LD_SUPPORT_MERGE) {
         config.server.ldSupport.datasetId = process.env.IOTA_LD_SUPPORT_MERGE === 'true';
     }
 
@@ -473,9 +472,6 @@ function processEnvironmentVariables() {
         config.multiCore = config.multiCore === true;
     }
 
-    if (process.env.IOTA_DEFAULT_EXPRESSION_LANGUAGE) {
-        config.defaultExpressionLanguage = process.env.IOTA_DEFAULT_EXPRESSION_LANGUAGE;
-    }
     if (process.env.IOTA_RELAX_TEMPLATE_VALIDATION) {
         config.relaxTemplateValidation = process.env.IOTA_RELAX_TEMPLATE_VALIDATION === 'true';
     } else {
@@ -484,7 +480,9 @@ function processEnvironmentVariables() {
     if (process.env.IOTA_DEFAULT_ENTITY_NAME_CONJUNCTION) {
         config.defaultEntityNameConjunction = process.env.IOTA_DEFAULT_ENTITY_NAME_CONJUNCTION;
     } else {
-        config.defaultEntityNameConjunction = config.defaultEntityNameConjunction ? config.defaultEntityNameConjunction : ':';
+        config.defaultEntityNameConjunction = config.defaultEntityNameConjunction
+            ? config.defaultEntityNameConjunction
+            : ':';
     }
 }
 
@@ -539,7 +537,6 @@ function ngsiVersion() {
     return 'unknown';
 }
 
-
 /**
  * It checks if a combination of typeInformation or common Config is LD
  *

package/lib/fiware-iotagent-lib.js

@@ -125,16 +125,6 @@ function doActivate(newConfig, callback) {
         }
     }
 
-    if (
-        newConfig.defaultExpressionLanguage &&
-        (newConfig.defaultExpressionLanguage === 'legacy' || newConfig.defaultExpressionLanguage === 'jexl')
-    ) {
-        logger.info(context, 'Using ' + newConfig.defaultExpressionLanguage + ' as default expression language');
-    } else {
-        logger.info(context, 'Default expression language not set, or invalid, using legacy configuration');
-        newConfig.defaultExpressionLanguage = 'legacy';
-    }
-
     config.setConfig(newConfig); //after chaging some configuration, we re apply the configuration
 
     logger.info(context, 'Activating IOT Agent NGSI Library.');

package/lib/jexlTranformsMap.js

@@ -78,7 +78,8 @@ const map = {
     toisostring: (d) => new Date(d).toISOString(),
     // https://developer.mozilla.org/es/docs/Web/JavaScript/Reference/Global_Objects/Date/toLocaleString
     localestring: (d, timezone, options) => new Date(d).toLocaleString(timezone, options),
-    now: () => Date.now()
+    now: () => Date.now(),
+    hextostring: (val) => new TextDecoder().decode(new Uint8Array(val.match(/.{1,2}/g).map(byte => parseInt(byte, 16))))
 };
 
 exports.map = map;

package/lib/model/Device.js

@@ -51,7 +51,6 @@ const Device = new Schema({
     creationDate: { type: Date, default: Date.now },
     internalAttributes: Object,
     autoprovision: Boolean,
-    expressionLanguage: String,
     explicitAttrs: Group.ExplicitAttrsType,
     ngsiVersion: String
 });

package/lib/model/Group.js

@@ -59,7 +59,6 @@ const Group = new Schema({
     attributes: Array,
     internalAttributes: Array,
     autoprovision: Boolean,
-    expressionLanguage: String,
     explicitAttrs: ExplicitAttrsType,
     defaultEntityNameConjunction: String,
     ngsiVersion: String,