iotagent-node-lib 3.4.4 → 4.0.1

package/README.md

@@ -95,6 +95,10 @@ decommission devices. [API](doc/api.md).
 
 Contributions to development can be found [here](doc/devel/development.md) - additional contributions are welcome.
 
+If you are whishing to test the library, or include new tests (either as part of a contribution or as a new feature or
+as a bug report), you can use the functional tests suite included in the project. The tests are described using a JSON
+file. You can find more information about the test suite in the [Functional Tests Guide](test/functional/README.md).
+
 ### Agent Console
 
 A command-line client to experiment with the library is packed with it. The command-line client can be started using the

package/doc/admin.md

@@ -16,14 +16,12 @@
         -   [providerUrl](#providerurl)
         -   [iotaVersion](#iotaversion)
         -   [dieOnUnexpectedError](#dieonunexpectederror)
-        -   [singleConfigurationMode](#singleconfigurationmode)
         -   [timestamp](#timestamp)
         -   [defaultResource](#defaultresource)
         -   [defaultKey](#defaultkey)
         -   [componentName](#componentname)
         -   [pollingExpiration](#pollingexpiration)
         -   [pollingDaemonFrequency](#pollingdaemonfrequency)
-        -   [autocast](#autocast)
         -   [multiCore](#multicore)
         -   [fallbackTenant](#fallbacktenant)
         -   [fallbackPath](#fallbackpath)
@@ -262,7 +260,7 @@ the `mongob` section (as described bellow). E.g.:
 
 It configures the MongoDB driver for those repositories with 'mongodb' type. If the `host` parameter is a list of
 comma-separated IPs, they will be considered to be part of a Replica Set. In that case, the optional property
-`replicaSet` should contain the Replica Set name. If the database requires authentication, username (`username`),
+`replicaSet` should contain the Replica Set name. If the database requires authentication, username (`user`),
 password (`password`) and authSource (`authSource`) can be set. If the database requires TLS/SSL connection but any
 validation of the certificate chain is not mandatory, all you need is to set the ssl (`ssl`) option as `true` to connect
 the database. If you need to add more complex option(s) such as `retryWrites=true` or `w=majority` when connection
@@ -348,10 +346,6 @@ IoTA).
 
 if this flag is activated, the IoTAgent will not capture global exception, thus dying upon any unexpected error.
 
-#### `singleConfigurationMode`
-
-enables the Single Configuration mode for backwards compatibility (see description in the Overview). Default to false.
-
 #### `timestamp`
 
 if this flag is activated:
@@ -383,10 +377,6 @@ amount of time without being collected by the device, the expiration daemon will
 time between collection of expired commands in milliseconds. This attribute is optional (if this parameter doesn't exist
 the polling daemon won't be started).
 
-#### `autocast`
-
-When enabled, the IoT Agents will try to cast attribute's values considering the JSON native type (only for NGSI v2).
-
 #### `multiCore`
 
 When enabled, the IoT Agents runs in multi-thread environment to take advantage of multi-core systems. It allows two
@@ -483,10 +473,8 @@ overrides.
 | IOTA_MONGO_RETRY_TIME                | `mongodb.retryTime`             |
 | IOTA_MONGO_SSL                       | `mongodb.ssl`                   |
 | IOTA_MONGO_EXTRAARGS                 | `mongodb.extraArgs`             |
-| IOTA_SINGLE_MODE                     | `singleConfigurationMode`       |
 | IOTA_POLLING_EXPIRATION              | `pollingExpiration`             |
 | IOTA_POLLING_DAEMON_FREQ             | `pollingDaemonFrequency`        |
-| IOTA_AUTOCAST                        | `autocast`                      |
 | IOTA_MULTI_CORE                      | `multiCore`                     |
 | IOTA_JSON_LD_CONTEXT                 | `jsonLdContext`                 |
 | IOTA_FALLBACK_TENANT                 | `fallbackTenant`                |

package/doc/api.md

@@ -27,8 +27,8 @@
     -   [Measurement transformation](#measurement-transformation)
         -   [Measurement transformation definition](#measurement-transformation-definition)
         -   [Measurement transformation execution](#measurement-transformation-execution)
+        -   [Measurement transformation order](#measurement-transformation-order)
         -   [Multientity measurement transformation support (`object_id`)](#multientity-measurement-transformation-support-object_id)
-    -   [Timestamp Compression](#timestamp-compression)
     -   [Timestamp Processing](#timestamp-processing)
     -   [Overriding global Context Broker host](#overriding-global-context-broker-host)
     -   [Multitenancy, FIWARE Service and FIWARE ServicePath](#multitenancy-fiware-service-and-fiware-servicepath)
@@ -179,9 +179,9 @@ information coming from the device (measures) is mapped to the Context Broker at
 All of them have the same syntax, a list of objects with the following attributes:
 
 -   **object_id** (optional): name of the attribute as coming from the device.
--   **name** (mandatory): ID of the attribute in the target entity in the Context Broker. Note that `id` and `type`
-    are not valid attribute names at Context Broker. Thus, although a measure named `id` or `type` will not break the IOT Agent, they 
-    are silently ignored and never progress toward Context Broker entities.
+-   **name** (mandatory): ID of the attribute in the target entity in the Context Broker. Note that `id` and `type` are
+    not valid attribute names at Context Broker. Thus, although a measure named `id` or `type` will not break the IoT
+    Agent, they are silently ignored and never progress toward Context Broker entities.
 -   **type** (mandatory): name of the type of the attribute in the target entity.
 -   **metadata** (optional): additional static metadata for the attribute in the target entity. (e.g. `unitCode`)
 
@@ -211,9 +211,9 @@ Additionally for commands (which are attributes of type `command`) the following
 -   **contentType**: `content-type` header used when send command by HTTP transport (ignored in other kinds of
     transports)
 
-Note that, when information comming from devices, this means measures, are not defined neither in the group, nor in the 
-device, the IoT agent will store that information into the destination entity using the same attribute name than the 
-measure name, unless `explicitAttrs` is defined. Measures `id` or `type` names are invalid, and will be ignored. 
+Note that, when information coming from devices, this means measures, are not defined neither in the group, nor in the
+device, the IoT agent will store that information into the destination entity using the same attribute name than the
+measure name, unless `explicitAttrs` is defined. Measures `id` or `type` names are invalid, and will be ignored.
 
 ## Multientity support
 
@@ -408,7 +408,7 @@ Case 1 (default):
 "explicitAttrs": false
 ```
 
-every measure will be propagated to NGSI interface.
+every measure will be propagated to NGSI interface, including all static attributes.
 
 Case 2:
 
@@ -416,7 +416,14 @@ Case 2:
 "explicitAttrs": true
 ```
 
-just measures defined in active, static (plus conditionally TimeInstant) will be propagated to NGSI interface.
+In this case, should only progress active and static attributes defined in the device or group provision (`TimeInstant`
+attribute will be also included if enabled), including also all static attributes. In other words, having
+`"explicitAttrs":true` would prevent the IoTA creating attributes into the related entity within the context broker from
+measures that are not explicitly defined in the device or group provision.
+
+Note that attributes defined in the provision that are not receiving a measure (or having a expression defined that is
+resulting `null`) will not progress (this means, the NGSI request to update the entity in the context broker is not
+going to include that attribute) unless `skipValue` is defined to other value than `null`
 
 Case 3:
 
@@ -426,7 +433,11 @@ Case 3:
 
 just NGSI attributes defined in the array (identified by their attribute names, not by their object_id, plus
 conditionally TimeInstant) will be propagated to NGSI interface (note that in this case the value of `explicitAttrs` is
-not a JSON but a JEXL Array that looks likes a JSON).
+not a JSON but a JEXL Array that looks likes a JSON). Only static attributes included in that array will be propagated
+to NGSI interface.
+
+All attributes contained in the array must be defined as `attributes` or `static_attributes`. Not defined measures
+(`object_id`) will be dropped, even if they are defined in the `explicitAttrs` array.
 
 Case 4:
 
@@ -436,7 +447,11 @@ Case 4:
 
 just NGSI attributes defined in the array (identified by their attribute names and/or by their object_id) will be
 propagated to NGSI interface (note that in this case the value of `explicitAttrs` is not a JSON but a JEXL Array/Object
-that looks likes a JSON). This is necessary when same attribute names are used within multiple entities.
+that looks likes a JSON). This is necessary when same attribute names are used within multiple entities. Only static
+attributes included in that array will be propagated to NGSI interface.
+
+Note that in the previous case show above, when selecting the object_id (with `{object_id:'active_id'}`), the attribute 
+must be defined. In other words, it would not work if the attribute with the corresponding `object_id`, is not defined.
 
 Case 5:
 
@@ -759,6 +774,94 @@ following to CB:
 
 [Interactive expression `spaces | trim`][5]
 
+### Measurement transformation order
+
+The IoTA executes the transformaion looping over the `attributes` provision field. Every time a new expression is
+evaluated, the JEXL context is updated with the expression result. The order defined in the `attributes` array is taken
+for expression evaluation. This should be considered when using **nested expressions**, that uses values calculated in
+other attributes.
+
+For example, let's consider the following provision for a device which send a measure named `level`:
+
+```json
+"attributes": [
+    {
+        "name": "correctedLevel",
+        "type": "Number",
+        "expression": "level * 0.897"
+    },
+    {
+        "name": "normalizedLevel",
+        "type": "Number",
+        "expression": "correctedLevel / 100"
+    }
+]
+```
+
+The expression for `correctedLevel` is evaluated first (using `level` measure as input). Next, the `normalizedLevel` is
+evaluated (using `correctedLevel` calculated attribute, just calculated before).
+
+Note that if we reserve the order, this way:
+
+```json
+"attributes": [
+    {
+        "name": "normalizedLevel",
+        "type": "Number",
+        "expression": "correctedLevel / 100"
+    },
+    {
+        "name": "correctedLevel",
+        "type": "Number",
+        "expression": "level * 0.897"
+    },
+]
+```
+
+It is not going to work. The first expression expects a `correctedLevel` which is neither a measure (remember the only
+measure sent by the device is named `level`) nor a previously calculated attribute. Thus, `correctedLevel` will end with
+a `null` value, so will not be part of the update request send to the Context Broker unless `skipValue` (check
+[Devices](#devices) section avobe) is defined with a different value thant the default one (`null`).
+
+In conclusion: **the order of attributes in the `attributes` arrays at provising time matters with regards to nested
+expression evaluation**.
+
+Let's consider the following example. It is an anti-pattern but it's quite illustrative on how ordering works:
+
+```json
+"attributes": [
+    {
+        "name": "a",
+        "type": "Number",
+        "expression": "b*10"
+    },
+    {
+        "name": "b",
+        "type": "Number",
+        "expression": "a*10"
+    }
+]
+```
+
+When receiving a measure with the following values:
+
+```json
+{
+    "a": 10,
+    "b": 20
+}
+```
+
+Then, as they are executed sequentially, the first attribute expression to be evaluated will be `a`, taking the value of
+the attribute `b` multiplied by 10, in this case, `200`. After that, the second attribute expression to be evaluated is
+the one holded by `b`. In this case, that attribute would take 10 times the value of `a`. In that case, since the JEXL
+context was updated with the lastest execution, the value of `b` will be `2000`, being update at Context Broker entity:
+
+```json
+    "a": {"value": 200, "type": "Number"},
+    "b": {"value": 2000, "type": "Number"}
+```
+
 ### Multientity measurement transformation support (`object_id`)
 
 To allow support for measurement transformation in combination with multi entity feature, where the same attribute is
@@ -846,13 +949,6 @@ Will now generate the following NGSI v2 payload:
 }
 ```
 
-## Timestamp Compression
-
-This functionality changes all the timestamp attributes found in the entity, and all the timestamp metadata found in any
-attribute, from the basic complete calendar timestamp of the ISO8601 (e.g.: 20071103T131805) to the extended complete
-calendar timestamp (e.g.: +002007-11-03T13:18). The middleware expects to receive the basic format in updates and return
-it in queries (and viceversa, receive the extended one in queries and return it in updates).
-
 ## Timestamp Processing
 
 The IOTA processes the entity attributes looking for a `TimeInstant` attribute. If one is found, for NGSI v2, then it
@@ -1120,6 +1216,7 @@ Config group is represented by a JSON object with the following fields:
 | `ngsiVersion`                  | ✓        | string         |            | optional string value used in mixed mode to switch between **NGSI-v2** and **NGSI-LD** payloads. Possible values are: `v2` or `ld`. The default is `v2`. When not running in mixed mode, this field is ignored.                                                           |
 | `defaultEntityNameConjunction` | ✓        | string         |            | optional string value to set default conjunction string used to compose a default `entity_name` when is not provided at device provisioning time.                                                                                                                         |
 | `autoprovision`                | ✓        | bool           | ✓?         | optional boolean: If `false`, autoprovisioned devices (i.e. devices that are not created with an explicit provision operation but when the first measure arrives) are not allowed in this group. Default (in the case of omitting the field) is `true`.                   |
+| `payloadType`                  | ✓        | string         |            | optional string value used to switch between **IoTAgent**, **NGSI-v2** and **NGSI-LD** measure payloads types. Possible values are: `iotagent`, `ngsiv2` or `ngsild`. The default is `iotagent`.                                                                          |
 
 ### Config group operations
 
@@ -1340,6 +1437,7 @@ the API resource fields and the same fields in the database model.
 | `internal_attributes` | ✓        | `array`   |            | List of internal attributes with free format for specific IoT Agent configuration.                                                                                                                                                                               |
 | `explicitAttrs`       | ✓        | `boolean` | ✓          | Field to support selective ignore of measures so that IOTA doesn’t progress. See details in [specific section](#explicitly-defined-attributes-explicitattrs)                                                                                                     |
 | `ngsiVersion`         | ✓        | `string`  |            | string value used in mixed mode to switch between **NGSI-v2** and **NGSI-LD** payloads. The default is `v2`. When not running in mixed mode, this field is ignored.                                                                                              |
+| `payloadType`         | ✓        | `string`  |            | optional string value used to switch between **IoTAgent**, **NGSI-v2** and **NGSI-LD** measure payloads types. Possible values are: `iotagent`, `ngsiv2` or `ngsild`. The default is `iotagent`.                                                                 |
 
 ### Device operations
 

package/doc/devel/architecture.md

@@ -79,18 +79,6 @@ configuration was assigned to a particular subservice and just one configuration
 relation between a Device and a Configuration didn't need the type to discriminate between Configurations. That's why
 for those agents, type was not a mandatory parameter.
 
-In order to allow backward-compatibility with those agents, the IoT Agent Library now implement a compatibility mode:
-the **Single Configuration Mode**, that makes the agent behave like the old agents. In this mode:
-
--   Each Subservice can contain just one Configuration. If a second Configuration is created for a Subservice, an error
-    is raised.
-
--   Each Device provisioned for a Subservice is automatically assigned to the Subservice one Configuration if there is
-    any.
-
-This compatibility has to be set for the whole IoT Agent, and there is no option of having both modes simultaneously
-running. Transitions from one mode to the other should be made with care, and may involve data migration.
-
 #### Registration
 
 Whenever a device is registered, the IoT Agent reads the device's entity information from the request or, if that

package/doc/index.md

@@ -11,7 +11,7 @@ Broker using their own native protocols. IoT Agents should also be able to deal
 platform (authentication and authorization of the channel) and provide other common services to the device programmer.
 
 Github's [README.md](https://github.com/telefonicaid/iotagent-node-lib/blob/master/README.md) provides a good
-documentation summary. The [API reference](doc/api.md) and the [Development documentation](devel/development.md) cover
+documentation summary. The [API reference](api.md) and the [Development documentation](devel/development.md) cover
 more advanced topics.
 
 ## Background

package/doc/roadmap.md

@@ -13,7 +13,7 @@ only, and this section may be revised to provide newer information at any time.
 
 Disclaimer:
 
--   This section has been last updated in March 2022. Please take into account its content could be obsolete.
+-   This section has been last updated in August 2023. Please take into account its content could be obsolete.
 -   Note we develop this software in Agile way, so development plan is continuously under review. Thus, this roadmap has
     to be understood as rough plan of features to be done along time which is fully valid only at the time of writing
     it. This roadmap has not be understood as a commitment on features and/or dates.
@@ -26,16 +26,22 @@ The following list of features are planned to be addressed in the short term, an
 product:
 
 -   cgroup literal in configuration groups management API (community)
--   Metadata processing improvements
--   Improve command functionalities (binary data + expression + mapping)
+-   Refactor Append Mode & initial entity ([#1413](https://github.com/telefonicaid/iotagent-node-lib/issues/1413))
+-   Native support for NGSI-v2 and LD ingestion ([#1451](https://github.com/telefonicaid/iotagent-node-lib/issues/1451))
+-   Remove plugins structure (bidirectional plugin) ([#1413](https://github.com/telefonicaid/iotagent-node-lib/issues/1413))
+-   Add init and improve log traces ([#1452](https://github.com/telefonicaid/iotagent-node-lib/issues/1452))
+-   Remove InMemory registry for Devices and Groups ([#1429](https://github.com/telefonicaid/iotagent-node-lib/issues/1429))
 
 ### Medium term
 
 The following list of features are planned to be addressed in the medium term, typically within the subsequent
 release(s) generated in the next 9 months after the next planned release:
 
--   Accept JEXL Expressions for entity name in autoprovisioned devices (#1145)
--   Refactor entities-NGSI-v2.js module (#1166)
+-   Allow to add metadata to attributes from measures ([#1453](https://github.com/telefonicaid/iotagent-node-lib/issues/1453))
+-   Cache support ([#1467](https://github.com/telefonicaid/iotagent-node-lib/issues/1467))
+-   MQTT per group advanced configuration (MQTT broker, topics) ([#1454](https://github.com/telefonicaid/iotagent-node-lib/issues/1454))
+-   Subscription based commands ([#1455](https://github.com/telefonicaid/iotagent-node-lib/issues/1455))
+-   Remove registration support ([#1456](https://github.com/telefonicaid/iotagent-node-lib/issues/1456))
 
 ### Long term
 
@@ -43,23 +49,29 @@ The following list of features are proposals regarding the longer-term evolution
 development of these features has not yet been scheduled for a release in the near future. Please feel free to contact
 us if you wish to get involved in the implementation or influence the roadmap:
 
+-   Use the lightweight ingestion mechanism for connection oriented updates implemented in Context Broker (testing pending) ([#1457](https://github.com/telefonicaid/iotagent-node-lib/issues/1457))
+-   Add support to other transport protocols (BacNET, Modbus, etc) ([#1458](https://github.com/telefonicaid/iotagent-node-lib/issues/1458))
+-   Dynamic attribute generation (based on values array) ([#1459](https://github.com/telefonicaid/iotagent-node-lib/issues/1459))
 -   Incremental introduccion of ECMAScript6 syntax (previous analysis of which sub-set of interesting aspect we want to
     take)
--   Use the lightweight ingestion mechanism for connection oriented updates implemented in Context Broker
--   Add support to other transport protocols (BacNET, Modbus, etc)
 
 ### Features already completed
 
 The following list contains all features that were in the roadmap and have already been implemented.
 
+-   Refactor entities-NGSI-v2.js module ([#1166](https://github.com/telefonicaid/iotagent-node-lib/issues/1166)) ([3.0.0](https://github.com/telefonicaid/iotagent-node-lib/releases/tag/3.0.0))
+-   Accept JEXL Expressions for entity name in autoprovisioned devices (entityNameExp) ([#1145](https://github.com/telefonicaid/iotagent-node-lib/issues/1145)) ([2.22.0](https://github.com/telefonicaid/iotagent-node-lib/releases/tag/2.22.0))
+-   Improve command functionalities (binary data + expression + mapping) ([2.22.0](https://github.com/telefonicaid/iotagent-node-lib/releases/tag/2.22.0))
 -   Support for "delta" measures (i.e. "temperature _increased_ in 5 degress" instead of "temperature _is_ 25")
 -   Allow to handle binary messages ([iota-ul#530](https://github.com/telefonicaid/iotagent-ul/issues/530))
--   Removal support for NGSIv1 (#966) ([2.18.0](https://github.com/telefonicaid/iotagent-node-lib/releases/tag/2.18.0))
+-   Removal support for NGSIv1 ([#966](https://github.com/telefonicaid/iotagent-node-lib/issues/966)) ([2.18.0](https://github.com/telefonicaid/iotagent-node-lib/releases/tag/2.18.0))
 -   Selectively ignore measure in the southbound interface
     ([iotagent-json#416](https://github.com/telefonicaid/iotagent-json/issues/416),
     [iotagent-ul#372](https://github.com/telefonicaid/iotagent-ul/issues/372))
     ([2.13.0](https://github.com/telefonicaid/iotagent-node-lib/releases/tag/2.13.0))
--   JEXL support in expressions (#801, #687, #868)
+-   JEXL support in expressions ([#801](https://github.com/telefonicaid/iotagent-node-lib/issues/801), 
+     [#687](https://github.com/telefonicaid/iotagent-node-lib/issues/687), 
+     [#868](https://github.com/telefonicaid/iotagent-node-lib/issues/868))
     ([2.13.0](https://github.com/telefonicaid/iotagent-node-lib/releases/tag/2.13.0))
--   Add MongoDB authentication support (#844)
+-   Add MongoDB authentication support ([#844](https://github.com/telefonicaid/iotagent-node-lib/issues/844))
     ([2.12.0](https://github.com/telefonicaid/iotagent-node-lib/releases/tag/2.12.0))

package/lib/commonConfig.js

@@ -142,13 +142,11 @@ function processEnvironmentVariables() {
         'IOTA_MONGO_PORT',
         'IOTA_MONGO_DB',
         'IOTA_MONGO_REPLICASET',
-        'IOTA_AUTOCAST',
         'IOTA_MONGO_PASSWORD',
         'IOTA_MONGO_AUTH_SOURCE',
         'IOTA_MONGO_RETRIES',
         'IOTA_MONGO_USER',
         'IOTA_MONGO_RETRY_TIME',
-        'IOTA_SINGLE_MODE',
         'IOTA_POLLING_EXPIRATION',
         'IOTA_POLLING_DAEMON_FREQ',
         'IOTA_MULTI_CORE',
@@ -444,11 +442,6 @@ function processEnvironmentVariables() {
         }
     }
 
-    // Other configuration properties
-    if (process.env.IOTA_SINGLE_MODE) {
-        config.singleConfigurationMode = process.env.IOTA_SINGLE_MODE === 'true';
-    }
-
     if (process.env.IOTA_POLLING_EXPIRATION) {
         config.pollingExpiration = process.env.IOTA_POLLING_EXPIRATION;
     }
@@ -457,10 +450,6 @@ function processEnvironmentVariables() {
         config.pollingDaemonFrequency = process.env.IOTA_POLLING_DAEMON_FREQ;
     }
 
-    if (process.env.IOTA_AUTOCAST) {
-        config.autocast = process.env.IOTA_AUTOCAST === 'true';
-    }
-
     if (process.env.IOTA_MULTI_CORE) {
         config.multiCore = process.env.IOTA_MULTI_CORE === 'true';
     } else {

package/lib/errors.js

@@ -229,9 +229,9 @@ class BadAnswer {
     }
 }
 class BadTimestamp {
-    constructor(payload) {
+    constructor(payload, entityName) {
         this.name = 'BAD_TIMESTAMP';
-        this.message = 'Invalid ISO8601 timestamp [' + payload + ']';
+        this.message = 'Invalid ISO8601 timestamp [' + payload + '] in [' + entityName + ']';
         this.code = 400;
     }
 }

package/lib/model/Device.js

@@ -52,7 +52,8 @@ const Device = new Schema({
     internalAttributes: Object,
     autoprovision: Boolean,
     explicitAttrs: Group.ExplicitAttrsType,
-    ngsiVersion: String
+    ngsiVersion: String,
+    payloadType: String
 });
 
 function load(db) {

package/lib/model/Group.js

@@ -62,7 +62,8 @@ const Group = new Schema({
     explicitAttrs: ExplicitAttrsType,
     defaultEntityNameConjunction: String,
     ngsiVersion: String,
-    entityNameExp: String
+    entityNameExp: String,
+    payloadType: String
 });
 
 function load(db) {

package/lib/model/dbConn.js

@@ -72,9 +72,15 @@ function init(host, db, port, options, callback) {
         return previous;
     }
 
+    url = 'mongodb://';
+
+    if (options.auth) {
+        url += options.auth.user + ':' + options.auth.password + '@';
+    }
+
     const hosts = host.split(',').map(addPort).reduce(commaConcat, '');
 
-    url = 'mongodb://' + hosts + '/' + db;
+    url += hosts + '/' + db;
 
     if (options.extraArgs) {
         if (options.extraArgs instanceof Object && Object.keys(options.extraArgs).length > 0) {
@@ -112,6 +118,7 @@ function init(host, db, port, options, callback) {
         // but not sure if current mongoose version is still using mongodb 3.x internally
         // probably mongodb-connectionoptions-test.js needs to be fixed if useNewUrlParser is removed at the end
         options.useNewUrlParser = true;
+        options.useUnifiedTopology = true;
         mongoose.set('useCreateIndex', true);
         /* eslint-disable-next-line no-unused-vars */
         const candidateDb = mongoose.createConnection(url, options, function (error, result) {
@@ -203,16 +210,6 @@ function configureDb(callback) {
                 options.replicaSet = currentConfig.mongodb.replicaSet;
             }
 
-            if (currentConfig.mongodb.user && currentConfig.mongodb.password) {
-                options.auth = {};
-                options.auth.user = currentConfig.mongodb.user;
-                options.auth.password = currentConfig.mongodb.password;
-            }
-
-            if (currentConfig.mongodb.authSource) {
-                options.authSource = currentConfig.mongodb.authSource;
-            }
-
             if (currentConfig.mongodb.ssl) {
                 options.ssl = currentConfig.mongodb.ssl;
             }
@@ -221,6 +218,20 @@ function configureDb(callback) {
                 options.extraArgs = currentConfig.mongodb.extraArgs;
             }
 
+            if (currentConfig.mongodb.user && currentConfig.mongodb.password) {
+                options.auth = {};
+                options.auth.user = currentConfig.mongodb.user;
+                options.auth.password = currentConfig.mongodb.password;
+                // authSource only applies if auth is set
+                if (currentConfig.mongodb.authSource) {
+                    // Overload extraArgs if it was set
+                    options.extraArgs = {
+                        ...options.extraArgs,
+                        authSource: currentConfig.mongodb.authSource
+                    };
+                }
+            }
+
             init(config.getConfig().mongodb.host, dbName, port, options, callback);
         }
     } else {

package/lib/plugins/expressionPlugin.js

@@ -57,13 +57,8 @@ function contextAvailable(expression, context, typeInformation) {
     return jexlParser.contextAvailable(expression, context);
 }
 
-function extractVariables(expression) {
-    return jexlParser.extractVariables(expression);
-}
-
 exports.parse = parse;
 exports.setJEXLTransforms = setJEXLTransforms;
 exports.applyExpression = applyExpression;
 exports.extractContext = extractContext;
 exports.contextAvailable = contextAvailable;
-exports.extractVariables = extractVariables;

package/lib/plugins/jexlParser.js

@@ -28,8 +28,6 @@
 /* eslint-disable no-unused-vars */
 
 const jexl = require('jexl');
-const grammar = require('jexl/dist/grammar').getGrammar();
-const Lexer = require('jexl/dist/Lexer');
 const errors = require('../errors');
 const logger = require('logops');
 const fillService = require('../services/common/domain').fillService;
@@ -47,7 +45,7 @@ function parse(expression, context, callback) {
         result = jexl.evalSync(expression, context);
         //avoid undefined result
         result = result !== undefined ? result : null;
-        logger.debug(logContext, 'parse expression "[%j]" over "[%j]" result "[%j]" ', expression, context, result);
+        logger.debug(logContext, 'parse expression %j over %j result %j ', expression, context, result);
     } catch (e) {
         error = new errors.InvalidExpression(expression);
         if (callback) {
@@ -64,31 +62,6 @@ function parse(expression, context, callback) {
     }
 }
 
-function extractVariables(expression) {
-    const inst = new Lexer(grammar);
-    let variables = [];
-
-    try {
-        const tokens = inst.tokenize(expression);
-
-        // Keep only root attributes, removing the dot and sub-attributes. For example, if we have
-        // a.0.b, a.1.b and a.2.b, we will only keep a
-        // Additionaly, it will remove the function calls, since they are also detected as identifiers
-        variables = tokens.filter(function (token, index, array) {
-            return (
-                (token.type === ' ' && array[index - 1].type !== 'dot') ||
-                (token.type === 'identifier' && array[index + 1] && array[index + 1].type !== 'openParen')
-            );
-        });
-
-        // Return only array of values
-        return variables.map((a) => a.value);
-    } catch (e) {
-        logger.warn(logContext, 'Wrong expression found "[%j]" error: "[%s]", it will be ignored', expression, e);
-        return false;
-    }
-}
-
 function extractContext(attributeList) {
     const context = {};
     let value;
@@ -125,12 +98,24 @@ function extractContext(attributeList) {
 
 function applyExpression(expression, context, typeInformation) {
     logContext = fillService(logContext, typeInformation);
+    // Delete null values from context. Related:
+    // https://github.com/telefonicaid/iotagent-node-lib/issues/1440
+    // https://github.com/TomFrost/Jexl/issues/133
+    deleteNullsAndNaN(context);
     const result = parse(expression, context);
-    logger.debug(logContext, 'applyExpression "[%j]" over "[%j]" result "[%j]" ', expression, context, result);
+    logger.debug(logContext, 'applyExpression %j over %j result %j ', expression, context, result);
     const expressionResult = result !== undefined ? result : expression;
     return expressionResult;
 }
 
+function deleteNullsAndNaN(object) {
+    for (let key in object) {
+        if (object[key] === null || Number.isNaN(object[key])) {
+            delete object[key];
+        }
+    }
+}
+
 function isTransform(identifier) {
     return jexl.getTransform(identifier) !== (null || undefined);
 }
@@ -141,7 +126,7 @@ function contextAvailable(expression, context) {
         jexl.evalSync(expression, context);
         return true;
     } catch (e) {
-        logger.warn(logContext, 'Wrong expression found "[%j]" over "[%j]", it will be ignored', expression, context);
+        logger.info(logContext, 'Wrong expression found %j over %j, it will be ignored', expression, context);
         return false;
     }
 }
@@ -189,7 +174,6 @@ function setTransforms(configMap) {
     logger.info(logContext, message);
 }
 
-exports.extractVariables = extractVariables;
 exports.extractContext = extractContext;
 exports.contextAvailable = contextAvailable;
 exports.applyExpression = applyExpression;