iotagent-node-lib 4.6.0 → 4.8.0

package/doc/models/models.md

@@ -0,0 +1,260 @@
+# MongoDB Datamodel
+
+This file displays IoT Agent datamodel stored in the MongoDB database.
+
+## Collections
+
+### Devices
+
+The collection 'devices' sotres information about the iotagent devices
+
+Fields:
+
+-   **\_ID** _ObjectId_: unique object ID used by mongoDB
+-   **id** _string_: id of device
+-   **type** _string_: entity type used
+-   **name** _string_: entity name used
+-   **lazy** _array_: array of lazy attributes of device
+-   **active** _array_: array of active attributes of device
+-   **commands** _array_: array of commands of device
+-   **apikey** _string_: apikey of device
+-   **endpoint** _string_: endpoint used by push commands when http
+-   **resource** _string_: iotagent resource
+-   **protocol** _string_: device protocol (JSON)
+-   **transport** _string_: device transport (http, mqtt, amqp)
+-   **staticAttributes** _array_: array of static attributes of device
+-   **subscriptions** _array_: subscriptions of device
+-   **service** _string_: service which the device belongs to
+-   **subservice** _string_: subservice which the rule belongs to.
+-   **polling** _boolean_: if device uses polling for commands
+-   **timezone** _string_: timezone of device
+-   **timestamp** _boolean_ timestamp of device
+-   **registrationId** _string_: registrationId of device
+-   **internalId** _string_: internalId of device
+-   **creationDate** _date_: creationDate of device
+-   **internalAttributes** _object_: internalAttributes of device
+-   **autoprovision** _boolean_: if device support autoprovision
+-   **explicitAttrs** _enum_:
+-   **ngsiVersion** _string_: ngsi version used by device
+-   **payloadType** _string_: payloadType used by device
+-   **useCBflowControl** _boolean_: if CBFlow will be used by device when updates in CB
+-   **storeLastMeasure** _boolean_: if device store last measure received
+-   **lastMeasure** _object_: last measure received by device
+-   **oldCtxt** _object_: jexl Ctxt used at last measure
+
+Example:
+
+```json
+{
+    "_id": {
+        "$oid": "680b4b338d0e60f98718a8b2"
+    },
+    "lazy": [],
+    "commands": [
+        {
+            "name": "reset",
+            "type": "command",
+            "value": "",
+            "expression": "{ set: brand + '_' + reset }",
+            "object_id": "reset"
+        },
+        {
+            "name": "cmd1",
+            "type": "command",
+            "value": "",
+            "expression": "brand",
+            "object_id": "cmd1"
+        },
+        {
+            "name": "cmd2",
+            "type": "command",
+            "value": "",
+            "expression": "reset",
+            "object_id": "cmd2"
+        }
+    ],
+    "staticAttributes": [],
+    "creationDate": {
+        "$date": {
+            "$numberLong": "1745570611491"
+        }
+    },
+    "id": "disp",
+    "type": "thing",
+    "name": "thing:disp",
+    "service": "smartcity",
+    "subservice": "/",
+    "registrationId": "680b4b33956cc1ed0205840a",
+    "apikey": "APIKEY",
+    "protocol": "IoTA-JSON",
+    "transport": "HTTP",
+    "polling": false,
+    "active": [],
+    "oldCtxt": {
+        "level": "33",
+        "brand": "o1",
+        "id": "disp",
+        "type": "thing",
+        "service": "smartcity",
+        "subservice": "/",
+        "entity_name": "thing:disp",
+        "TimeInstant": "2025-04-25T08:43:31.496Z"
+    },
+    "subscriptions": []
+}
+```
+
+### Groups
+
+The collection groups' stores information about the iotagent groups of devices
+
+Fields:
+
+-   **\_ID** _ObjectId_: unique object ID used by mongoDB
+-   **url** _string_: url used by group of devices
+-   **resource** _string_: iotagent resource
+-   **apikey** _string_: apikey used by group of devices
+-   **endpoint** _string_: endpoint used by push commands when http
+-   **transport** _string_: group transport (http, mqtt, amqp)
+-   **type** _string_: entity type used
+-   **service** _string_: service which the group of device belongs to
+-   **subservice** _string_: subservice which the group of devices belongs to
+-   **description** _string_: description of group of devices
+-   **trust** _string_: keystone trust id used when devices of this group request to CB
+-   **cbHost** _string_: CB endpoint used by devices of this group
+-   **timezone** _string_: timezone used by group of devices
+-   **timestamp** _boolean_: timestamp of group
+-   **commands** _array_: array of commands of device group
+-   **staticAttributes** _array_: array of static attributes of device group
+-   **lazy** _array_: array of lazy attributes of device group
+-   **attributes** _array_: array of active attributes of device group
+-   **internalAttributes** _array_: array of internal attributes used by devices of group
+-   **autoprovision** _boolean_: if devices of group supports autoprovision
+-   **explicitAttrs** _enum_: explicit attributes configuration used by devices of group
+-   **defaultEntityNameConjunction** _string_:
+-   **ngsiVersion** _string_: ngsi version used by devices of group
+-   **entityNameExp** _string_: entity name expression used by devics of group
+-   **payloadType** _string_: payloadType used by devices of group
+-   **useCBflowControl** _boolean_: payloadType used by device group
+-   **storeLastMeasure** _boolean_: if devices of group store last measure received
+
+Example:
+
+```json
+{
+    "_id": {
+        "$oid": "67a1e6447ae8b4ba4478f019"
+    },
+    "commands": [
+        {
+            "name": "reset",
+            "type": "command",
+            "value": "",
+            "expression": "{ set: brand + '_' + reset }"
+        },
+        {
+            "name": "cmd1",
+            "type": "command",
+            "value": "",
+            "expression": "brand"
+        },
+        {
+            "name": "cmd2",
+            "type": "command",
+            "value": "",
+            "expression": "reset"
+        }
+    ],
+    "staticAttributes": [
+        {
+            "name": "brand",
+            "type": "Text",
+            "value": "o1",
+            "metadata": {}
+        }
+    ],
+    "attributes": [],
+    "resource": "/iot/json",
+    "apikey": "APIKEY",
+    "type": "thing",
+    "service": "smartcity",
+    "subservice": "/",
+    "description": "miJSON",
+    "timestamp": true,
+    "internalAttributes": [],
+    "lazy": [],
+    "transport": "HTTP",
+    "endpoint": "'https://eoykcmmm.m.pipedream.net' + '/' + service + '/' + subservice + '/' + id + '/' + type"
+}
+```
+
+### Commands
+
+The collection 'commands' stores information about the commands
+
+Fields:
+
+-   **\_ID** _ObjectId_: unique object ID used by mongoDB
+-   **deviceId** _string_: device ID of the device
+-   **type** _string_: type of the command
+-   **name** _string_: name of the command
+-   **value** _object_: value of the command
+-   **service** _string_: service which the device command belongs to
+-   **subservice** _string_: subservice which the device command belongs to
+-   **execTs** _date_: related with new commands functionality (stored but not yet in use)
+-   **status** _string_: related with new commands functionality (stored but not yet in use)
+-   **info** _string_: related with new commands functionality (stored but not yet in use)
+-   **onDelivered** _Object_: related with new commands functionality (stored but not yet in use)
+-   **onOk**: _Object_: related with new commands functionality (stored but not yet in use)
+-   **onError** _Object_: related with new commands functionality (stored but not yet in use)
+-   **onInfo** _Object_: related with new commands functionality (stored but not yet in use)
+-   **cmdExecution** \_Boolean\_\_: related with new commands functionality (stored but not yet in use)
+-   **dateExpiration**: { type: Date }: related with new commands functionality (stored but not yet in use)
+-   **creationDate** _date_: creation date of command
+
+Example:
+
+```json
+{
+    "_id": {
+        "$oid": "680b4b538d0e60f98718a8eb"
+    },
+    "creationDate": {
+        "$date": {
+            "$numberLong": "1745570643252"
+        }
+    },
+    "name": "cmd1",
+    "type": "command",
+    "value": "on",
+    "deviceId": "disp3",
+    "service": "smartcity",
+    "subservice": "/"
+}
+```
+
+## Indexes
+
+### Devices
+
+An index guarantees that every device is identified by the tuple (service, subservice, apikey, id)
+
+The index is created/ensured when iotagent starts, but it can be created from a mongoDB shell with
+
+```javascript
+db.devices.ensureIndex({ service: 1, subservice: 1, apikey: 1, id: 1 }, { unique: true });
+```
+
+### Groups
+
+An index guarantees that every group is identified by the tuple (apikey, resource)
+
+The index is created/ensured when iotagent starts, but it can be created from a mongoDB shell with
+
+```javascript
+db.groups.ensureIndex({ apikey: 1, resource: 1 }, { unique: true });
+```
+
+### Commands
+
+None index is defined

package/doc/requirements.txt

@@ -1,4 +1,4 @@
 mkdocs==1.2.4
 Pygments==2.15.0
 Markdown==3.3.4
-jinja2==3.1.4
+jinja2==3.1.6

package/docker/Mosquitto/Dockerfile

@@ -1,4 +1,4 @@
-ARG  IMAGE_TAG=12.6-slim
+ARG  IMAGE_TAG=12.10-slim
 FROM debian:${IMAGE_TAG}
 
 ARG CLEAN_DEV_TOOLS

package/lib/commonConfig.js

@@ -157,7 +157,9 @@ function processEnvironmentVariables() {
         'IOTA_FALLBACK_PATH',
         'IOTA_LD_SUPPORT_NULL',
         'IOTA_LD_SUPPORT_DATASET_ID',
-        'IOTA_EXPRESS_LIMIT'
+        'IOTA_EXPRESS_LIMIT',
+        'IOTA_USE_CB_FLOW_CONTROL',
+        'IOTA_STORE_LAST_MEASURE'
     ];
     const iotamVariables = [
         'IOTA_IOTAM_URL',
@@ -350,6 +352,11 @@ function processEnvironmentVariables() {
         config.defaultResource = process.env.IOTA_DEFAULT_RESOURCE;
     }
 
+    // Default transport
+    if (process.env.IOTA_DEFAULT_TRANSPORT !== undefined) {
+        config.defaultTransport = process.env.IOTA_DEFAULT_TRANSPORT;
+    }
+
     // Default explicitAttrs
     if (process.env.IOTA_EXPLICIT_ATTRS !== undefined) {
         config.explicitAttrs = process.env.IOTA_EXPLICIT_ATTRS;
@@ -474,6 +481,16 @@ function processEnvironmentVariables() {
     } else {
         config.expressLimit = config.expressLimit ? config.expressLimit : '1mb';
     }
+    if (process.env.IOTA_USE_CB_FLOW_CONTROL) {
+        config.useCBflowControl = process.env.IOTA_USE_CB_FLOW_CONTROL === 'true';
+    } else {
+        config.useCBflowControl = config.useCBflowControl === true;
+    }
+    if (process.env.IOTA_STORE_LAST_MEASURE) {
+        config.storeLastMeasure = process.env.IOTA_STORE_LAST_MEASURE === 'true';
+    } else {
+        config.storeLastMeasure = config.storeLastMeasure === true;
+    }
 }
 
 function setConfig(newConfig) {
@@ -503,7 +520,9 @@ function getConfigForTypeInformation() {
         multiCore: config.multiCore,
         relaxTemplateValidation: config.relaxTemplateValidation,
         defaultEntityNameConjunction: config.defaultEntityNameConjunction,
-        defaultType: config.defaultType
+        defaultType: config.defaultType,
+        useCBflowControl: config.useCBflowControl,
+        storeLastMeasure: config.storeLastMeasure
     };
     return conf;
 }

package/lib/fiware-iotagent-lib.js

@@ -47,17 +47,20 @@ const context = {
 
 /* eslint-disable-next-line  no-unused-vars */
 function activateStatLogs(newConfig, callback) {
-    async.series([
-        apply(statsRegistry.globalLoad, {
-            deviceCreationRequests: 0,
-            deviceRemovalRequests: 0,
-            measureRequests: 0,
-            raiseAlarm: 0,
-            releaseAlarm: 0,
-            updateEntityRequestsOk: 0,
-            updateEntityRequestsError: 0
-        })
-    ], callback);
+    async.series(
+        [
+            apply(statsRegistry.globalLoad, {
+                deviceCreationRequests: 0,
+                deviceRemovalRequests: 0,
+                measureRequests: 0,
+                raiseAlarm: 0,
+                releaseAlarm: 0,
+                updateEntityRequestsOk: 0,
+                updateEntityRequestsError: 0
+            })
+        ],
+        callback
+    );
 }
 
 /**
@@ -314,6 +317,7 @@ exports.setDataUpdateHandler = contextServer.setUpdateHandler;
 exports.setCommandHandler = contextServer.setCommandHandler;
 exports.setMergePatchHandler = contextServer.setMergePatchHandler;
 exports.setDataQueryHandler = contextServer.setQueryHandler;
+exports.executeUpdateSideEffects = contextServer.executeUpdateSideEffects;
 exports.setConfigurationHandler = contextServer.setConfigurationHandler;
 exports.setRemoveConfigurationHandler = contextServer.setRemoveConfigurationHandler;
 exports.setProvisioningHandler = contextServer.setProvisioningHandler;

package/lib/jexlTranformsMap.js

@@ -26,63 +26,210 @@
 JEXL avaliable transformations*/
 
 const map = {
-    jsonparse: (str) => JSON.parse(str),
-    jsonstringify: (obj) => JSON.stringify(obj),
+    jsonparse: (val) => {
+        const safeOperation =
+            (fn) =>
+            (...args) => {
+                try {
+                    return fn(...args);
+                } catch (e) {
+                    return null;
+                }
+            };
+        return safeOperation(JSON.parse)(val);
+    },
+    jsonstringify: (val) => {
+        const safeOperation =
+            (fn) =>
+            (...args) => {
+                try {
+                    return fn(...args);
+                } catch (e) {
+                    return null;
+                }
+            };
+        return safeOperation(JSON.stringify)(val);
+    },
     indexOf: (val, char) => String(val).indexOf(char),
     length: (val) => String(val).length,
     trim: (val) => String(val).trim(),
     substr: (val, int1, int2) => String(val).substr(int1, int2),
-    addreduce: (arr) => arr.reduce((i, v) => i + v),
+    addreduce: (arr) => {
+        const safeOperation =
+            (fn) =>
+            (...args) => {
+                try {
+                    return fn(...args);
+                } catch (e) {
+                    return null;
+                }
+            };
+        return safeOperation((arr) => arr.reduce((i, v) => i + v))(arr);
+    },
     lengtharray: (arr) => arr.length,
     typeof: (val) => typeof val,
-    isarray: (arr) => Array.isArray(arr),
-    isnan: (val) => isNaN(val),
-    parseint: (val) => parseInt(val),
-    parsefloat: (val) => parseFloat(val),
-    toisodate: (val) => new Date(val).toISOString(),
-    timeoffset: (isostr) => new Date(isostr).getTimezoneOffset(),
-    tostring: (val) => val.toString(),
-    urlencode: (val) => encodeURI(val),
-    urldecode: (val) => decodeURI(val),
+    isarray: Array.isArray,
+    isnan: isNaN,
+    parseint: (val) => {
+        const safeParseNumber = (fn) => (val) => {
+            const result = fn(val);
+            return isNaN(result) ? null : result;
+        };
+        return safeParseNumber((val) => parseInt(val, 10))(val);
+    },
+    parsefloat: (val) => {
+        const safeParseNumber = (fn) => (val) => {
+            const result = fn(val);
+            return isNaN(result) ? null : result;
+        };
+        return safeParseNumber(parseFloat)(val);
+    },
+    toisodate: (val) => {
+        const safeDateOperation = (fn) => (val) => {
+            const date = new Date(val);
+            return isNaN(date.getTime()) ? null : fn(date);
+        };
+        return safeDateOperation((date) => date.toISOString())(val);
+    },
+    timeoffset: (val) => {
+        const safeDateOperation = (fn) => (val) => {
+            const date = new Date(val);
+            return isNaN(date.getTime()) ? null : fn(date);
+        };
+        return safeDateOperation((date) => date.getTimezoneOffset())(val);
+    },
+    tostring: (val) => {
+        const safeOperation =
+            (fn) =>
+            (...args) => {
+                try {
+                    return fn(...args);
+                } catch (e) {
+                    return null;
+                }
+            };
+        return safeOperation((val) => val.toString())(val);
+    },
+    urlencode: encodeURI,
+    urldecode: (val) => {
+        const safeOperation =
+            (fn) =>
+            (...args) => {
+                try {
+                    return fn(...args);
+                } catch (e) {
+                    return null;
+                }
+            };
+        return safeOperation(decodeURI)(val);
+    },
     replacestr: (str, from, to) => str.replace(from, to),
-    replaceregexp: (str, reg, to) => str.replace(new RegExp(reg), to),
-    replaceallstr: (str, from, to) => str.replaceAll(from, to),
-    replaceallregexp: (str, reg, to) => str.replaceAll(new RegExp(reg, 'g'), to),
+    replaceregexp: (str, reg, to) => {
+        const safeOperation =
+            (fn) =>
+            (...args) => {
+                try {
+                    return fn(...args);
+                } catch (e) {
+                    return null;
+                }
+            };
+        return safeOperation((str, reg, to) => str.replace(new RegExp(reg), to))(str, reg, to);
+    },
+    replaceallregexp: (str, reg, to) => {
+        const safeOperation =
+            (fn) =>
+            (...args) => {
+                try {
+                    return fn(...args);
+                } catch (e) {
+                    return null;
+                }
+            };
+        return safeOperation((str, reg, to) => str.replace(new RegExp(reg, 'g'), to))(str, reg, to);
+    },
     split: (str, ch) => str.split(ch),
     joinarrtostr: (arr, ch) => arr.join(ch),
     concatarr: (arr, arr2) => arr.concat(arr2),
     mapper: (val, values, choices) => choices[values.findIndex((target) => target === val)],
     thmapper: (val, values, choices) =>
-        choices[
-            values.reduce((acc, curr, i) => (acc === 0 || acc ? acc : val <= curr ? (acc = i) : (acc = null)), null)
-        ],
+        choices[values.reduce((acc, curr, i) => (acc !== null ? acc : val <= curr ? i : null), null)],
     bitwisemask: (i, mask, op, shf) =>
         (op === '&' ? parseInt(i) & mask : op === '|' ? parseInt(i) | mask : op === '^' ? parseInt(i) ^ mask : i) >>
         shf,
     slice: (arr, init, end) => arr.slice(init, end),
-    addset: (arr, x) => {
-        return Array.from(new Set(arr).add(x));
-    },
+    addset: (arr, x) => Array.from(new Set(arr).add(x)),
     removeset: (arr, x) => {
-        let s = new Set(arr);
+        const s = new Set(arr);
         s.delete(x);
         return Array.from(s);
     },
     touppercase: (val) => String(val).toUpperCase(),
     tolowercase: (val) => String(val).toLowerCase(),
-    floor: (val) => Math.floor(val),
-    ceil: (val) => Math.ceil(val),
-    round: (val) => Math.round(val),
-    tofixed: (val, decimals) => Number.parseFloat(val).toFixed(decimals),
-    gettime: (d) => new Date(d).getTime(),
-    toisostring: (d) => new Date(d).toISOString(),
-    // https://developer.mozilla.org/es/docs/Web/JavaScript/Reference/Global_Objects/Date/toLocaleString
-    localestring: (d, timezone, options) => new Date(d).toLocaleString(timezone, options),
-    now: () => Date.now(),
-    hextostring: (val) =>
-        new TextDecoder().decode(new Uint8Array(val.match(/.{1,2}/g).map((byte) => parseInt(byte, 16)))),
-    valuePicker: (val,pick) => Object.entries(val).filter(([, v]) => v === pick).map(([k,]) => k),
-    valuePickerMulti: (val,pick) => Object.entries(val).filter(([, v]) => pick.includes(v)).map(([k,]) => k)
+    floor: Math.floor,
+    ceil: Math.ceil,
+    round: Math.round,
+    tofixed: (val, decimals) => {
+        const num = Number.parseFloat(val);
+        const dec = Number.parseInt(decimals);
+        return isNaN(num) || isNaN(dec) ? null : num.toFixed(dec);
+    },
+    gettime: (val) => {
+        const safeDateOperation = (fn) => (val) => {
+            const date = new Date(val);
+            return isNaN(date.getTime()) ? null : fn(date);
+        };
+        return safeDateOperation((date) => date.getTime())(val);
+    },
+    toisostring: (val) => {
+        const safeDateOperation = (fn) => (val) => {
+            const date = new Date(val);
+            return isNaN(date.getTime()) ? null : fn(date);
+        };
+        return safeDateOperation((date) => date.toISOString())(val);
+    },
+    localestring: (d, timezone, options) => {
+        const safeOperation =
+            (fn) =>
+            (...args) => {
+                try {
+                    return fn(...args);
+                } catch (e) {
+                    return null;
+                }
+            };
+        return safeOperation((d, timezone, options) => new Date(d).toLocaleString(timezone, options))(
+            d,
+            timezone,
+            options
+        );
+    },
+    now: Date.now,
+    hextostring: (val) => {
+        const safeOperation =
+            (fn) =>
+            (...args) => {
+                try {
+                    return fn(...args);
+                } catch (e) {
+                    return null;
+                }
+            };
+        return safeOperation((val) => {
+            if (typeof val !== 'string' || !/^[0-9a-fA-F]+$/.test(val) || val.length % 2 !== 0) {
+                return null;
+            }
+            return new TextDecoder().decode(new Uint8Array(val.match(/.{1,2}/g).map((byte) => parseInt(byte, 16))));
+        })(val);
+    },
+    valuePicker: (val, pick) =>
+        Object.entries(val)
+            .filter(([, v]) => v === pick)
+            .map(([k]) => k),
+    valuePickerMulti: (val, pick) =>
+        Object.entries(val)
+            .filter(([, v]) => pick.includes(v))
+            .map(([k]) => k)
 };
 
 exports.map = map;

package/lib/model/Command.js

@@ -31,11 +31,20 @@ const Command = new Schema({
     value: Object,
     service: { type: String, lowercase: true },
     subservice: String,
+    execTs: { type: Date },
+    status: String,
+    info: String,
+    onDelivered: Object,
+    onOk: Object,
+    onError: Object,
+    onInfo: Object,
+    cmdExecution: Boolean,
+    dateExpiration: { type: Date },
     creationDate: { type: Date, default: Date.now }
 });
 
-function load(db) {
-    module.exports.model = db.model('Command', Command);
+function load() {
+    module.exports.model = mongoose.model('Command', Command);
     module.exports.internalSchema = Command;
 }
 

package/lib/model/Device.js

@@ -53,11 +53,16 @@ const Device = new Schema({
     autoprovision: Boolean,
     explicitAttrs: Group.ExplicitAttrsType,
     ngsiVersion: String,
-    payloadType: String
+    payloadType: String,
+    useCBflowControl: Boolean,
+    storeLastMeasure: Boolean,
+    lastMeasure: Object,
+    oldCtxt: Object
 });
 
-function load(db) {
-    module.exports.model = db.model('Device', Device);
+function load() {
+    Device.index({ service: 1, subservice: 1, apikey: 1, id: 1 }, { unique: true });
+    module.exports.model = mongoose.model('Device', Device);
     module.exports.internalSchema = Device;
 }
 

package/lib/model/Group.js

@@ -65,12 +65,14 @@ const Group = new Schema({
     defaultEntityNameConjunction: String,
     ngsiVersion: String,
     entityNameExp: String,
-    payloadType: String
+    payloadType: String,
+    useCBflowControl: Boolean,
+    storeLastMeasure: Boolean
 });
 
-function load(db) {
+function load() {
     Group.index({ apikey: 1, resource: 1 }, { unique: true });
-    module.exports.model = db.model('Group', Group);
+    module.exports.model = mongoose.model('Group', Group);
     module.exports.internalSchema = Group;
 }