iotagent-node-lib 4.5.0 → 4.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (124) hide show
  1. package/.github/workflows/ci.yml +0 -1
  2. package/Changelog +12 -0
  3. package/README.md +67 -272
  4. package/config.js +3 -1
  5. package/doc/README.md +1 -1
  6. package/doc/admin.md +40 -18
  7. package/doc/api.md +532 -136
  8. package/doc/deprecated.md +4 -0
  9. package/doc/devel/architecture.md +5 -135
  10. package/doc/devel/development.md +224 -12
  11. package/doc/getting-started.md +114 -53
  12. package/doc/requirements.txt +1 -1
  13. package/doc/roadmap.md +5 -5
  14. package/docker/Mosquitto/Dockerfile +2 -2
  15. package/docker/Mosquitto/README.md +14 -11
  16. package/lib/commonConfig.js +21 -2
  17. package/lib/constants.js +3 -0
  18. package/lib/fiware-iotagent-lib.js +12 -15
  19. package/lib/jexlTranformsMap.js +3 -1
  20. package/lib/model/Command.js +2 -2
  21. package/lib/model/Device.js +7 -3
  22. package/lib/model/Group.js +5 -3
  23. package/lib/model/dbConn.js +53 -115
  24. package/lib/services/commands/commandRegistryMongoDB.js +115 -75
  25. package/lib/services/common/alarmManagement.js +3 -0
  26. package/lib/services/common/iotManagerService.js +3 -1
  27. package/lib/services/devices/deviceRegistryMemory.js +36 -0
  28. package/lib/services/devices/deviceRegistryMongoDB.js +160 -87
  29. package/lib/services/devices/deviceService.js +33 -3
  30. package/lib/services/devices/devices-NGSI-v2.js +6 -1
  31. package/lib/services/groups/groupRegistryMongoDB.js +120 -83
  32. package/lib/services/groups/groupService.js +1 -1
  33. package/lib/services/ngsi/entities-NGSI-LD.js +320 -570
  34. package/lib/services/ngsi/entities-NGSI-v2.js +51 -3
  35. package/lib/services/ngsi/ngsiService.js +34 -1
  36. package/lib/services/northBound/deviceGroupAdministrationServer.js +42 -6
  37. package/lib/services/northBound/deviceProvisioningServer.js +12 -4
  38. package/lib/services/northBound/northboundServer.js +2 -0
  39. package/lib/services/stats/statsRegistry.js +128 -101
  40. package/lib/templates/createDevice.json +0 -24
  41. package/lib/templates/createDeviceLax.json +0 -23
  42. package/lib/templates/deviceGroup.json +1 -25
  43. package/lib/templates/updateDevice.json +12 -24
  44. package/lib/templates/updateDeviceLax.json +12 -23
  45. package/package.json +5 -5
  46. package/scripts/legacy_expression_tool/README.md +0 -1
  47. package/test/functional/README.md +22 -17
  48. package/test/functional/config-test.js +3 -2
  49. package/test/functional/functional-tests-runner.js +9 -4
  50. package/test/functional/functional-tests.js +4 -4
  51. package/test/functional/testCases.js +245 -4
  52. package/test/functional/testUtils.js +2 -2
  53. package/test/unit/examples/deviceProvisioningRequests/provisionFullDevice.json +1 -13
  54. package/test/unit/examples/groupProvisioningRequests/multipleConfigGroupsCreation.json +44 -0
  55. package/test/unit/examples/groupProvisioningRequests/provisionDuplicateConfigGroup.json +35 -0
  56. package/test/unit/examples/groupProvisioningRequests/provisionFullConfigGroup.json +36 -0
  57. package/test/unit/examples/groupProvisioningRequests/provisionFullConfigGroupAlternate.json +36 -0
  58. package/test/unit/examples/groupProvisioningRequests/provisionFullGroup.json +1 -0
  59. package/test/unit/general/config-multi-core-test.js +1 -2
  60. package/test/unit/general/contextBrokerKeystoneSecurityAccess-test.js +5 -4
  61. package/test/unit/general/deviceService-test.js +106 -3
  62. package/test/unit/general/statistics-service_test.js +1 -74
  63. package/test/unit/memoryRegistry/deviceRegistryMemory_test.js +6 -5
  64. package/test/unit/mongodb/mongodb-configGroup-registry-test.js +452 -0
  65. package/test/unit/mongodb/mongodb-connectionoptions-test.js +9 -42
  66. package/test/unit/mongodb/mongodb-group-registry-test.js +34 -33
  67. package/test/unit/mongodb/mongodb-service-registry-test.js +477 -0
  68. package/test/unit/ngsi-ld/examples/contextRequests/updateContextExpressionPlugin1a.json +4 -4
  69. package/test/unit/ngsi-ld/examples/contextRequests/updateContextExpressionPlugin2.json +22 -22
  70. package/test/unit/ngsi-ld/examples/contextRequests/updateContextExpressionPlugin29.json +4 -4
  71. package/test/unit/ngsi-ld/examples/contextRequests/updateContextExpressionPlugin32.json +14 -15
  72. package/test/unit/ngsi-ld/examples/contextRequests/updateContextMultientityPlugin1.json +23 -23
  73. package/test/unit/ngsi-ld/examples/contextRequests/updateContextMultientityPlugin15.json +0 -5
  74. package/test/unit/ngsi-ld/examples/contextRequests/updateContextMultientityPlugin4.json +11 -16
  75. package/test/unit/ngsi-ld/examples/contextRequests/updateContextMultientityPlugin5.json +23 -28
  76. package/test/unit/ngsi-ld/examples/contextRequests/updateContextMultientityPlugin6.json +8 -13
  77. package/test/unit/ngsi-ld/examples/contextRequests/updateContextMultientityPlugin7.json +0 -5
  78. package/test/unit/ngsi-ld/examples/contextRequests/updateContextMultientityPlugin8.json +24 -29
  79. package/test/unit/ngsi-ld/examples/contextRequests/updateContextMultientityTimestampPlugin2.json +12 -17
  80. package/test/unit/ngsi-ld/examples/contextRequests/updateContextStaticLinkedAttributes.json +12 -10
  81. package/test/unit/ngsi-ld/expressions/jexlBasedTransformations-test.js +1 -104
  82. package/test/unit/ngsi-ld/general/config-jsonld-contexts-test.js +1 -2
  83. package/test/unit/ngsi-ld/plugins/multientity-plugin_test.js +4 -5
  84. package/test/unit/ngsi-ld/provisioning/listProvisionedDevices-test.js +0 -4
  85. package/test/unit/ngsi-mixed/provisioning/ngsi-versioning-test.js +8 -5
  86. package/test/unit/ngsiv2/expressions/jexlBasedTransformations-test.js +42 -41
  87. package/test/unit/ngsiv2/general/contextBrokerOAuthSecurityAccess-test.js +11 -10
  88. package/test/unit/ngsiv2/general/deviceService-test.js +98 -4
  89. package/test/unit/ngsiv2/general/https-support-test.js +1 -1
  90. package/test/unit/ngsiv2/general/iotam-autoregistration-test.js +195 -0
  91. package/test/unit/ngsiv2/lazyAndCommands/active-devices-attribute-update-test.js +4 -3
  92. package/test/unit/ngsiv2/lazyAndCommands/command-test.js +6 -5
  93. package/test/unit/ngsiv2/lazyAndCommands/lazy-devices-test.js +17 -16
  94. package/test/unit/ngsiv2/lazyAndCommands/polling-commands-test.js +10 -18
  95. package/test/unit/ngsiv2/ngsiService/active-devices-test.js +21 -20
  96. package/test/unit/ngsiv2/ngsiService/staticAttributes-test.js +8 -7
  97. package/test/unit/ngsiv2/plugins/alias-plugin_test.js +12 -11
  98. package/test/unit/ngsiv2/plugins/custom-plugin_test.js +3 -2
  99. package/test/unit/ngsiv2/plugins/multientity-plugin_test.js +28 -27
  100. package/test/unit/ngsiv2/provisioning/device-group-api-test.js +265 -4
  101. package/test/unit/ngsiv2/provisioning/device-provisioning-api_test.js +12 -11
  102. package/test/unit/ngsiv2/provisioning/device-provisioning-configGroup-api_test.js +1190 -0
  103. package/test/unit/ngsiv2/provisioning/device-registration_test.js +5 -4
  104. package/test/unit/ngsiv2/provisioning/listProvisionedDevices-test.js +6 -9
  105. package/test/unit/ngsiv2/provisioning/provisionDeviceMultientity-test.js +1 -1
  106. package/test/unit/ngsiv2/provisioning/removeProvisionedDevice-test.js +5 -4
  107. package/test/unit/ngsiv2/provisioning/updateProvisionedDevices-test.js +8 -7
  108. package/test/unit/statsRegistry/openmetrics-test.js +167 -0
  109. package/lib/templates/queryContext.json +0 -25
  110. package/test/unit/examples/deviceProvisioningRequests/provisionBidirectionalDevice.json +0 -35
  111. package/test/unit/examples/deviceProvisioningRequests/provisionDeviceBidirectionalGroup.json +0 -17
  112. package/test/unit/examples/groupProvisioningRequests/bidirectionalGroup.json +0 -31
  113. package/test/unit/general/statistics-persistence_test.js +0 -121
  114. package/test/unit/ngsi-ld/examples/contextRequests/createBidirectionalDevice.json +0 -17
  115. package/test/unit/ngsi-ld/examples/contextRequests/updateContextProcessTimestamp.json +0 -12
  116. package/test/unit/ngsi-ld/examples/subscriptionRequests/bidirectionalNotification.json +0 -13
  117. package/test/unit/ngsi-ld/examples/subscriptionRequests/bidirectionalNotificationWithDatasetId.json +0 -21
  118. package/test/unit/ngsi-ld/examples/subscriptionRequests/bidirectionalNotificationWithMetadata.json +0 -17
  119. package/test/unit/ngsi-ld/examples/subscriptionRequests/bidirectionalSubscriptionRequest.json +0 -23
  120. package/test/unit/ngsi-ld/plugins/timestamp-processing-plugin_test.js +0 -132
  121. package/test/unit/ngsiv2/examples/contextRequests/createBidirectionalDevice.json +0 -8
  122. package/test/unit/ngsiv2/examples/subscriptionRequests/bidirectionalNotification.json +0 -13
  123. package/test/unit/ngsiv2/examples/subscriptionRequests/bidirectionalNotificationWithMetadata.json +0 -19
  124. package/test/unit/ngsiv2/examples/subscriptionRequests/bidirectionalSubscriptionRequest.json +0 -24
@@ -261,7 +261,7 @@ function sendQueryValueNgsi2(entityName, attributes, typeInformation, token, cal
261
261
  * @param {String} token User token to identify against the PEP Proxies (optional).
262
262
  */
263
263
  function sendUpdateValueNgsi2(entityName, originMeasures, originTypeInformation, token, callback) {
264
- //aux function used to builf JEXL context.
264
+ //aux functions used to builf JEXL context.
265
265
  //it returns a flat object from an Attr array
266
266
  function reduceAttrToPlainObject(attrs, initObj = {}) {
267
267
  if (attrs !== undefined && Array.isArray(attrs)) {
@@ -273,6 +273,27 @@ function sendUpdateValueNgsi2(entityName, originMeasures, originTypeInformation,
273
273
  return initObj;
274
274
  }
275
275
  }
276
+ //it returns a metadata object using the same structure described
277
+ // at https://github.com/telefonicaid/fiware-orion/blob/master/doc/manuals/orion-api.md#metadata-support
278
+ function reduceMetadataAttrToPlainObject(attrs, initObj = {}) {
279
+ if (attrs !== undefined && Array.isArray(attrs)) {
280
+ return attrs.reduce((result, item) => {
281
+ if (result['metadata'] === undefined) {
282
+ result['metadata'] = {};
283
+ }
284
+ if (item.metadata !== undefined) {
285
+ result['metadata'][item.name] = {};
286
+ for (var meta in item.metadata) {
287
+ result['metadata'][item.name][meta] = item.metadata[meta].value;
288
+ }
289
+ }
290
+ return result;
291
+ }, initObj);
292
+ } else {
293
+ return initObj;
294
+ }
295
+ }
296
+
276
297
  //Make a clone and overwrite
277
298
  let idTypeSSSList = pluginUtils.getIdTypeServSubServiceFromDevice(originTypeInformation);
278
299
 
@@ -321,7 +342,15 @@ function sendUpdateValueNgsi2(entityName, originMeasures, originTypeInformation,
321
342
  jexlctxt = reduceAttrToPlainObject(typeInformation.staticAttributes, jexlctxt);
322
343
  //id type Service and Subservice
323
344
  jexlctxt = reduceAttrToPlainObject(idTypeSSSList, jexlctxt);
324
-
345
+ //metadata attributes
346
+ jexlctxt = reduceMetadataAttrToPlainObject(typeInformation.active, jexlctxt);
347
+ //metadata static attributes
348
+ jexlctxt = reduceMetadataAttrToPlainObject(typeInformation.staticAttributes, jexlctxt);
349
+
350
+ //recover oldctxt
351
+ if (typeInformation.oldCtxt) {
352
+ jexlctxt.oldCtxt = typeInformation.oldCtxt;
353
+ }
325
354
  logger.debug(
326
355
  context,
327
356
  'sendUpdateValueNgsi2 loop with: entityName=%s, measures=%j, typeInformation=%j, initial jexlContext=%j, timestamp=%j',
@@ -511,6 +540,16 @@ function sendUpdateValueNgsi2(entityName, originMeasures, originTypeInformation,
511
540
  }
512
541
  newAttrMeta['value'] = metaValueExpression;
513
542
  currentAttr.metadata[metaKey] = newAttrMeta;
543
+
544
+ //RE-Populate de JEXLcontext
545
+ // It is possible metadata is still not in ctxt
546
+ if (!jexlctxt.metadata) {
547
+ jexlctxt.metadata = {};
548
+ }
549
+ if (!jexlctxt.metadata[currentAttr.name]) {
550
+ jexlctxt.metadata[currentAttr.name] = {};
551
+ }
552
+ jexlctxt.metadata[currentAttr.name][currentAttr.metadata[metaKey]] = metaValueExpression;
514
553
  }
515
554
  }
516
555
  }
@@ -617,9 +656,15 @@ function sendUpdateValueNgsi2(entityName, originMeasures, originTypeInformation,
617
656
  }
618
657
  }
619
658
  }
659
+ //Add jexlctxt to typeInformation without oldCtxt
660
+ const oldCJexlctxt = { ...jexlctxt };
661
+ delete oldCJexlctxt.oldCtxt;
662
+ originTypeInformation['oldCtxt'] = oldCJexlctxt;
620
663
  } // end for (let measures of originMeasures)
621
-
622
664
  let url = '/v2/op/update';
665
+ if (originTypeInformation.useCBflowControl) {
666
+ url += '?options=flowControl';
667
+ }
623
668
  let options = NGSIUtils.createRequestObject(url, originTypeInformation, token);
624
669
  options.json = payload;
625
670
 
@@ -645,6 +690,9 @@ function sendUpdateValueNgsi2(entityName, originMeasures, originTypeInformation,
645
690
  if (!multi) {
646
691
  // recreate options object to use single entity update
647
692
  url = '/v2/entities?options=upsert';
693
+ if (originTypeInformation.useCBflowControl) {
694
+ url += ',flowControl';
695
+ }
648
696
  options = NGSIUtils.createRequestObject(url, originTypeInformation, token);
649
697
  delete payload.actionType;
650
698
 
@@ -26,6 +26,8 @@
26
26
 
27
27
  const async = require('async');
28
28
  const apply = async.apply;
29
+ const statsRegistry = require('../stats/statsRegistry');
30
+ const deviceService = require('../devices/deviceService');
29
31
  const intoTrans = require('../common/domain').intoTrans;
30
32
  const fillService = require('./../common/domain').fillService;
31
33
  const errors = require('../../errors');
@@ -67,7 +69,38 @@ function init() {
67
69
  * @param {String} token User token to identify against the PEP Proxies (optional).
68
70
  */
69
71
  function sendUpdateValue(entityName, attributes, typeInformation, token, callback) {
70
- entityHandler.sendUpdateValue(entityName, attributes, typeInformation, token, callback);
72
+ const newCallback = statsRegistry.withStats('updateEntityRequestsOk', 'updateEntityRequestsError', callback);
73
+ const additionalCallback = (data, next) => {
74
+ if (typeInformation.oldCtxt) {
75
+ logger.debug(context, 'StoreOldCtxt %j', typeInformation.oldCtxt);
76
+ deviceService.storeDeviceField('oldCtxt', typeInformation.oldCtxt, typeInformation, function () {
77
+ next(null, data);
78
+ });
79
+ } else {
80
+ next(null, data);
81
+ }
82
+ };
83
+ const wrappedNewCallback = (err, result) => {
84
+ if (err) {
85
+ newCallback(err);
86
+ } else {
87
+ additionalCallback(result, (additionalErr, modifiedResult) => {
88
+ if (additionalErr) {
89
+ newCallback(additionalErr);
90
+ }
91
+ newCallback(null, modifiedResult || result);
92
+ });
93
+ }
94
+ };
95
+ // check config about store last measure
96
+ if (typeInformation.storeLastMeasure) {
97
+ logger.debug(context, 'StoreLastMeasure for %j', typeInformation);
98
+ deviceService.storeDeviceField('lastMeasure', attributes, typeInformation, function () {
99
+ return entityHandler.sendUpdateValue(entityName, attributes, typeInformation, token, wrappedNewCallback);
100
+ });
101
+ } else {
102
+ entityHandler.sendUpdateValue(entityName, attributes, typeInformation, token, wrappedNewCallback);
103
+ }
71
104
  }
72
105
 
73
106
  /**
@@ -29,13 +29,19 @@ const restUtils = require('./restUtils');
29
29
  const groupService = require('./../groups/groupService');
30
30
  const async = require('async');
31
31
  const apply = async.apply;
32
- const templateGroup = require('../../templates/deviceGroup.json');
32
+ const templateGroupService = require('../../templates/deviceGroup.json');
33
33
  let configurationHandler;
34
34
  let removeConfigurationHandler;
35
35
  let configurationMiddleware = [];
36
36
  const _ = require('underscore');
37
37
  const mandatoryHeaders = ['fiware-service', 'fiware-servicepath'];
38
38
  const mandatoryParameters = ['resource', 'apikey'];
39
+ const constants = require('../../constants');
40
+
41
+ // Create new template for configuration groups replacing services by CONFIGGROUP_TERM
42
+ const templateGroup = JSON.parse(JSON.stringify(templateGroupService));
43
+ templateGroup.properties[constants.CONFIGGROUP_TERM] = templateGroup.properties.services;
44
+ delete templateGroup.properties.services;
39
45
 
40
46
  const apiToInternal = {
41
47
  entity_type: 'type',
@@ -118,6 +124,8 @@ function applyConfigurationMiddlewares(newConfiguration, callback) {
118
124
  * @param {Function} next Invokes the next middleware in the chain.
119
125
  */
120
126
  function handleCreateDeviceGroup(req, res, next) {
127
+ req.body = checkAndModifyGroupRecv(req._parsedUrl.pathname, req.body); // #FIXME1649: This line should be removed when /iot/services support is removed
128
+
121
129
  for (let i = 0; i < req.body.services.length; i++) {
122
130
  req.body.services[i] = applyMap(apiToInternal, req.body.services[i]);
123
131
  req.body.services[i].service = req.headers['fiware-service'];
@@ -161,8 +169,7 @@ function handleListDeviceGroups(req, res, next) {
161
169
  } else {
162
170
  translatedGroup = applyMap(internalToApi, group);
163
171
  }
164
-
165
- res.status(200).send(translatedGroup);
172
+ res.status(200).send(checkAndModifyGroupResp(req._parsedUrl.pathname, translatedGroup)); // #FIXME1649: Response should not use checkAndModifyGroupResp function when /iot/services support is removed
166
173
  }
167
174
  };
168
175
 
@@ -263,25 +270,37 @@ function handleDeleteDeviceGroups(req, res, next) {
263
270
  *
264
271
  * @param {Object} router Express request router object.
265
272
  */
273
+ // #FIXME1649: Routes using /iot/services path should be removed when support for it is removed
266
274
  function loadContextRoutes(router) {
267
275
  router.post(
268
276
  '/iot/services',
269
277
  restUtils.checkRequestAttributes('headers', mandatoryHeaders),
278
+ restUtils.checkBody(templateGroupService),
279
+ handleCreateDeviceGroup
280
+ );
281
+
282
+ router.post(
283
+ '/iot/groups',
284
+ restUtils.checkRequestAttributes('headers', mandatoryHeaders),
270
285
  restUtils.checkBody(templateGroup),
271
286
  handleCreateDeviceGroup
272
287
  );
273
288
 
274
- router.get('/iot/services', restUtils.checkRequestAttributes('headers', mandatoryHeaders), handleListDeviceGroups);
289
+ router.get(
290
+ ['/iot/services', '/iot/groups'],
291
+ restUtils.checkRequestAttributes('headers', mandatoryHeaders),
292
+ handleListDeviceGroups
293
+ );
275
294
 
276
295
  router.put(
277
- '/iot/services',
296
+ ['/iot/services', '/iot/groups'],
278
297
  restUtils.checkRequestAttributes('headers', mandatoryHeaders),
279
298
  restUtils.checkRequestAttributes('query', mandatoryParameters),
280
299
  handleModifyDeviceGroups
281
300
  );
282
301
 
283
302
  router.delete(
284
- '/iot/services',
303
+ ['/iot/services', '/iot/groups'],
285
304
  restUtils.checkRequestAttributes('headers', mandatoryHeaders),
286
305
  restUtils.checkRequestAttributes('query', mandatoryParameters),
287
306
  handleDeleteDeviceGroups
@@ -306,6 +325,23 @@ function clear(callback) {
306
325
  callback();
307
326
  }
308
327
 
328
+ // #FIXME1649: This function should be removed when /iot/services support is removed
329
+ function checkAndModifyGroupResp(route, payload) {
330
+ if (route === '/iot/groups') {
331
+ payload['groups'] = payload['services'];
332
+ delete payload['services'];
333
+ }
334
+ return payload;
335
+ }
336
+ // #FIXME1649: This function should be removed when /iot/services support is removed
337
+ function checkAndModifyGroupRecv(route, payload) {
338
+ if (route === '/iot/groups') {
339
+ payload['services'] = payload['groups'];
340
+ delete payload['groups'];
341
+ }
342
+ return payload;
343
+ }
344
+
309
345
  exports.loadContextRoutes = loadContextRoutes;
310
346
  exports.setConfigurationHandler = setConfigurationHandler;
311
347
  exports.setRemoveConfigurationHandler = setRemoveConfigurationHandler;
@@ -64,7 +64,10 @@ const provisioningAPITranslation = {
64
64
  explicitAttrs: 'explicitAttrs',
65
65
  ngsiVersion: 'ngsiVersion',
66
66
  entityNameExp: 'entityNameExp',
67
- payloadType: 'payloadType'
67
+ payloadType: 'payloadType',
68
+ useCBflowControl: 'useCBflowControl',
69
+ storeLastMeasure: 'storeLastMeasure',
70
+ lastMeasure: 'lastMeasure'
68
71
  };
69
72
 
70
73
  /**
@@ -143,7 +146,10 @@ function handleProvision(req, res, next) {
143
146
  autoprovision: body.autoprovision,
144
147
  explicitAttrs: body.explicitAttrs,
145
148
  ngsiVersion: body.ngsiVersion,
146
- payloadType: body.payloadType
149
+ payloadType: body.payloadType,
150
+ useCBflowControl: body.useCBflowControl,
151
+ storeLastMeasure: body.storeLastMeasure,
152
+ lastMeasure: body.lastMeasure
147
153
  });
148
154
  }
149
155
 
@@ -182,7 +188,6 @@ function attributeToProvisioningAPIFormat(attribute) {
182
188
  type: attribute.type,
183
189
  expression: attribute.expression,
184
190
  skipValue: attribute.skipValue,
185
- reverse: attribute.reverse,
186
191
  entity_name: attribute.entity_name,
187
192
  entity_type: attribute.entity_type,
188
193
  mqtt: attribute.mqtt,
@@ -221,7 +226,10 @@ function toProvisioningAPIFormat(device) {
221
226
  autoprovision: device.autoprovision,
222
227
  explicitAttrs: device.explicitAttrs,
223
228
  ngsiVersion: device.ngsiVersion,
224
- payloadType: device.payloadType
229
+ payloadType: device.payloadType,
230
+ useCBflowControl: device.useCBflowControl,
231
+ storeLastMeasure: device.storeLastMeasure,
232
+ lastMeasure: device.lastMeasure
225
233
  };
226
234
  }
227
235
 
@@ -33,6 +33,7 @@ const intoTrans = domainUtils.intoTrans;
33
33
  const deviceProvisioning = require('./deviceProvisioningServer');
34
34
  const deviceUpdating = require('./deviceProvisioningServer');
35
35
  const groupProvisioning = require('./deviceGroupAdministrationServer');
36
+ const statsRegistry = require('../stats/statsRegistry');
36
37
  const logger = require('logops');
37
38
  const context = {
38
39
  op: 'IoTAgentNGSI.NorthboundServer'
@@ -83,6 +84,7 @@ function start(config, callback) {
83
84
  northboundServer.router.get('/version', middlewares.retrieveVersion);
84
85
  northboundServer.router.put('/admin/log', middlewares.changeLogLevel);
85
86
  northboundServer.router.get('/admin/log', middlewares.getLogLevel);
87
+ northboundServer.router.get('/metrics', statsRegistry.openmetricsHandler);
86
88
 
87
89
  northboundServer.app.use(baseRoot, northboundServer.router);
88
90
  contextServer.loadContextRoutes(northboundServer.router);
@@ -23,16 +23,9 @@
23
23
 
24
24
  /* eslint-disable no-prototype-builtins */
25
25
 
26
- const async = require('async');
27
26
  const _ = require('underscore');
28
- const apply = async.apply;
29
27
  const logger = require('logops');
30
- const config = require('../../commonConfig');
31
- const dbService = require('../../model/dbConn');
32
28
  let globalStats = {};
33
- let currentStats = {};
34
- let timerActions = [];
35
- let timerHandler;
36
29
  const statsContext = {
37
30
  op: 'IoTAgentNGSI.TimedStats'
38
31
  };
@@ -45,30 +38,14 @@ const statsContext = {
45
38
  * @param {Number} value Value to be added to the total.
46
39
  */
47
40
  function add(key, value, callback) {
48
- if (currentStats[key]) {
49
- currentStats[key] += value;
50
- } else {
51
- currentStats[key] = value;
52
- }
53
-
54
41
  if (globalStats[key]) {
55
42
  globalStats[key] += value;
56
43
  } else {
57
44
  globalStats[key] = value;
58
45
  }
59
-
60
46
  callback(null);
61
47
  }
62
48
 
63
- /**
64
- * Get the current value of a particular stat.
65
- *
66
- * @param {String} key Name of the stat to retrive.
67
- */
68
- function getCurrent(key, callback) {
69
- callback(null, currentStats[key]);
70
- }
71
-
72
49
  /**
73
50
  * Get the global value of the selected attribute.
74
51
  *
@@ -85,112 +62,162 @@ function getAllGlobal(callback) {
85
62
  callback(null, globalStats);
86
63
  }
87
64
 
88
- /**
89
- * Get all the current stats currently stored in the repository.
90
- */
91
- function getAllCurrent(callback) {
92
- callback(null, currentStats);
93
- }
94
-
95
65
  /**
96
66
  * Loads the values passed as parameters into the global statistics repository.
97
67
  *
98
68
  * @param {Object} values Key-value map with the values to be load.
99
69
  */
100
70
  function globalLoad(values, callback) {
101
- globalStats = values;
102
- currentStats = {};
103
-
104
- for (const i in values) {
105
- if (values.hasOwnProperty(i)) {
106
- currentStats[i] = 0;
107
- }
108
- }
109
-
71
+ globalStats = _.clone(values);
110
72
  callback(null);
111
73
  }
112
74
 
113
75
  /**
114
- * Reset each of the current stats to value zero.
76
+ * Chooses the appropiate content type and version based on Accept header
77
+ *
78
+ * @param {String} accepts The accepts header
115
79
  */
116
- function resetCurrent(callback) {
117
- for (const i in currentStats) {
118
- if (currentStats.hasOwnProperty(i)) {
119
- currentStats[i] = 0;
80
+ function matchContentType(accepts) {
81
+ const requestedType = [];
82
+ const vlabel = 'version=';
83
+ const clabel = 'charset=';
84
+ const qlabel = 'q=';
85
+ for (const expression of accepts.split(',')) {
86
+ const parts = expression.split(';').map((part) => part.trim());
87
+ const mediaType = parts[0];
88
+ let version = null;
89
+ let charset = null;
90
+ let preference = null;
91
+ for (let part of parts.slice(1)) {
92
+ if (part.startsWith(vlabel)) {
93
+ version = part.substring(vlabel.length).trim();
94
+ } else if (part.startsWith(clabel)) {
95
+ charset = part.substring(clabel.length).trim();
96
+ } else if (part.startsWith(qlabel)) {
97
+ preference = parseFloat(part.substring(qlabel.length).trim());
98
+ }
120
99
  }
100
+ requestedType.push({
101
+ mediaType: mediaType,
102
+ version: version,
103
+ charset: charset,
104
+ preference: preference || 1.0
105
+ });
121
106
  }
122
-
123
- callback();
124
- }
125
-
126
- /**
127
- * Executes all the stored timer actions when a timer click is received.
128
- */
129
- function tickHandler() {
130
- process.nextTick(apply(async.series, timerActions));
131
- }
132
-
133
- /**
134
- * Adds a new timer action to the timerActions Array, activating the timer if it was not previously activated.
135
- *
136
- * @param {Function} handler Action to be executed. Should take two statistics objects and a callback.
137
- */
138
- function addTimerAction(handler, callback) {
139
- if (!timerHandler && config.getConfig().stats.interval) {
140
- timerHandler = setInterval(tickHandler, config.getConfig().stats.interval);
107
+ // If both text/plain and openmetrics are accepted,
108
+ // prefer openmetrics
109
+ const mediaTypePref = {
110
+ 'application/openmetrics-text': 1.0,
111
+ 'text/plain': 0.5
141
112
  }
142
-
143
- timerActions.push(apply(handler, currentStats, globalStats));
144
- callback();
113
+ // sort requests by priority descending
114
+ requestedType.sort(function (a, b) {
115
+ if (a.preference === b.preference) {
116
+ // same priority, sort by media type.
117
+ return (mediaTypePref[b.mediaType] || 0) - (mediaTypePref[a.mediaType] || 0);
118
+ }
119
+ return b.preference - a.preference;
120
+ });
121
+ for (const req of requestedType) {
122
+ switch(req.mediaType) {
123
+ case 'application/openmetrics-text':
124
+ req.version = req.version || '1.0.0';
125
+ req.charset = req.charset || 'utf-8';
126
+ if (
127
+ (req.version === '1.0.0' || req.version === '0.0.1') &&
128
+ (req.charset === 'utf-8')) {
129
+ return req;
130
+ }
131
+ break;
132
+ case 'text/plain':
133
+ case 'text/*':
134
+ case '*/*':
135
+ req.version = req.version || '0.0.4';
136
+ req.charset = req.charset || 'utf-8';
137
+ if (
138
+ (req.version === '0.0.4') &&
139
+ (req.charset === 'utf-8')) {
140
+ req.mediaType = 'text/plain';
141
+ return req;
142
+ }
143
+ break;
144
+ }
145
+ }
146
+ return null;
145
147
  }
146
148
 
147
149
  /**
148
- * Clear the actions array and stop the timers.
150
+ * Predefined http handler that returns current openmetrics data
149
151
  */
150
- function clearTimers(callback) {
151
- if (timerHandler) {
152
- clearInterval(timerHandler);
153
- timerHandler = undefined;
152
+ /* eslint-disable-next-line no-unused-vars */
153
+ function openmetricsHandler(req, res) {
154
+ // Content-Type:
155
+ // - For openmetrics collectors, it MUST BE 'application/openmetrics-text; version=1.0.0; charset=utf-8'. See:
156
+ // https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#overall-structure
157
+ // - For prometheus compatible collectors, it SHOULD BE 'text/plain; version=0.0.4; charset=utf-8'. See:
158
+ // https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md
159
+ // - Caveat: Some versions of prometheus have been observed to send multivalued Accept headers such as
160
+ // Accept: application/openmetrics-text; version=0.0.1,text/plain;version=0.0.4;q=0.5,*/*;q=0.1
161
+ let reqType = {
162
+ mediaType: 'application/openmetrics-text',
163
+ version: '1.0.0',
164
+ charset: 'utf-8'
154
165
  }
155
-
156
- timerActions = [];
157
- callback();
166
+ if (req.headers.accept) {
167
+ // WORKAROUND: express version 4 does not parse properly the openmetrics Accept header,
168
+ // it won't match the regular expressions supported by `express.accepts`.
169
+ // So we must parse these key-value pairs ourselves.
170
+ reqType = matchContentType(req.headers.accept);
171
+ if (reqType === null) {
172
+ logger.error(statsContext, 'Unsupported media type: %s', req.headers.accept);
173
+ res.status(406).send('Not Acceptable');
174
+ return;
175
+ }
176
+ }
177
+ const contentType = `${reqType.mediaType};version=${reqType.version};charset=${reqType.charset}`;
178
+ // The actual payload is the same for all supported content types
179
+ const metrics = [];
180
+ for (const key in globalStats) {
181
+ if (globalStats.hasOwnProperty(key)) {
182
+ metrics.push('# HELP ' + key + ' global metric for ' + key);
183
+ metrics.push('# TYPE ' + key + ' counter');
184
+ metrics.push(key + ' ' + globalStats[key]);
185
+ }
186
+ }
187
+ // Expositions MUST END WITH '#EOF'
188
+ // See https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md
189
+ metrics.push('# EOF');
190
+ res.set('Content-Type', contentType);
191
+ res.status(200).send(metrics.join('\n'));
158
192
  }
159
193
 
160
194
  /**
161
- * Predefined stats action that logs the stats to the standard log.
195
+ * Wraps a callback with stats, incrementing the given counters
196
+ * depending on the parameters passed to the callback:
162
197
  *
163
- * @param {Object} currentValues Current stat values.
164
- * @param {Object} globalValues Global stat values.
165
- */
166
- function logStats(currentValues, globalValues, callback) {
167
- logger.info(statsContext, 'Global stat values:\n%s\n', JSON.stringify(globalValues, null, 4));
168
- logger.info(statsContext, 'Current stat values:\n%s\n', JSON.stringify(currentValues, null, 4));
169
-
170
- resetCurrent(callback);
171
- }
172
-
173
- /**
174
- * Predefined action that persists the current value of the stats in the MongoDb instance.
198
+ * - If the callback receives an error, the errCounter is incremented.
199
+ * - If the callback receives no error, the okCounter is incremented.
175
200
  *
176
- * @param {Object} currentValues Current stat values.
177
- * @param {Object} globalValues Global stat values.
201
+ * @param {String} okCounter Name of the counter to increment on success.
202
+ * @param {String} errCounter Name of the counter to increment on error.
203
+ * @param {Function} callback Callback to wrap. It must be a function that can
204
+ * expect any number of parameters, but the first one must
205
+ * be an indication of the error occured, if any.
178
206
  */
179
- function mongodbPersistence(currentValues, globalValues, callback) {
180
- const statStamp = _.clone(globalValues);
181
-
182
- statStamp.timestamp = new Date().toISOString();
183
- dbService.db.collection('kpis').insertOne(statStamp, callback);
207
+ function withStats(okCounter, errCounter, callback) {
208
+ function accounting(...args) {
209
+ const counter = args.length > 0 && args[0] ? errCounter : okCounter;
210
+ add(counter, 1, function () {
211
+ callback(...args);
212
+ });
213
+ }
214
+ return accounting;
184
215
  }
185
216
 
186
217
  exports.add = add;
187
- exports.getCurrent = getCurrent;
188
218
  exports.getGlobal = getGlobal;
189
219
  exports.getAllGlobal = getAllGlobal;
190
- exports.getAllCurrent = getAllCurrent;
191
220
  exports.globalLoad = globalLoad;
192
- exports.resetCurrent = resetCurrent;
193
- exports.clearTimers = clearTimers;
194
- exports.addTimerAction = addTimerAction;
195
- exports.logStats = logStats;
196
- exports.mongodbPersistence = mongodbPersistence;
221
+ exports.withStats = withStats;
222
+ exports.openmetricsHandler = openmetricsHandler;
223
+ exports.matchContentType = matchContentType;
@@ -119,30 +119,6 @@
119
119
  "type": "string",
120
120
  "pattern": "^([^<>;'=\"]+)+$"
121
121
  },
122
- "reverse": {
123
- "description": "Define the attribute as bidirectional",
124
- "type": "array",
125
- "items": {
126
- "type": "object",
127
- "additionalProperties": false,
128
- "properties": {
129
- "object_id": {
130
- "description": "ID of the attribute in the device",
131
- "type": "string",
132
- "pattern": "^([^<>();'=\"]+)+$"
133
- },
134
- "type": {
135
- "description": "Type of the attribute in the target entity",
136
- "type": "string",
137
- "pattern": "^([^<>();'=\"]+)+$"
138
- },
139
- "expression": {
140
- "description": "Optional expression for measurement transformation",
141
- "type": "string"
142
- }
143
- }
144
- }
145
- },
146
122
  "metadata": {
147
123
  "description": "Attribute Metadata",
148
124
  "type": "object"
@@ -116,29 +116,6 @@
116
116
  "description": "Optional entity type for multientity updatess",
117
117
  "type": "string",
118
118
  "pattern": "^([^<>;'=\"]+)+$"
119
- },
120
- "reverse": {
121
- "description": "Define the attribute as bidirectional",
122
- "type": "array",
123
- "items": {
124
- "type": "object",
125
- "additionalProperties": false,
126
- "properties": {
127
- "object_id": {
128
- "description": "ID of the attribute in the device",
129
- "type": "string"
130
- },
131
- "type": {
132
- "description": "Type of the attribute in the target entity",
133
- "type": "string",
134
- "pattern": "^([^<>();'=\"]+)+$"
135
- },
136
- "expression": {
137
- "description": "Optional expression for measurement transformation",
138
- "type": "string"
139
- }
140
- }
141
- }
142
119
  }
143
120
  }
144
121
  }