iotagent-node-lib 3.2.0 → 3.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (184) hide show
  1. package/.github/ISSUE_TEMPLATE/bug_report.yml +134 -0
  2. package/.github/ISSUE_TEMPLATE/config.yml +16 -0
  3. package/.github/ISSUE_TEMPLATE/feature_request.yml +55 -0
  4. package/.github/advanced-issue-labeler.yml +30 -0
  5. package/.github/workflows/issue-labeler.yml +43 -0
  6. package/README.md +10 -11
  7. package/doc/README.md +16 -0
  8. package/doc/admin.md +565 -0
  9. package/doc/api.md +32 -85
  10. package/doc/deprecated.md +16 -10
  11. package/doc/{architecture.md → devel/architecture.md} +3 -3
  12. package/doc/{Contribution.md → devel/contribution-guidelines.md} +43 -35
  13. package/doc/devel/development.md +1879 -0
  14. package/doc/{northboundinteractions.md → devel/northboundinteractions.md} +18 -33
  15. package/doc/index.md +3 -5
  16. package/doc/requirements.txt +1 -1
  17. package/docker/Mosquitto/Dockerfile +1 -1
  18. package/docker/Mosquitto/README.md +1 -0
  19. package/lib/commonConfig.js +0 -5
  20. package/lib/fiware-iotagent-lib.js +1 -1
  21. package/lib/jexlTranformsMap.js +2 -1
  22. package/lib/model/Device.js +0 -1
  23. package/lib/model/Group.js +0 -1
  24. package/lib/model/dbConn.js +1 -7
  25. package/lib/plugins/jexlParser.js +1 -1
  26. package/lib/request-shim.js +2 -2
  27. package/lib/services/commands/commandService.js +1 -1
  28. package/lib/services/common/genericMiddleware.js +1 -1
  29. package/lib/services/common/iotManagerService.js +0 -1
  30. package/lib/services/devices/deviceRegistryMemory.js +2 -2
  31. package/lib/services/devices/deviceRegistryMongoDB.js +32 -19
  32. package/lib/services/devices/deviceService.js +44 -43
  33. package/lib/services/devices/devices-NGSI-LD.js +14 -2
  34. package/lib/services/devices/devices-NGSI-mixed.js +0 -2
  35. package/lib/services/devices/devices-NGSI-v2.js +23 -104
  36. package/lib/services/groups/groupService.js +1 -1
  37. package/lib/services/ngsi/entities-NGSI-LD.js +3 -3
  38. package/lib/services/ngsi/entities-NGSI-v2.js +28 -19
  39. package/lib/services/northBound/deviceProvisioningServer.js +14 -8
  40. package/lib/templates/createDevice.json +0 -4
  41. package/lib/templates/createDeviceLax.json +0 -4
  42. package/lib/templates/deviceGroup.json +1 -5
  43. package/lib/templates/updateDevice.json +4 -0
  44. package/lib/templates/updateDeviceLax.json +11 -0
  45. package/mkdocs.yml +6 -11
  46. package/package.json +3 -3
  47. package/scripts/legacy_expression_tool/README.md +280 -0
  48. package/scripts/legacy_expression_tool/legacy_expression_tool.py +423 -0
  49. package/scripts/legacy_expression_tool/requirements.txt +3 -0
  50. package/test/unit/examples/deviceProvisioningRequests/provisionMinimumDevice4.json +0 -1
  51. package/test/unit/general/contextBrokerKeystoneSecurityAccess-test.js +5 -15
  52. package/test/unit/mongodb/mongodb-registry-test.js +1 -1
  53. package/test/unit/ngsi-ld/general/contextBrokerOAuthSecurityAccess-test.js +66 -65
  54. package/test/unit/ngsi-ld/general/https-support-test.js +1 -1
  55. package/test/unit/ngsi-ld/lazyAndCommands/command-test.js +8 -7
  56. package/test/unit/ngsi-ld/lazyAndCommands/merge-patch-test.js +31 -30
  57. package/test/unit/ngsi-ld/lazyAndCommands/polling-commands-test.js +12 -11
  58. package/test/unit/ngsi-ld/ngsiService/subscriptions-test.js +41 -39
  59. package/test/unit/ngsi-ld/provisioning/device-provisioning-api_test.js +122 -122
  60. package/test/unit/ngsi-ld/provisioning/device-registration_test.js +28 -28
  61. package/test/unit/ngsi-ld/provisioning/device-update-registration_test.js +18 -17
  62. package/test/unit/ngsi-ld/provisioning/singleConfigurationMode-test.js +7 -7
  63. package/test/unit/ngsi-ld/provisioning/updateProvisionedDevices-test.js +8 -7
  64. package/test/unit/ngsi-mixed/provisioning/ngsi-versioning-test.js +33 -37
  65. package/test/unit/ngsiv2/examples/contextRequests/updateContext.json +2 -0
  66. package/test/unit/ngsiv2/examples/contextRequests/updateContext1.json +3 -1
  67. package/test/unit/ngsiv2/examples/contextRequests/updateContext3WithStatic.json +2 -0
  68. package/test/unit/ngsiv2/examples/contextRequests/updateContext4.json +4 -1
  69. package/test/unit/ngsiv2/examples/contextRequests/updateContext5.json +12 -0
  70. package/test/unit/ngsiv2/examples/contextRequests/updateContext6.json +12 -0
  71. package/test/unit/ngsiv2/examples/contextRequests/updateContextAliasPlugin1.json +2 -0
  72. package/test/unit/ngsiv2/examples/contextRequests/updateContextAliasPlugin2.json +3 -1
  73. package/test/unit/ngsiv2/examples/contextRequests/updateContextAliasPlugin3.json +3 -1
  74. package/test/unit/ngsiv2/examples/contextRequests/updateContextAliasPlugin4.json +3 -1
  75. package/test/unit/ngsiv2/examples/contextRequests/updateContextAliasPlugin5.json +3 -1
  76. package/test/unit/ngsiv2/examples/contextRequests/updateContextAliasPlugin6.json +3 -1
  77. package/test/unit/ngsiv2/examples/contextRequests/updateContextAliasPlugin7.json +3 -1
  78. package/test/unit/ngsiv2/examples/contextRequests/updateContextAliasPlugin8.json +3 -1
  79. package/test/unit/ngsiv2/examples/contextRequests/updateContextAliasPlugin9.json +3 -1
  80. package/test/unit/ngsiv2/examples/contextRequests/updateContextAutocast1.json +2 -0
  81. package/test/unit/ngsiv2/examples/contextRequests/updateContextAutocast2.json +2 -0
  82. package/test/unit/ngsiv2/examples/contextRequests/updateContextAutocast3.json +3 -1
  83. package/test/unit/ngsiv2/examples/contextRequests/updateContextAutocast4.json +3 -1
  84. package/test/unit/ngsiv2/examples/contextRequests/updateContextAutocast5.json +3 -1
  85. package/test/unit/ngsiv2/examples/contextRequests/updateContextAutocast6.json +3 -1
  86. package/test/unit/ngsiv2/examples/contextRequests/updateContextAutocast7.json +3 -1
  87. package/test/unit/ngsiv2/examples/contextRequests/updateContextCommandError.json +3 -1
  88. package/test/unit/ngsiv2/examples/contextRequests/updateContextCommandExpired.json +3 -1
  89. package/test/unit/ngsiv2/examples/contextRequests/updateContextCommandFinish.json +3 -1
  90. package/test/unit/ngsiv2/examples/contextRequests/updateContextCommandStatus.json +2 -0
  91. package/test/unit/ngsiv2/examples/contextRequests/updateContextCommandStatus2.json +2 -0
  92. package/test/unit/ngsiv2/examples/contextRequests/updateContextCompressTimestamp1.json +3 -1
  93. package/test/unit/ngsiv2/examples/contextRequests/updateContextCompressTimestamp2.json +3 -1
  94. package/test/unit/ngsiv2/examples/contextRequests/updateContextExpressionPlugin1.json +2 -12
  95. package/test/unit/ngsiv2/examples/contextRequests/updateContextExpressionPlugin11.json +2 -4
  96. package/test/unit/ngsiv2/examples/contextRequests/updateContextExpressionPlugin12.json +2 -4
  97. package/test/unit/ngsiv2/examples/contextRequests/updateContextExpressionPlugin13.json +3 -1
  98. package/test/unit/ngsiv2/examples/contextRequests/updateContextExpressionPlugin2.json +2 -12
  99. package/test/unit/ngsiv2/examples/contextRequests/updateContextExpressionPlugin29.json +2 -12
  100. package/test/unit/ngsiv2/examples/contextRequests/updateContextExpressionPlugin3.json +2 -4
  101. package/test/unit/ngsiv2/examples/contextRequests/updateContextExpressionPlugin30.json +2 -0
  102. package/test/unit/ngsiv2/examples/contextRequests/updateContextExpressionPlugin31.json +2 -0
  103. package/test/unit/ngsiv2/examples/contextRequests/updateContextExpressionPlugin32.json +2 -0
  104. package/test/unit/ngsiv2/examples/contextRequests/updateContextExpressionPlugin33.json +2 -0
  105. package/test/unit/ngsiv2/examples/contextRequests/updateContextExpressionPlugin34.json +2 -0
  106. package/test/unit/ngsiv2/examples/contextRequests/updateContextExpressionPlugin35.json +2 -0
  107. package/test/unit/ngsiv2/examples/contextRequests/updateContextExpressionPlugin36.json +1 -0
  108. package/test/unit/ngsiv2/examples/contextRequests/updateContextExpressionPlugin4.json +2 -0
  109. package/test/unit/ngsiv2/examples/contextRequests/updateContextExpressionPlugin40.json +1 -1
  110. package/test/unit/ngsiv2/examples/contextRequests/updateContextExpressionPlugin41.json +1 -10
  111. package/test/unit/ngsiv2/examples/contextRequests/updateContextExpressionPlugin5.json +2 -4
  112. package/test/unit/ngsiv2/examples/contextRequests/updateContextExpressionPlugin6.json +2 -4
  113. package/test/unit/ngsiv2/examples/contextRequests/updateContextExpressionPlugin7.json +2 -4
  114. package/test/unit/ngsiv2/examples/contextRequests/updateContextExpressionPlugin8.json +2 -12
  115. package/test/unit/ngsiv2/examples/contextRequests/updateContextExpressionPlugin9.json +2 -4
  116. package/test/unit/ngsiv2/examples/contextRequests/updateContextExpressionSkip.json +12 -0
  117. package/test/unit/ngsiv2/examples/contextRequests/updateContextMultientityJexlExpressionPlugin1.json +1 -1
  118. package/test/unit/ngsiv2/examples/contextRequests/updateContextMultientityPlugin1.json +1 -1
  119. package/test/unit/ngsiv2/examples/contextRequests/updateContextMultientityPlugin10.json +1 -1
  120. package/test/unit/ngsiv2/examples/contextRequests/updateContextMultientityPlugin11.json +1 -1
  121. package/test/unit/ngsiv2/examples/contextRequests/updateContextMultientityPlugin12.json +1 -1
  122. package/test/unit/ngsiv2/examples/contextRequests/updateContextMultientityPlugin13.json +1 -1
  123. package/test/unit/ngsiv2/examples/contextRequests/updateContextMultientityPlugin14.json +1 -1
  124. package/test/unit/ngsiv2/examples/contextRequests/updateContextMultientityPlugin15.json +1 -1
  125. package/test/unit/ngsiv2/examples/contextRequests/updateContextMultientityPlugin16.json +1 -1
  126. package/test/unit/ngsiv2/examples/contextRequests/updateContextMultientityPlugin17.json +1 -1
  127. package/test/unit/ngsiv2/examples/contextRequests/updateContextMultientityPlugin2.json +1 -1
  128. package/test/unit/ngsiv2/examples/contextRequests/updateContextMultientityPlugin25.json +2 -6
  129. package/test/unit/ngsiv2/examples/contextRequests/updateContextMultientityPlugin3.json +1 -1
  130. package/test/unit/ngsiv2/examples/contextRequests/updateContextMultientityPlugin4.json +1 -1
  131. package/test/unit/ngsiv2/examples/contextRequests/updateContextMultientityPlugin5.json +1 -1
  132. package/test/unit/ngsiv2/examples/contextRequests/updateContextMultientityPlugin6.json +1 -1
  133. package/test/unit/ngsiv2/examples/contextRequests/updateContextMultientityPlugin7.json +1 -1
  134. package/test/unit/ngsiv2/examples/contextRequests/updateContextMultientityPlugin8.json +1 -1
  135. package/test/unit/ngsiv2/examples/contextRequests/updateContextMultientityPlugin9.json +1 -1
  136. package/test/unit/ngsiv2/examples/contextRequests/updateContextMultientityTimestampPlugin1.json +1 -1
  137. package/test/unit/ngsiv2/examples/contextRequests/updateContextMultientityTimestampPlugin2.json +1 -1
  138. package/test/unit/ngsiv2/examples/contextRequests/updateContextMultientityTimestampPlugin3.json +1 -1
  139. package/test/unit/ngsiv2/examples/contextRequests/updateContextMultientityTimestampPlugin4.json +2 -0
  140. package/test/unit/ngsiv2/examples/contextRequests/updateContextProcessTimestamp.json +2 -0
  141. package/test/unit/ngsiv2/examples/contextRequests/updateContextStaticAttributes.json +2 -0
  142. package/test/unit/ngsiv2/examples/contextRequests/updateContextStaticAttributesMetadata.json +3 -1
  143. package/test/unit/ngsiv2/examples/contextRequests/updateContextTimestamp.json +3 -1
  144. package/test/unit/ngsiv2/examples/contextRequests/updateContextTimestampFalse.json +12 -0
  145. package/test/unit/ngsiv2/examples/contextRequests/updateContextTimestampFalseTimeInstant.json +12 -0
  146. package/test/unit/ngsiv2/examples/contextRequests/updateContextTimestampOverride.json +2 -0
  147. package/test/unit/ngsiv2/examples/contextRequests/updateContextTimestampOverrideWithoutMilis.json +2 -0
  148. package/test/unit/ngsiv2/examples/contextRequests/updateContextTimestampTimezone.json +3 -1
  149. package/test/unit/ngsiv2/expressions/jexlBasedTransformations-test.js +144 -85
  150. package/test/unit/ngsiv2/general/contextBrokerOAuthSecurityAccess-test.js +20 -53
  151. package/test/unit/ngsiv2/general/https-support-test.js +2 -6
  152. package/test/unit/ngsiv2/lazyAndCommands/command-test.js +4 -10
  153. package/test/unit/ngsiv2/lazyAndCommands/polling-commands-test.js +8 -24
  154. package/test/unit/ngsiv2/ngsiService/active-devices-test.js +146 -65
  155. package/test/unit/ngsiv2/ngsiService/autocast-test.js +14 -21
  156. package/test/unit/ngsiv2/ngsiService/staticAttributes-test.js +3 -5
  157. package/test/unit/ngsiv2/ngsiService/subscriptions-test.js +11 -20
  158. package/test/unit/ngsiv2/plugins/alias-plugin_test.js +20 -30
  159. package/test/unit/ngsiv2/plugins/compress-timestamp-plugin_test.js +4 -6
  160. package/test/unit/ngsiv2/plugins/custom-plugin_test.js +1 -2
  161. package/test/unit/ngsiv2/plugins/multientity-plugin_test.js +3 -5
  162. package/test/unit/ngsiv2/plugins/timestamp-processing-plugin_test.js +2 -3
  163. package/test/unit/ngsiv2/provisioning/device-group-api-test.js +2 -3
  164. package/test/unit/ngsiv2/provisioning/device-provisioning-api_test.js +13 -156
  165. package/test/unit/ngsiv2/provisioning/device-registration_test.js +9 -13
  166. package/test/unit/ngsiv2/provisioning/device-update-registration_test.js +4 -10
  167. package/test/unit/ngsiv2/provisioning/singleConfigurationMode-test.js +0 -11
  168. package/test/unit/ngsiv2/provisioning/updateProvisionedDevices-test.js +0 -8
  169. package/test/unit/plugins/capture-provision-inPlugins_test.js +0 -6
  170. package/.nyc_output/33364de2-1199-4ec2-b33c-cae063ef8cc4.json +0 -1
  171. package/.nyc_output/processinfo/33364de2-1199-4ec2-b33c-cae063ef8cc4.json +0 -1
  172. package/.nyc_output/processinfo/index.json +0 -1
  173. package/doc/config-basic-example.js +0 -20
  174. package/doc/development.md +0 -285
  175. package/doc/howto.md +0 -645
  176. package/doc/installationguide.md +0 -370
  177. package/doc/operations.md +0 -127
  178. package/doc/usermanual.md +0 -900
  179. package/lib/plugins/bidirectionalData.js +0 -356
  180. package/test/unit/ngsi-ld/plugins/bidirectional-plugin_test.js +0 -697
  181. package/test/unit/ngsiv2/plugins/bidirectional-plugin_test.js +0 -599
  182. /package/doc/{NorthboundInteractions.postman_collection → devel/NorthboundInteractions.postman_collection} +0 -0
  183. /package/doc/{echo.js → devel/echo.js} +0 -0
  184. /package/doc/{finalResult.js → devel/finalResult.js} +0 -0
@@ -0,0 +1,423 @@
1
+ #
2
+ # Copyright 2023 Telefonica Investigación y Desarrollo, S.A.U
3
+ #
4
+ # This file is part of fiware-iotagent-lib
5
+ #
6
+ # fiware-iotagent-lib is free software: you can redistribute it and/or
7
+ # modify it under the terms of the GNU Affero General Public License as
8
+ # published by the Free Software Foundation, either version 3 of the License,
9
+ # or (at your option) any later version.
10
+ #
11
+ # fiware-iotagent-lib is distributed in the hope that it will be useful,
12
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
13
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
14
+ # See the GNU Affero General Public License for more details.
15
+ #
16
+ # You should have received a copy of the GNU Affero General Public
17
+ # License along with fiware-iotagent-lib.
18
+ # If not, see http://www.gnu.org/licenses/.
19
+ #
20
+ # Author by: Miguel Angel Pedraza
21
+ #
22
+
23
+ from pymongo import MongoClient
24
+ from bson import json_util, ObjectId
25
+ import json
26
+ import re
27
+
28
+ from datetime import datetime
29
+
30
+ import argparse
31
+
32
+ import pandas as pd
33
+
34
+
35
+ def parse_json(data):
36
+ return json.loads(json_util.dumps(data))
37
+
38
+ document_replaced_list = []
39
+ find_document_occurrences = []
40
+ found_legacy_expressions = []
41
+ translation_legacy_expressions = []
42
+ document_occurrences_backup = []
43
+
44
+ debug = False
45
+ commit = False
46
+ replacement_count=0
47
+
48
+
49
+ # Init time
50
+ now = datetime.now()
51
+ init_time = now.strftime("%Y%m%dT%H%M%S_")
52
+
53
+ # Create the CLI argunments parser
54
+ parser = argparse.ArgumentParser(description='Tool to migrate legacy expressions in IoT Agents')
55
+ parser.add_argument('--database', help='Database name', required=True)
56
+ parser.add_argument('--collection', help='Collection name', required=True)
57
+ parser.add_argument('--translation', help='Translation file', required=False)
58
+ parser.add_argument('--debug', help='Debug mode', required=False, action='store_true')
59
+ parser.add_argument('--commit', help='Commit changes to database', required=False, action='store_true')
60
+ parser.add_argument('--mongouri', help='Database connection URI', required=False, default='mongodb://localhost:27017/')
61
+ parser.add_argument('--expressionlanguage', help='How to handle expressionLanguage values. Can be: delete, ignore, jexl or jexlall', required=False, default='ignore')
62
+ parser.add_argument('--statistics', help='Show statistics at the end of the execution. Possible values: service subservice', required=False, default='service')
63
+ parser.add_argument('--regexservice', help='FIWARE service filter', required=False, default='.*')
64
+ parser.add_argument('--regexservicepath', help='FIWARE servicepath filter', required=False, default='.*')
65
+ parser.add_argument('--regexdeviceid', help='Device ID filter', required=False, default='.*')
66
+ parser.add_argument('--regexentitytype', help='Entity type filter', required=False, default='.*')
67
+ parser.add_argument('--service', help='FIWARE service filter', required=False)
68
+ parser.add_argument('--servicepath', help='FIWARE servicepath filter', required=False)
69
+ parser.add_argument('--deviceid', help='Device ID filter', required=False, default='')
70
+ parser.add_argument('--entitytype', help='Entity type filter', required=False)
71
+ args = vars(parser.parse_args())
72
+
73
+
74
+ if args['debug']:
75
+ debug = True
76
+
77
+ if args['commit'] == True:
78
+ print('INFO: Running the script in commit mode, this will update the database')
79
+ commit = True
80
+
81
+ if args['translation'] != None and args['translation'] != '':
82
+ with open(args['translation']) as f:
83
+ translation_legacy_expressions = json.load(f)
84
+ elif (args['translation'] == None or args['translation'] == '') and commit == True:
85
+ print('ERROR: Translation file is required in commit mode')
86
+ exit(1)
87
+
88
+ mongodb_db = args['database']
89
+ mongodb_collection = args['collection']
90
+
91
+ _regex_legacy_expression = '\\${.*(@)'
92
+
93
+ # Create a filter for the query
94
+ filter = {
95
+ '$and':[
96
+ {'$or':[
97
+ {'active': {'$elemMatch': {'expression': {'$regex': _regex_legacy_expression}}}},
98
+ {'active': {'$elemMatch': {'entity_name': {'$regex': _regex_legacy_expression}}}},
99
+ {'active': {'$elemMatch': {'reverse': {'$elemMatch': {'expression':{'$regex': _regex_legacy_expression}}}}}},
100
+ {'attributes': {'$elemMatch': {'expression': {'$regex': _regex_legacy_expression}}}}, # grups
101
+ {'attributes': {'$elemMatch': {'entity_name': {'$regex': _regex_legacy_expression}}}}, # grups
102
+ {'attributes': {'$elemMatch': {'reverse': {'$elemMatch': {'expression':{'$regex': _regex_legacy_expression}}}}}}, # grups
103
+ {'commands': {'$elemMatch': {'expression': {'$regex': _regex_legacy_expression}}}},
104
+ {'endpoint': {'$regex': _regex_legacy_expression}},
105
+ {'entityNameExp': {'$regex': _regex_legacy_expression}},
106
+ {'explicitAttrs': {'$regex': _regex_legacy_expression}},
107
+ ]
108
+ }
109
+ ]
110
+ }
111
+
112
+ if args['expressionlanguage'] == 'delete':
113
+ filter['$and'][0]['$or'].append({'expressionLanguage':{'$exists': True}})
114
+ expressionlanguage = 'delete'
115
+ elif args['expressionlanguage'] == 'jexlall':
116
+ filter['$and'][0]['$or'].append({'expressionLanguage':{'$exists': True}})
117
+ expressionlanguage = 'jexlall'
118
+ elif args['expressionlanguage'] == 'jexl':
119
+ expressionlanguage = 'jexl'
120
+ else:
121
+ expressionlanguage = 'ignore'
122
+
123
+ if args['regexdeviceid'] != '.*':
124
+ filter_device_id = args['regexdeviceid']
125
+ filter['$and'].append({'id': {'$regex': filter_device_id}})
126
+ if debug:
127
+ print('Filtering by regex device ID: ' + str(filter_device_id))
128
+
129
+ if args['regexentitytype'] != '.*':
130
+ filter_entity_type = args['regexentitytype']
131
+ filter['$and'].append({'type': {'$regex': filter_entity_type}})
132
+ if debug:
133
+ print('Filtering by regex entity type: ' + str(filter_entity_type))
134
+
135
+ if args['regexservice'] != '.*':
136
+ fiware_service = args['regexservice']
137
+ filter['$and'].append({'service': {'$regex': fiware_service}})
138
+ if debug:
139
+ print('Filtering by regex service: ' + str(fiware_service))
140
+
141
+ if args['regexservicepath'] != '.*':
142
+ fiware_servicepath = args['regexservicepath']
143
+ filter['$and'].append({'subservice': {'$regex': fiware_servicepath}})
144
+ if debug:
145
+ print('Filtering by regex servicepath: ' + fiware_servicepath)
146
+
147
+ if args['deviceid']:
148
+ filter['$and'].append({'id': args['deviceid']})
149
+ if debug:
150
+ print('Filtering by device ID: ' + str(args['deviceid']))
151
+
152
+ if args['entitytype']:
153
+ filter['$and'].append({'type': args['entitytype']})
154
+ if debug:
155
+ print('Filtering by entity type: ' + str(args['entitytype']))
156
+
157
+ if args['service']:
158
+ filter['$and'].append({'service': args['service']})
159
+ if debug:
160
+ print('Filtering by service: ' + str(args['service']))
161
+
162
+ if args['servicepath']:
163
+ filter['$and'].append({'subservice': args['servicepath']})
164
+ if debug:
165
+ print('Filtering by servicepath: ' + str(args['servicepath']))
166
+
167
+ # Create a client instance of the MongoClient class
168
+ if debug:
169
+ print('Connected to: '+str(args['mongouri']))
170
+ client = MongoClient(args['mongouri']) # Create a client instance of the MongoClient class
171
+
172
+ if debug:
173
+ print('Running in debug mode')
174
+ print('MongoDB Query: ' + str(filter))
175
+
176
+ # Execute find query
177
+ result_cursor = client[mongodb_db][mongodb_collection].find(
178
+ filter=filter
179
+ )
180
+
181
+ # Loop through the results
182
+ for occurrence in result_cursor:
183
+
184
+ # Append the expression to the backup list
185
+ document_occurrences_backup.append(parse_json(occurrence))
186
+
187
+ occurrence_id = str(occurrence['_id'])
188
+
189
+ # Find the legacy expressions and replace them
190
+ if 'active' in occurrence:
191
+ for active in occurrence['active']:
192
+ if 'expression' in active:
193
+ if re.search(_regex_legacy_expression, active['expression']):
194
+
195
+ if active['expression'] not in found_legacy_expressions:
196
+ found_legacy_expressions.append(active['expression'])
197
+ find_document_occurrences.append({'_id':occurrence_id, 'expression':active['expression'], 'type':'active.expression', 'service':occurrence['service'], 'subservice':occurrence['subservice'], 'expressionIndex':found_legacy_expressions.index(active['expression'])})
198
+ if debug:
199
+ print ('ocurrence: ' + occurrence_id + ' active: ' + str(active['expression']))
200
+ if translation_legacy_expressions!=[]:
201
+ # Do the replacement of the legacy expression
202
+ if active['expression'] in translation_legacy_expressions[0]:
203
+ active['expression'] = translation_legacy_expressions[1][translation_legacy_expressions[0].index(active['expression'])]
204
+ if debug:
205
+ print(' Replaced expression: "' + active['expression'] + '" in document: ' + occurrence_id)
206
+ else:
207
+ print('ERROR: Expression not found in translation file: ' + active['expression'] + ' in document: ' + occurrence_id)
208
+
209
+ if 'entity_name' in active:
210
+ if re.search(_regex_legacy_expression, active['entity_name']):
211
+ if active['entity_name'] not in found_legacy_expressions:
212
+ found_legacy_expressions.append(active['entity_name'])
213
+ find_document_occurrences.append({'_id':occurrence_id, 'expression':active['entity_name'], 'type':'active.entity_name', 'service':occurrence['service'], 'subservice':occurrence['subservice'], 'expressionIndex':found_legacy_expressions.index(active['entity_name'])})
214
+ if debug:
215
+ print ('ocurrence: ' + occurrence_id + ' active: ' + str(active['entity_name']))
216
+ if translation_legacy_expressions!=[]:
217
+ # Do the replacement of the legacy expression
218
+ if active['entity_name'] in translation_legacy_expressions[0]:
219
+ active['entity_name'] = translation_legacy_expressions[1][translation_legacy_expressions[0].index(active['entity_name'])]
220
+ if debug:
221
+ print(' Replaced expression: "' + active['entity_name'] + '" in document: ' + occurrence_id)
222
+ else:
223
+ print('ERROR: Expression not found in translation file: ' + active['entity_name'] + ' in document: ' + occurrence_id)
224
+
225
+ if 'reverse' in active:
226
+ if 'expression' in active['reverse']:
227
+ if re.search(_regex_legacy_expression, active['reverse']['expression']):
228
+ if active['reverse']['expression'] not in found_legacy_expressions:
229
+ found_legacy_expressions.append(active['reverse']['expression'])
230
+ find_document_occurrences.append({'_id':occurrence_id, 'expression':active['reverse']['expression'], 'type':'active.reverse.expression', 'service':occurrence['service'], 'subservice':occurrence['subservice'], 'expressionIndex':found_legacy_expressions.index(active['reverse']['expression'])})
231
+ if debug:
232
+ print ('ocurrence: ' + occurrence_id + ' active: ' + str(active['reverse']['expression']))
233
+ if translation_legacy_expressions!=[]:
234
+ # Do the replacement of the legacy expression
235
+ if active['reverse']['expression'] in translation_legacy_expressions[0]:
236
+ active['reverse']['expression'] = translation_legacy_expressions[1][translation_legacy_expressions[0].index(active['reverse']['expression'])]
237
+ if debug:
238
+ print(' Replaced expression: "' + active['reverse']['expression'] + '" in document: ' + occurrence_id)
239
+ else:
240
+ print('ERROR: Expression not found in translation file: ' + active['reverse']['expression'] + ' in document: ' + occurrence_id)
241
+
242
+ if 'attributes' in occurrence:
243
+ for attribute in occurrence['attributes']:
244
+ if 'expression' in attribute:
245
+ if re.search(_regex_legacy_expression, attribute['expression']):
246
+ if attribute['expression'] not in found_legacy_expressions:
247
+ found_legacy_expressions.append(attribute['expression'])
248
+ find_document_occurrences.append({'_id':occurrence_id, 'expression':attribute['expression'], 'type':'attribute.expression', 'service':occurrence['service'], 'subservice':occurrence['subservice'], 'expressionIndex':found_legacy_expressions.index(attribute['expression'])})
249
+ if debug:
250
+ print ('ocurrence: ' + occurrence_id + ' attribute: ' + str(attribute['expression']))
251
+ if translation_legacy_expressions!=[]:
252
+ # Do the replacement of the legacy expression
253
+ if attribute['expression'] in translation_legacy_expressions[0]:
254
+ attribute['expression'] = translation_legacy_expressions[1][translation_legacy_expressions[0].index(attribute['expression'])]
255
+ if debug:
256
+ print(' Replaced expression: "' + attribute['expression'] + '" in document: ' + occurrence_id)
257
+ else:
258
+ print('ERROR: Expression not found in translation file: ' + attribute['expression'] + ' in document: ' + occurrence_id)
259
+
260
+ if 'entity_name' in attribute:
261
+ if re.search(_regex_legacy_expression, attribute['entity_name']):
262
+ if attribute['entity_name'] not in found_legacy_expressions:
263
+ found_legacy_expressions.append(attribute['entity_name'])
264
+ find_document_occurrences.append({'_id':occurrence_id, 'expression':attribute['entity_name'], 'type':'attribute.entity_name', 'service':occurrence['service'], 'subservice':occurrence['subservice'], 'expressionIndex':found_legacy_expressions.index(attribute['entity_name'])})
265
+ if debug:
266
+ print ('ocurrence: ' + occurrence_id + ' attribute: ' + str(attribute['entity_name']))
267
+ if translation_legacy_expressions!=[]:
268
+ # Do the replacement of the legacy expression
269
+ if attribute['entity_name'] in translation_legacy_expressions[0]:
270
+ attribute['entity_name'] = translation_legacy_expressions[1][translation_legacy_expressions[0].index(attribute['entity_name'])]
271
+ if debug:
272
+ print(' Replaced expression: "' + attribute['entity_name'] + '" in document: ' + occurrence_id)
273
+ else:
274
+ print('ERROR: Expression not found in translation file: ' + attribute['entity_name'] + ' in document: ' + occurrence_id)
275
+
276
+ if 'reverse' in attribute:
277
+ if 'expression' in attribute['reverse']:
278
+ if re.search(_regex_legacy_expression, attribute['reverse']['expression']):
279
+ if attribute['reverse']['expression'] not in found_legacy_expressions:
280
+ found_legacy_expressions.append(attribute['reverse']['expression'])
281
+ find_document_occurrences.append({'_id':occurrence_id, 'expression':attribute['reverse']['expression'], 'type':'attribute.reverse.expression', 'service':occurrence['service'], 'subservice':occurrence['subservice'], 'expressionIndex':found_legacy_expressions.index(attribute['reverse']['expression'])})
282
+ if debug:
283
+ print ('ocurrence: ' + occurrence_id + ' attribute: ' + str(attribute['reverse']['expression']))
284
+ if translation_legacy_expressions!=[]:
285
+ # Do the replacement of the legacy expression
286
+ if attribute['reverse']['expression'] in translation_legacy_expressions[0]:
287
+ attribute['reverse']['expression'] = translation_legacy_expressions[1][translation_legacy_expressions[0].index(attribute['reverse']['expression'])]
288
+ if debug:
289
+ print(' Replaced expression: "' + attribute['reverse']['expression'] + '" in document: ' + occurrence_id)
290
+ else:
291
+ print('ERROR: Expression not found in translation file: ' + attribute['reverse']['expression'] + ' in document: ' + occurrence_id)
292
+
293
+ if 'commands' in occurrence:
294
+ for command in occurrence['commands']:
295
+ if 'expression' in command:
296
+ if re.search(_regex_legacy_expression, command['expression']):
297
+ if command['expression'] not in found_legacy_expressions:
298
+ found_legacy_expressions.append(command['expression'])
299
+ find_document_occurrences.append({'_id':occurrence_id, 'expression':command['expression'], 'type':'command.expression', 'service':occurrence['service'], 'subservice':occurrence['subservice'], 'expressionIndex':found_legacy_expressions.index(command['expression'])})
300
+ if debug:
301
+ print ('ocurrence: ' + occurrence_id + ' command: ' + str(command['expression']))
302
+ if translation_legacy_expressions!=[]:
303
+ # Do the replacement of the legacy expression
304
+ if command['expression'] in translation_legacy_expressions[0]:
305
+ command['expression'] = translation_legacy_expressions[1][translation_legacy_expressions[0].index(command['expression'])]
306
+ if debug:
307
+ print(' Replaced expression: "' + command['expression'] + '" in document: ' + occurrence_id)
308
+ else:
309
+ print('ERROR: Expression not found in translation file: ' + command['expression'] + ' in document: ' + occurrence_id)
310
+
311
+ if 'endpoint' in occurrence:
312
+ if re.search(_regex_legacy_expression, occurrence['endpoint']):
313
+ if occurrence['endpoint'] not in found_legacy_expressions:
314
+ found_legacy_expressions.append(occurrence['endpoint'])
315
+ find_document_occurrences.append({'_id':occurrence_id, 'expression':occurrence['endpoint'], 'type':'endpoint', 'service':occurrence['service'], 'subservice':occurrence['subservice'], 'expressionIndex':found_legacy_expressions.index(occurrence['endpoint'])})
316
+ if debug:
317
+ print ('ocurrence: ' + occurrence_id + ' endpoint: ' + str(occurrence['endpoint']))
318
+ if translation_legacy_expressions!=[]:
319
+ # Do the replacement of the legacy expression
320
+ if occurrence['endpoint'] in translation_legacy_expressions[0]:
321
+ occurrence['endpoint'] = translation_legacy_expressions[1][translation_legacy_expressions[0].index(occurrence['endpoint'])]
322
+ if debug:
323
+ print(' Replaced expression: "' + occurrence['endpoint'] + '" in document: ' + occurrence_id)
324
+ else:
325
+ print('ERROR: Expression not found in translation file: ' + occurrence['endpoint'] + ' in document: ' + occurrence_id)
326
+
327
+ if 'entityNameExp' in occurrence:
328
+ if re.search(_regex_legacy_expression, occurrence['entityNameExp']):
329
+ if occurrence['entityNameExp'] not in found_legacy_expressions:
330
+ found_legacy_expressions.append(occurrence['entityNameExp'])
331
+ find_document_occurrences.append({'_id':occurrence_id, 'expression':occurrence['entityNameExp'], 'type':'entityNameExp', 'service':occurrence['service'], 'subservice':occurrence['subservice'], 'expressionIndex':found_legacy_expressions.index(occurrence['entityNameExp'])})
332
+ if debug:
333
+ print ('ocurrence: ' + occurrence_id + ' entityNameExp: ' + str(occurrence['entityNameExp']))
334
+ if translation_legacy_expressions!=[]:
335
+ # Do the replacement of the legacy expression
336
+ if occurrence['entityNameExp'] in translation_legacy_expressions[0]:
337
+ occurrence['entityNameExp'] = translation_legacy_expressions[1][translation_legacy_expressions[0].index(occurrence['entityNameExp'])]
338
+ if debug:
339
+ print(' Replaced expression: "' + occurrence['entityNameExp'] + '" in document: ' + occurrence_id)
340
+ else:
341
+ print('ERROR: Expression not found in translation file: ' + occurrence['entityNameExp'] + ' in document: ' + occurrence_id)
342
+
343
+ if 'explicitAttrs' in occurrence:
344
+ if re.search(_regex_legacy_expression, str(occurrence['explicitAttrs'])): # Note that explicitAttrs can be a boolean value. For that reason, we convert ocurrence['explicitAttrs'] to string, otherwise "TypeError: expected string or bytes-like object" will be raised
345
+ if occurrence['explicitAttrs'] not in found_legacy_expressions:
346
+ found_legacy_expressions.append(occurrence['explicitAttrs'])
347
+ find_document_occurrences.append({'_id':occurrence_id, 'expression':occurrence['explicitAttrs'], 'type':'explicitAttrs', 'service':occurrence['service'], 'subservice':occurrence['subservice'], 'expressionIndex':found_legacy_expressions.index(occurrence['explicitAttrs'])})
348
+ if debug:
349
+ print ('ocurrence: ' + occurrence_id + ' explicitAttrs: ' + str(occurrence['explicitAttrs']))
350
+ if translation_legacy_expressions!=[]:
351
+ # Do the replacement of the legacy expression
352
+ if occurrence['explicitAttrs'] in translation_legacy_expressions[0]:
353
+ occurrence['explicitAttrs'] = translation_legacy_expressions[1][translation_legacy_expressions[0].index(occurrence['explicitAttrs'])]
354
+ if debug:
355
+ print(' Replaced expression: "' + occurrence['explicitAttrs'] + '" in document: ' + occurrence_id)
356
+ else:
357
+ print('ERROR: Expression not found in translation file: ' + occurrence['explicitAttrs'] + ' in document: ' + occurrence_id)
358
+
359
+ if 'expressionLanguage' in occurrence:
360
+ if expressionlanguage == 'delete':
361
+ if debug:
362
+ print ('ocurrence: ' + occurrence_id + ' expressionLanguage: ' + str(occurrence['expressionLanguage']))
363
+ del occurrence['expressionLanguage']
364
+ elif expressionlanguage == 'jexl' or expressionlanguage == 'jexlall':
365
+ if debug:
366
+ print ('ocurrence: ' + occurrence_id + ' expressionLanguage: ' + str(occurrence['expressionLanguage']))
367
+ occurrence['expressionLanguage'] = 'jexl'
368
+ else:
369
+ if expressionlanguage == 'jexl' or expressionlanguage == 'jexlall':
370
+ if debug:
371
+ print ('ocurrence: ' + occurrence_id + ' expressionLanguage: ' + 'undefined')
372
+ occurrence['expressionLanguage'] = 'jexl'
373
+
374
+
375
+ # Update element in the database
376
+ if commit and translation_legacy_expressions!=[]:
377
+ client[mongodb_db][mongodb_collection].replace_one(
378
+ {'_id': occurrence['_id']},
379
+ occurrence
380
+ )
381
+ replacement_count+=1
382
+
383
+ # Update element in the list of documents with
384
+ document_replaced_list.append(parse_json(occurrence))
385
+
386
+ # Print the counts
387
+ print ('\nFound ' + str(len(find_document_occurrences)) + ' legacy expressions in ' + str(len(document_replaced_list)) + ' documents')
388
+ if commit:
389
+ print ('Updated ' + str(replacement_count) + ' documents in the database')
390
+
391
+ # write the results to files
392
+ f1 = open(init_time+"legacy_expression_occurrences.json", "w")
393
+ f1.write(json.dumps(find_document_occurrences,indent=4))
394
+ f1.close()
395
+ f2 = open(init_time+"legacy_expressions_list.json", "w")
396
+ f2.write(json.dumps(found_legacy_expressions,indent=4))
397
+ f2.close()
398
+ f3 = open(init_time+"documents_replaced.json", "w")
399
+ f3.write(json.dumps(document_replaced_list,indent=4))
400
+ f3.close()
401
+ f4 = open(init_time+"documents_backup.json", "w")
402
+ f4.write(json.dumps(document_occurrences_backup,indent=4))
403
+ f4.close()
404
+
405
+ if args['statistics']:
406
+
407
+ # Load data into pandas dataframe
408
+ df = pd.DataFrame(find_document_occurrences, columns=['_id', 'expression', 'type', 'service', 'subservice', 'expressionIndex'])
409
+
410
+ # Configure pandas to display all data
411
+ pd.set_option('expand_frame_repr', False)
412
+ pd.set_option('display.max_rows', None) # more options can be specified also
413
+ pd.set_option('display.max_columns', None) # more options can be specified also
414
+
415
+ table_collums = ['service']
416
+
417
+ if args['statistics'] == 'subservice':
418
+ table_collums.append('subservice')
419
+
420
+ new = df.pivot_table(index='expression', columns=table_collums, values=['_id'], aggfunc='count', fill_value=0, margins=True)
421
+
422
+ # Display all data
423
+ print(new)
@@ -0,0 +1,3 @@
1
+ matplotlib==3.7.1
2
+ pandas==2.0.2
3
+ pymongo==4.3.3
@@ -3,7 +3,6 @@
3
3
  {
4
4
  "device_id": "MicroLight1",
5
5
  "protocol": "GENERIC_PROTO",
6
- "expressionLanguage": "jexl",
7
6
  "attributes": [
8
7
  {
9
8
  "name": "attr_name",
@@ -128,8 +128,7 @@ describe('NGSI-v2 - Secured access to the Context Broker with Keystone', functio
128
128
  .matchHeader('fiware-service', 'smartgondor')
129
129
  .matchHeader('fiware-servicepath', 'electricity')
130
130
  .matchHeader('X-Auth-Token', '12345679ABCDEF')
131
- .patch('/v2/entities/light1/attrs')
132
- .query({ type: 'Light' })
131
+ .post('/v2/entities?options=upsert')
133
132
  .reply(204);
134
133
  iotAgentLib.activate(iotAgentConfig, done);
135
134
  });
@@ -166,8 +165,7 @@ describe('NGSI-v2 - Secured access to the Context Broker with Keystone', functio
166
165
  .matchHeader('fiware-service', 'smartgondor')
167
166
  .matchHeader('fiware-servicepath', 'electricity')
168
167
  .matchHeader('X-Auth-Token', '12345679ABCDEF')
169
- .patch('/v2/entities/light1/attrs')
170
- .query({ type: 'Light' })
168
+ .post('/v2/entities?options=upsert')
171
169
  .reply(403, { name: 'ACCESS_FORBIDDEN' });
172
170
 
173
171
  iotAgentLib.activate(iotAgentConfig, done);
@@ -199,8 +197,7 @@ describe('NGSI-v2 - Secured access to the Context Broker with Keystone', functio
199
197
  .matchHeader('fiware-service', 'smartgondor')
200
198
  .matchHeader('fiware-servicepath', 'electricity')
201
199
  .matchHeader('X-Auth-Token', '12345679ABCDEF')
202
- .patch('/v2/entities/light1/attrs')
203
- .query({ type: 'Light' });
200
+ .post('/v2/entities?options=upsert');
204
201
 
205
202
  iotAgentLib.activate(iotAgentConfig, done);
206
203
  });
@@ -278,13 +275,6 @@ describe('NGSI-v2 - Secured access to the Context Broker with Keystone', functio
278
275
 
279
276
  contextBrokerMock = nock('http://192.168.1.1:1026');
280
277
 
281
- contextBrokerMock
282
- .matchHeader('fiware-service', 'smartgondor')
283
- .matchHeader('fiware-servicepath', 'electricity')
284
- .matchHeader('X-Auth-Token', '12345679ABCDEF')
285
- .post('/v2/entities?options=upsert')
286
- .reply(204);
287
-
288
278
  contextBrokerMock
289
279
  .post('/v2/registrations')
290
280
  .reply(201, null, { Location: '/v2/registrations/6319a7f5254b05844116584d' });
@@ -303,7 +293,7 @@ describe('NGSI-v2 - Secured access to the Context Broker with Keystone', functio
303
293
  });
304
294
 
305
295
  it('subscribe requests use auth header', function (done) {
306
- iotAgentLib.getDevice('Light1', 'smartgondor', 'electricity', function (error, device) {
296
+ iotAgentLib.getDevice('Light1', null, 'smartgondor', 'electricity', function (error, device) {
307
297
  iotAgentLib.subscribe(device, ['dimming'], null, function (error) {
308
298
  should.not.exist(error);
309
299
 
@@ -324,7 +314,7 @@ describe('NGSI-v2 - Secured access to the Context Broker with Keystone', functio
324
314
  'X-Subject-Token': '12345679ABCDEF'
325
315
  });
326
316
 
327
- iotAgentLib.getDevice('Light1', 'smartgondor', 'electricity', function (error, device) {
317
+ iotAgentLib.getDevice('Light1', null, 'smartgondor', 'electricity', function (error, device) {
328
318
  iotAgentLib.subscribe(device, ['dimming'], null, function (error) {
329
319
  iotAgentLib.unsubscribe(device, '51c0ac9ed714fb3b37d7d5a8', function (error) {
330
320
  contextBrokerMock.done();
@@ -387,7 +387,7 @@ describe('NGSI-v2 - MongoDB Device Registry', function () {
387
387
  });
388
388
 
389
389
  it('should be removed from MongoDB', function (done) {
390
- iotAgentLib.unregister(device1.id, 'smartgondor', 'gardens', function (error) {
390
+ iotAgentLib.unregister(device1.id, null, 'smartgondor', 'gardens', function (error) {
391
391
  iotAgentDb
392
392
  .db()
393
393
  .collection('devices')