recog 2.3.15 → 2.3.20

Sign up to get free protection for your applications and to get access to all the features.
Files changed (53) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/ci.yml +26 -0
  3. data/.snyk +10 -0
  4. data/LICENSE +1 -1
  5. data/bin/recog_standardize +8 -2
  6. data/cpe-remap.yaml +314 -170
  7. data/identifiers/README.md +24 -10
  8. data/identifiers/fields.txt +104 -0
  9. data/identifiers/hw_device.txt +5 -4
  10. data/identifiers/hw_family.txt +17 -0
  11. data/identifiers/hw_product.txt +87 -6
  12. data/identifiers/os_architecture.txt +0 -10
  13. data/identifiers/os_device.txt +12 -31
  14. data/identifiers/os_family.txt +2 -94
  15. data/identifiers/os_product.txt +45 -124
  16. data/identifiers/service_family.txt +14 -37
  17. data/identifiers/service_product.txt +283 -88
  18. data/identifiers/vendor.txt +99 -192
  19. data/lib/recog/version.rb +1 -1
  20. data/requirements.txt +1 -1
  21. data/update_cpes.py +110 -49
  22. data/xml/apache_modules.xml +60 -0
  23. data/xml/dns_versionbind.xml +40 -17
  24. data/xml/favicons.xml +163 -20
  25. data/xml/ftp_banners.xml +25 -25
  26. data/xml/hp_pjl_id.xml +1 -1
  27. data/xml/html_title.xml +561 -51
  28. data/xml/http_cookies.xml +266 -61
  29. data/xml/http_servers.xml +472 -96
  30. data/xml/http_wwwauth.xml +53 -26
  31. data/xml/ldap_searchresult.xml +10 -6
  32. data/xml/mdns_device-info_txt.xml +308 -10
  33. data/xml/ntp_banners.xml +15 -1
  34. data/xml/operating_system.xml +1 -0
  35. data/xml/rtsp_servers.xml +7 -0
  36. data/xml/sip_banners.xml +346 -8
  37. data/xml/sip_user_agents.xml +321 -7
  38. data/xml/smb_native_lm.xml +32 -1
  39. data/xml/smb_native_os.xml +158 -33
  40. data/xml/smtp_banners.xml +48 -7
  41. data/xml/smtp_expn.xml +1 -0
  42. data/xml/smtp_help.xml +2 -0
  43. data/xml/smtp_vrfy.xml +3 -1
  44. data/xml/snmp_sysdescr.xml +211 -42
  45. data/xml/ssh_banners.xml +127 -14
  46. data/xml/telnet_banners.xml +44 -14
  47. data/xml/tls_jarm.xml +140 -0
  48. data/xml/x509_issuers.xml +179 -7
  49. data/xml/x509_subjects.xml +252 -35
  50. metadata +6 -5
  51. data/identifiers/software_class.txt +0 -26
  52. data/identifiers/software_family.txt +0 -91
  53. data/identifiers/software_product.txt +0 -333
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 563ed349556c084c9c46b32ec41eece72724826ae355a639bd2dd55d4a990f3d
4
- data.tar.gz: 03e767f4fd9557af69c716515e74595fbb5f287883c6d4e742a1aea1a6cdb0ef
3
+ metadata.gz: cb4df95cbe1561c384b06be8c36fcea1e51df3c6cdb86a2a944715213d119ae8
4
+ data.tar.gz: 751fa73b20c6fb9f1c372be5503c07302101b77d729cdce3befee2981651f1ca
5
5
  SHA512:
6
- metadata.gz: a5a458eef93bb3f5b9f298523f38f728e74bac390b22978af2bbf1c74b1ff55354128df657210ae49542b4a4338a3bdf8d69387c02a06ae7b0f59dc45e237775
7
- data.tar.gz: 38d28282042fa32e206f795d63968fd45362563359e2a5d8a93070ac84b85e766c8bd8accebc518e6a06b8cd9d97a52cd5856928bc95de02857783b8669dbacc
6
+ metadata.gz: 6612cf0d0c5f19cd1a913123fe3c4fce9772ac82b7a07f78ace94f51b1681210a8dfacde3624b2b54c7d66f2f1530771d9a592c3d5bda8bde897d4f9713c2ef9
7
+ data.tar.gz: 7b935f573b7b4050b2b06e2b8965af9201bb385e2695f9859e6ecf233f93aadb9331d648b18a506528efbd97e2821d0b9816970bfd5df978262b05ec8aeb9f8e
@@ -0,0 +1,26 @@
1
+ name: CI
2
+
3
+ on: [push, pull_request]
4
+
5
+ jobs:
6
+ test:
7
+ name: 'Ruby: ${{ matrix.ruby-version }}'
8
+ runs-on: ubuntu-latest
9
+ strategy:
10
+ fail-fast: false
11
+ matrix:
12
+ ruby-version: ['2.5', '2.6', '2.7', '3.0', 'jruby-9.1.17.0', 'jruby']
13
+
14
+ steps:
15
+ - uses: actions/checkout@v2
16
+ - name: Set up Ruby
17
+ uses: ruby/setup-ruby@v1
18
+ with:
19
+ ruby-version: ${{ matrix.ruby-version }}
20
+ bundler-cache: true # runs 'bundle install' and caches installed gems automatically
21
+ - name: Run tests
22
+ run: |
23
+ bundle exec rake --version
24
+ bundle exec rake tests
25
+ env:
26
+ JRUBY_OPTS: --server -J-Xms512m -J-Xmx2G
data/.snyk ADDED
@@ -0,0 +1,10 @@
1
+ # Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities.
2
+ python: 3.6.0
3
+ version: v1.14.1
4
+ # ignores vulnerabilities until expiry date; change duration by modifying expiry date
5
+ ignore:
6
+ SNYK-PYTHON-PYYAML-590151:
7
+ - pyyaml:
8
+ reason: Project doesn't use vulnerable code path.
9
+ expires: 2021-06-01T00:00:00.000Z
10
+ patch: {}
data/LICENSE CHANGED
@@ -2,6 +2,6 @@ Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
2
2
  Source: https://github.com/rapid7/recog
3
3
 
4
4
  Files: *
5
- Copyright: 2014-2015, Rapid7, Inc.
5
+ Copyright: 2014, Rapid7, Inc.
6
6
  License: BSD-2-clause
7
7
 
@@ -50,6 +50,7 @@ end
50
50
 
51
51
  # Load the unique identifiers
52
52
  vendors = load_identifiers(File.join(bdir, "vendor.txt"))
53
+ fields = load_identifiers(File.join(bdir, "fields.txt"))
53
54
  os_arch = load_identifiers(File.join(bdir, "os_architecture.txt"))
54
55
  os_prod = load_identifiers(File.join(bdir, "os_product.txt"))
55
56
  os_family = load_identifiers(File.join(bdir, "os_family.txt"))
@@ -67,8 +68,12 @@ ARGV.each do |arg|
67
68
  ndb.fingerprints.each do |f|
68
69
  f.params.each do |k,v|
69
70
  paramIndex, val = v
71
+ if ! fields[k]
72
+ puts "FIELD MISSING: #{k}"
73
+ fields[k] = true
74
+ end
70
75
  next if paramIndex != 0
71
- next if val.index("{") != -1
76
+ next if val.index("{") != nil
72
77
  next if val.strip == ""
73
78
  case k
74
79
  when "os.vendor", "service.vendor", "service.component.vendor", "hw.vendor"
@@ -111,7 +116,7 @@ ARGV.each do |arg|
111
116
  puts "HW DEVICE MISSING: #{val}"
112
117
  hw_device[val] = true
113
118
  end
114
- when "service.product"
119
+ when "service.product", "service.component.product"
115
120
  if ! svc_prod[val]
116
121
  puts "SERVICE PRODUCT MISSING: #{val}"
117
122
  svc_prod[val] = true
@@ -131,6 +136,7 @@ exit if ! options.write
131
136
 
132
137
  # Write back the unique identifiers
133
138
  write_identifiers(vendors, File.join(bdir, "vendor.txt"))
139
+ write_identifiers(fields, File.join(bdir, "fields.txt"))
134
140
  write_identifiers(os_arch, File.join(bdir, "os_architecture.txt"))
135
141
  write_identifiers(os_prod, File.join(bdir, "os_product.txt"))
136
142
  write_identifiers(os_family, File.join(bdir, "os_family.txt"))
data/cpe-remap.yaml CHANGED
@@ -1,171 +1,315 @@
1
1
  mappings:
2
- alpine:
3
- vendor: alpinelinux
4
- apache:
5
- vendor: apache
6
- products:
7
- httpd: http_server
8
- apple:
9
- products:
10
- ios: iphone_os
11
- alt-n:
12
- vendor: altn
13
- bea:
14
- vendor: bea
15
- products:
16
- weblogic: weblogic_server
17
- blue_coat:
18
- vendor: bluecoat
19
- carnegie_mellon_university:
20
- vendor: cmu
21
- products:
22
- cyrus_imap: cyrus_imap_server
23
- centos:
24
- vendor: centos
25
- products:
26
- linux: centos
27
- centos_webpanel:
28
- vendor: centos-webpanel
29
- check_point:
30
- vendor: checkpoint
31
- cisco:
32
- vendor: cisco
33
- products:
34
- adaptive_security_appliance: adaptive_security_appliance_software
35
- pix: pix_firewall_software
36
- telepresence: telepresence_video_communication_server_software
37
- debian:
38
- vendor: debian
39
- products:
40
- linux: debian_linux
41
- embedthis:
42
- products:
43
- goahead_webserver: goahead
44
- f5:
45
- vendor: f5
46
- products:
47
- big-ip: big-ip_local_traffic_manager
48
- big-ip_ltm: big-ip_local_traffic_manager
49
- hp:
50
- vendor: hp
51
- products:
52
- ilo: integrated_lights_out
53
- tru64_unix: tru64
54
- ibm:
55
- vendor: ibm
56
- products:
57
- lotus_domino: lotus_domino_server
58
- ibm_domino: lotus_domino
59
- os/400: os_400
60
- jamf:
61
- products:
62
- jamf_pro: jamf
63
- juniper:
64
- vendor: juniper
65
- products:
66
- junos_os: junos
67
- kibana:
68
- vendor: elasticsearch
69
- cz.nic:
70
- vendor: knot-dns
71
- litespeed_technologies:
72
- vendor: litespeedtech
73
- linux:
74
- vendor: linux
75
- products:
76
- linux: linux_kernel
77
- mailenable:
78
- vendor: mailenable
79
- products:
80
- mail_server: mailenable
81
- microsoft:
82
- vendor: microsoft
83
- products:
84
- active_directory_controller: active_directory
85
- exchange_server_5.5: exchange_server
86
- exchange_2000_server: exchange_server
87
- exchange_2003_server: exchange_server
88
- exchange_2007_server: exchange_server
89
- lightweight_directory_server: active_directory_lightweight_directory_service
90
- windows_server_2003_datacenter_edition: windows_server_2003
91
- windows_server_2003_r2: windows_server_2003
92
- windows_2008_r2: windows_server_2008
93
- windows_server_2008_datacenter_edition: windows_server_2008
94
- windows_server_2008_r2: windows_server_2008
95
- windows_server_2008_r2_datacenter_edition: windows_server_2008
96
- windows_server_2012_r2: windows_server_2012
97
- nt: windows_nt
98
- windows_nt_desktop: windows_nt
99
- windows_nt_server: windows_nt
100
- windows_server_2000: windows_2000
101
- windows_2000_server: windows_2000
102
- windows_2000_datacenter_server: windows_2000
103
- pws: personal_web_server
104
- mod_ssl:
105
- vendor: modssl
106
- mod_wsgi:
107
- vendor: modwsgi
108
- mort_bay:
109
- vendor: mortbay
110
- nlnet_labs:
111
- vendor: nlnetlabs
112
- products:
113
- dnsd: name_server_daemon
114
- net-snmp:
115
- vendor: net-snmp
116
- products:
117
- snmp_agent: net-snmp
118
- palo_alto_networks:
119
- vendor: paloaltonetworks
120
- products:
121
- pa_firewall: pan-os
122
- parallels:
123
- products:
124
- plesk: parallels_plesk_panel
125
- plesk:
126
- vendor: parallels
127
- proftpd_project:
128
- vendor: proftpd
129
- realvnc_ltd.:
130
- vendor: realvnc
131
- red_hat:
132
- vendor: redhat
133
- products:
134
- cygwin_x_server_project: cygwin
135
- fedora_core_linux: fedora_core
136
- jboss_as: jboss_wildfly_application_server
137
- jboss_eap: jboss_enterprise_application_platform
138
- jbossweb: jboss_web_framework_kit
139
- red_hat_directory_server: directory_server
140
- squid_cache:
141
- vendor: squid-cache
142
- sun:
143
- vendor: sun
144
- products:
145
- solaris: sunos
146
- tandberg:
147
- vendor: cisco
148
- tightvnc:
149
- products:
150
- desktop: tightvnc
151
- ubiquiti:
152
- vendor: ui
153
- ubuntu:
154
- vendor: canonical
155
- products:
156
- linux: ubuntu_linux
157
- vandyke_software:
158
- vendor: vandyke
159
- vmware:
160
- vendor: vmware
161
- products:
162
- photon_linux: photon_os
163
- zimbra: zimbra_desktop
164
- vmware_esx_server: esx
165
- vmware_esxi_server: esxi
166
- wind_river:
167
- vendor: windriver
168
- x.org:
169
- vendor: x.org
170
- products:
171
- x.org_x11: x11
2
+ # The following section contains CPE application or 'a' remappings. These will
3
+ # ONLY be used for mapping Recog 'service' attributes.
4
+ a:
5
+ akamai:
6
+ products:
7
+ ghost: akamaighost
8
+ amazon:
9
+ products:
10
+ s3: amazon_simple_storage_service
11
+ cloudfront_load_balancer: amazon_cloudfront
12
+ apache:
13
+ products:
14
+ httpd: http_server
15
+ aprelium_technologies:
16
+ vendor: aprelium
17
+ alt-n:
18
+ vendor: altn
19
+ aruba_networks:
20
+ vendor: arubanetworks
21
+ bea:
22
+ products:
23
+ weblogic: weblogic_server
24
+ blue_coat:
25
+ vendor: bluecoat
26
+ carnegie_mellon_university:
27
+ vendor: cmu
28
+ products:
29
+ cyrus_imap: cyrus_imap_server
30
+ centos_webpanel:
31
+ vendor: centos-webpanel
32
+ check_point:
33
+ vendor: checkpoint
34
+ cherokee_project:
35
+ vendor: cherokee-project
36
+ cisco:
37
+ products:
38
+ apic: application_policy_infrastructure_controller
39
+ cloudflare:
40
+ products:
41
+ cloudflare_load_balancer: load_balancing
42
+ cpanel:
43
+ products:
44
+ cpanel_service_daemon: cpanel
45
+ crushftp:
46
+ products:
47
+ crushftp_web_interface: crushftp
48
+ cz.nic:
49
+ vendor: knot-dns
50
+ drupal:
51
+ products:
52
+ cms: drupal
53
+ embedthis:
54
+ products:
55
+ goahead_webserver: goahead
56
+ envoy_proxy:
57
+ vendor: envoyproxy
58
+ f5:
59
+ products:
60
+ big-ip: big-ip_local_traffic_manager
61
+ big-ip_ltm: big-ip_local_traffic_manager
62
+ fedora_project:
63
+ vendor: fedoraproject
64
+ google:
65
+ products:
66
+ google_web_services: web_server
67
+ ibm:
68
+ products:
69
+ lotus_domino: lotus_domino_server
70
+ ibm_domino: lotus_domino
71
+ ignite_realtime:
72
+ vendor: igniterealtime
73
+ intel:
74
+ products:
75
+ intel(r)_active_management_technology: active_management_technology
76
+ intel(r)_standard_manageability: standard_manageability
77
+ jamf:
78
+ products:
79
+ jamf_pro: jamf
80
+ kibana:
81
+ vendor: elasticsearch
82
+ kubernetes:
83
+ products:
84
+ nginx_ingress_controller: ingress-nginx
85
+ kodi:
86
+ products:
87
+ media_server: kodi
88
+ kong:
89
+ vendor: konghq
90
+ products:
91
+ gateway: kong_gateway
92
+ litespeed_technologies:
93
+ vendor: litespeedtech
94
+ lotus:
95
+ vendor: ibm
96
+ lynx_technology:
97
+ vendor: lynxtechnology
98
+ products:
99
+ twonky_media_server: twonky_server
100
+ mailenable:
101
+ products:
102
+ mail_server: mailenable
103
+ manageengine:
104
+ vendor: zohocorp
105
+ products:
106
+ adaudit_plus: manageengine_adaudit_plus
107
+ desktop_central: manageengine_desktop_central
108
+ opmanager: manageengine_opmanager
109
+ microsoft:
110
+ products:
111
+ active_directory_controller: active_directory
112
+ exchange_server_5.5: exchange_server
113
+ exchange_2000_server: exchange_server
114
+ exchange_2003_server: exchange_server
115
+ exchange_2007_server: exchange_server
116
+ lightweight_directory_server: active_directory_lightweight_directory_service
117
+ pws: personal_web_server
118
+ mod_ssl:
119
+ vendor: modssl
120
+ mod_wsgi:
121
+ vendor: modwsgi
122
+ # NIST took the vendor name from the website but apparently missed the `.in`
123
+ # in moinmo.in was part of the name
124
+ moinmoin:
125
+ vendor: moinmo
126
+ mort_bay:
127
+ vendor: mortbay
128
+ munin:
129
+ vendor: munin-monitoring
130
+ nlnet_labs:
131
+ vendor: nlnetlabs
132
+ products:
133
+ dnsd: name_server_daemon
134
+ net-snmp:
135
+ products:
136
+ snmp_agent: net-snmp
137
+ owncloud:
138
+ products:
139
+ owncloud_server: owncloud
140
+ parallels:
141
+ products:
142
+ plesk: parallels_plesk_panel
143
+ plesk:
144
+ vendor: parallels
145
+ proftpd_project:
146
+ vendor: proftpd
147
+ progress:
148
+ products:
149
+ openedge_explorer: openedge
150
+ pulse_secure:
151
+ vendor: pulsesecure
152
+ realvnc_ltd.:
153
+ vendor: realvnc
154
+ red_hat:
155
+ vendor: redhat
156
+ products:
157
+ cygwin_x_server_project: cygwin
158
+ jboss_as: jboss_wildfly_application_server
159
+ jboss_eap: jboss_enterprise_application_platform
160
+ jbossweb: jboss_web_framework_kit
161
+ red_hat_directory_server: directory_server
162
+ serv-u:
163
+ vendor: solarwinds
164
+ squid_cache:
165
+ vendor: squid-cache
166
+ ssh_communications_security:
167
+ vendor: ssh
168
+ products:
169
+ ssh_tectia_server: tectia_server
170
+ standard_networks:
171
+ vendor: ipswitch
172
+ swagger:
173
+ vendor: smartbear
174
+ synology:
175
+ products:
176
+ dsm: diskstation_manager
177
+ tightvnc:
178
+ products:
179
+ desktop: tightvnc
180
+ tor_project:
181
+ vendor: torproject
182
+ traefik_labs:
183
+ vendor: containous
184
+ products:
185
+ traefik_proxy: traefik
186
+ twistedmatrix:
187
+ products:
188
+ twisted_web: twistedweb
189
+ ubiquiti:
190
+ vendor: ui
191
+ vandyke_software:
192
+ vendor: vandyke
193
+ vmware:
194
+ products:
195
+ zimbra: zimbra_desktop
196
+ vcenter: vcenter_server
197
+ x.org:
198
+ products:
199
+ x.org_x11: x11
200
+
201
+ # The following section contains CPE operating system or 'o' remappings. These will
202
+ # ONLY be used for mapping Recog 'os' attributes.
203
+ o:
204
+ alpine:
205
+ vendor: alpinelinux
206
+ products:
207
+ linux: alpine_linux
208
+ apple:
209
+ products:
210
+ ios: iphone_os
211
+ centos:
212
+ products:
213
+ linux: centos
214
+ check_point:
215
+ vendor: checkpoint
216
+ cisco:
217
+ products:
218
+ adaptive_security_appliance: adaptive_security_appliance_software
219
+ nam: network_analysis_module_software
220
+ pix: pix_firewall_software
221
+ telepresence: telepresence_video_communication_server_software
222
+ vpn_3000_concentrator: vpn_3000_concentrator_series_software
223
+ wireless_lan_controller: wireless_lan_controller_software
224
+ citrix:
225
+ products:
226
+ netscaler: netscaler_firmware
227
+ netscaler_gateway: netscaler_gateway_firmware
228
+ cumulus:
229
+ vendor: cumulusnetworks
230
+ data_domain:
231
+ vendor: dell
232
+ products:
233
+ dd_os: emc_data_domain_os
234
+ debian:
235
+ products:
236
+ linux: debian_linux
237
+ hp:
238
+ products:
239
+ ilo: integrated_lights-out_firmware
240
+ ilo_firmware: integrated_lights-out_firmware
241
+ ilo_2: integrated_lights-out_2_firmware
242
+ ilo_3: integrated_lights-out_3_firmware
243
+ ilo_4: integrated_lights-out_4_firmware
244
+ ilo_5: integrated_lights-out_5_firmware
245
+ tru64_unix: tru64
246
+ ibm:
247
+ products:
248
+ os/400: os_400
249
+ i5/os: i5os
250
+ juniper:
251
+ products:
252
+ junos_os: junos
253
+ linux:
254
+ products:
255
+ linux: linux_kernel
256
+ microsoft:
257
+ products:
258
+ windows_server_2003_datacenter_edition: windows_server_2003
259
+ windows_server_2003_r2: windows_server_2003
260
+ windows_2008_r2: windows_server_2008
261
+ windows_server_2008_datacenter_edition: windows_server_2008
262
+ windows_server_2008_r2: windows_server_2008
263
+ windows_server_2008_r2_datacenter_edition: windows_server_2008
264
+ windows_server_2012_r2: windows_server_2012
265
+ nt: windows_nt
266
+ windows_nt_desktop: windows_nt
267
+ windows_nt_server: windows_nt
268
+ windows_server_2000: windows_2000
269
+ windows_2000_server: windows_2000
270
+ windows_2000_datacenter_server: windows_2000
271
+ oracle:
272
+ products:
273
+ ilom: integrated_lights_out_manager_firmware
274
+ palo_alto_networks:
275
+ vendor: paloaltonetworks
276
+ red_hat:
277
+ vendor: redhat
278
+ products:
279
+ fedora_core_linux: fedora_core
280
+ sun:
281
+ products:
282
+ solaris: sunos
283
+ ubiquiti:
284
+ vendor: ui
285
+ ubuntu:
286
+ vendor: canonical
287
+ products:
288
+ linux: ubuntu_linux
289
+ vmware:
290
+ products:
291
+ photon_linux: photon_os
292
+ vmware_esx_server: esx
293
+ vmware_esxi_server: esxi
294
+ wind_river:
295
+ vendor: windriver
296
+
297
+ # The following section contains CPE hardware or 'h' remappings. These will
298
+ # ONLY be used for mapping Recog 'hw' attributes.
299
+ h:
300
+ cisco:
301
+ products:
302
+ nam: network_analysis_module
303
+ citrix:
304
+ products:
305
+ netscaler_sdx_gateway: netscaler_sdx
306
+ emc:
307
+ products:
308
+ celerra: celerra_network_attached_storage
309
+ hp:
310
+ products:
311
+ ilo: integrated_lights-out
312
+ tandberg:
313
+ vendor: cisco
314
+ ubiquiti:
315
+ vendor: ui