recog 2.3.21 → 2.3.22

Sign up to get free protection for your applications and to get access to all the features.
Files changed (59) hide show
  1. checksums.yaml +4 -4
  2. data/.github/dependabot.yml +8 -0
  3. data/.github/workflows/verify.yml +89 -0
  4. data/CONTRIBUTING.md +6 -0
  5. data/README.md +17 -0
  6. data/bin/recog_standardize +28 -13
  7. data/bin/recog_verify +1 -2
  8. data/cpe-remap.yaml +13 -0
  9. data/features/verify.feature +14 -14
  10. data/identifiers/fields.txt +5 -4
  11. data/identifiers/hw_device.txt +6 -0
  12. data/identifiers/hw_family.txt +8 -0
  13. data/identifiers/hw_product.txt +51 -0
  14. data/identifiers/os_family.txt +1 -0
  15. data/identifiers/os_product.txt +10 -0
  16. data/identifiers/service_product.txt +12 -0
  17. data/identifiers/vendor.txt +49 -0
  18. data/lib/recog/db.rb +2 -1
  19. data/lib/recog/fingerprint.rb +18 -5
  20. data/lib/recog/verifier.rb +5 -5
  21. data/lib/recog/verifier_factory.rb +3 -3
  22. data/lib/recog/verify_reporter.rb +14 -4
  23. data/lib/recog/version.rb +1 -1
  24. data/spec/lib/fingerprint_self_test_spec.rb +1 -0
  25. data/spec/lib/recog/verify_reporter_spec.rb +69 -0
  26. data/tools/dev/hooks/pre-commit +21 -0
  27. data/update_cpes.py +1 -1
  28. data/xml/apache_os.xml +38 -38
  29. data/xml/dhcp_vendor_class.xml +206 -0
  30. data/xml/favicons.xml +148 -42
  31. data/xml/ftp_banners.xml +30 -16
  32. data/xml/h323_callresp.xml +99 -99
  33. data/xml/hp_pjl_id.xml +3 -3
  34. data/xml/html_title.xml +502 -25
  35. data/xml/http_cookies.xml +64 -56
  36. data/xml/http_servers.xml +74 -14
  37. data/xml/http_wwwauth.xml +107 -38
  38. data/xml/imap_banners.xml +3 -3
  39. data/xml/mdns_device-info_txt.xml +389 -26
  40. data/xml/mysql_banners.xml +1 -1
  41. data/xml/nntp_banners.xml +3 -3
  42. data/xml/ntp_banners.xml +64 -64
  43. data/xml/operating_system.xml +3 -3
  44. data/xml/pop_banners.xml +7 -7
  45. data/xml/rsh_resp.xml +3 -3
  46. data/xml/sip_banners.xml +27 -0
  47. data/xml/sip_user_agents.xml +54 -1
  48. data/xml/smtp_banners.xml +15 -15
  49. data/xml/smtp_ehlo.xml +1 -1
  50. data/xml/smtp_help.xml +10 -10
  51. data/xml/smtp_noop.xml +2 -2
  52. data/xml/snmp_sysdescr.xml +325 -200
  53. data/xml/snmp_sysobjid.xml +25 -25
  54. data/xml/ssh_banners.xml +7 -5
  55. data/xml/telnet_banners.xml +155 -20
  56. data/xml/tls_jarm.xml +26 -4
  57. data/xml/x509_issuers.xml +36 -0
  58. data/xml/x509_subjects.xml +136 -35
  59. metadata +7 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 972b7cc1ae69526b61f221eeefce61d192ccf4b1603342f94195bf6cd2ddba95
4
- data.tar.gz: 807831da5cdfd3160bca893367c92f4b817514758b996968829253f548d19709
3
+ metadata.gz: 5e33013be558344280c798bb77321b809289a604f55a9f8447bba1cdd2b93151
4
+ data.tar.gz: 4088c7af5a4dc2250562a610f2bbc6307436fefabe1203f1bd00fd2a9f9c0e3d
5
5
  SHA512:
6
- metadata.gz: a923e57f1f34fb74358756372fc3d3d08c20f0a0b9b1088905f57f4b09a2a56b2d9d4940d39e609ff50c164d15e285c9e1707032864d55f969a7ed4d72e68de5
7
- data.tar.gz: 62d17cd2cdf9c3a6d35b36e4ace9c20744d42cfd99a2e90f65e463540fceb551f35f7572179af261c8881116aeadc51986e371631dc451df31081d52d79a58c1
6
+ metadata.gz: abca30fbb5b218e69a2178c0b9b7337af15f1611d67076031ee5d489140505c558fe8981513d8d446bddc91b4166f304b88fed9eb23e22de0d4a5e10f2bfb668
7
+ data.tar.gz: 57c7e248435b5d52860cd2117a814176673c0e8cb193f854880b3244c91d9b745ee67ca1bc8f2f67f1f3046d26857ca198c81a0e129f9d50b8cc28349070d250
@@ -0,0 +1,8 @@
1
+ version: 2
2
+ updates:
3
+ - package-ecosystem: bundler
4
+ directory: "/"
5
+ schedule:
6
+ interval: daily
7
+ time: "11:00"
8
+ open-pull-requests-limit: 10
@@ -0,0 +1,89 @@
1
+ name: Verify
2
+
3
+ on:
4
+ push:
5
+ branches:
6
+ - master
7
+ paths:
8
+ - 'xml/**.xml'
9
+ pull_request:
10
+ paths:
11
+ - 'xml/**.xml'
12
+
13
+ jobs:
14
+ standardize:
15
+ name: 'Standardize'
16
+ runs-on: ubuntu-latest
17
+ strategy:
18
+ fail-fast: false
19
+
20
+ steps:
21
+ - uses: actions/checkout@v2
22
+ - uses: ruby/setup-ruby@v1
23
+ with:
24
+ bundler-cache: true # runs 'bundle install' and caches installed gems automatically
25
+ - name: Run recog standardize
26
+ run: bundle exec bin/recog_standardize xml/*.xml
27
+ ruby-verify:
28
+ name: 'Ruby Verify'
29
+ runs-on: ubuntu-latest
30
+ strategy:
31
+ fail-fast: false
32
+
33
+ steps:
34
+ - name: Checkout Ruby implementation
35
+ uses: actions/checkout@v2
36
+ - uses: ruby/setup-ruby@v1
37
+ with:
38
+ bundler-cache: true # runs 'bundle install' and caches installed gems automatically
39
+ - name: Run recog verify
40
+ run: bundle exec recog_verify --no-warnings xml/*.xml
41
+ java-verify:
42
+ name: 'Java Verify'
43
+ runs-on: ubuntu-latest
44
+ strategy:
45
+ fail-fast: false
46
+
47
+ steps:
48
+ - name: Checkout Java implementation
49
+ uses: actions/checkout@v2
50
+ with:
51
+ repository: rapid7/recog-java
52
+ - name: Checkout recog content
53
+ uses: actions/checkout@v2
54
+ with:
55
+ path: recog-content
56
+ - uses: actions/setup-java@v2
57
+ with:
58
+ distribution: zulu
59
+ java-version: '17'
60
+ - name: Cache Maven packages
61
+ uses: actions/cache@v2
62
+ with:
63
+ path: ~/.m2
64
+ key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
65
+ restore-keys: ${{ runner.os }}-m2
66
+ - name: Build with Maven
67
+ run: mvn --batch-mode --no-transfer-progress install -Dmaven.antrun.skip=true -DskipTests
68
+ - name: Run recog verify
69
+ run: mvn --batch-mode --no-transfer-progress --projects recog-verify exec:java -Dexec.mainClass="com.rapid7.recog.verify.RecogVerifier" -Dexec.args="--no-warnings recog-content/xml/*.xml"
70
+ go-verify:
71
+ name: 'Go Verify'
72
+ runs-on: ubuntu-latest
73
+ strategy:
74
+ fail-fast: false
75
+
76
+ steps:
77
+ - name: Checkout Go implementation
78
+ uses: actions/checkout@v2
79
+ with:
80
+ repository: RumbleDiscovery/recog-go
81
+ - name: Checkout recog content
82
+ uses: actions/checkout@v2
83
+ with:
84
+ path: recog-content
85
+ - uses: actions/setup-go@v2
86
+ with:
87
+ go-version: '^1.17.1'
88
+ - name: Run recog verify
89
+ run: go run cmd/recog_verify/main.go recog-content/xml/
data/CONTRIBUTING.md CHANGED
@@ -74,6 +74,12 @@ Generally, this should only need to be done once, or if you need to start over.
74
74
  git fetch --all
75
75
  ```
76
76
 
77
+ 1. Set up git hooks to help identify potential issues with your contributions:
78
+
79
+ ```bash
80
+ ln -sf ../../tools/dev/hooks/pre-commit .git/hooks/pre-commit
81
+ ```
82
+
77
83
  [^back to top](#contributing-to-recog)
78
84
 
79
85
  ### Branch and Improve
data/README.md CHANGED
@@ -76,6 +76,23 @@ The `example` string can be base64 encoded to permit the use of unprintable char
76
76
  </example>
77
77
  ````
78
78
 
79
+ Additionally, examples can be placed in a directory with the same base name as the XML file, in the same directory as the XML file:
80
+
81
+ ```
82
+ xml/services.xml
83
+ xml/services/file1
84
+ xml/services/file2
85
+ ...
86
+ ```
87
+
88
+ They can then be loaded using the `_filename` attribute:
89
+
90
+ ```xml
91
+ <example _filename="file1"/>
92
+ ```
93
+
94
+ This is useful for long examples.
95
+
79
96
  [^back to top](#recog-a-recognition-framework)
80
97
 
81
98
  ## Contributing
@@ -61,6 +61,7 @@ hw_device = load_identifiers(File.join(bdir, "hw_device.txt"))
61
61
  svc_prod = load_identifiers(File.join(bdir, "service_product.txt"))
62
62
  svc_family = load_identifiers(File.join(bdir, "service_family.txt"))
63
63
 
64
+ missing_count = 0
64
65
 
65
66
  ARGV.each do |arg|
66
67
  Dir.glob(arg).each do |file|
@@ -70,6 +71,7 @@ ARGV.each do |arg|
70
71
  paramIndex, val = v
71
72
  if ! fields[k]
72
73
  puts "FIELD MISSING: #{k}"
74
+ missing_count += 1
73
75
  fields[k] = true
74
76
  end
75
77
  next if paramIndex != 0
@@ -79,51 +81,61 @@ ARGV.each do |arg|
79
81
  when "os.vendor", "service.vendor", "service.component.vendor", "hw.vendor"
80
82
  if ! vendors[val]
81
83
  puts "VENDOR MISSING: #{val}"
84
+ missing_count += 1
82
85
  vendors[val] = true
83
86
  end
84
87
  when "os.arch"
85
88
  if ! os_arch[val]
86
89
  puts "OS ARCH MISSING: #{val}"
90
+ missing_count += 1
87
91
  os_arch[val] = true
88
92
  end
89
93
  when "os.product"
90
94
  if ! os_prod[val]
91
95
  puts "OS PRODUCT MISSING: #{val}"
96
+ missing_count += 1
92
97
  os_prod[val] = true
93
98
  end
94
99
  when "os.family"
95
100
  if ! os_family[val]
96
101
  puts "OS FAMILY MISSING: #{val}"
102
+ missing_count += 1
97
103
  os_family[val] = true
98
104
  end
99
105
  when "os.device"
100
106
  if ! os_device[val]
101
107
  puts "OS DEVICE MISSING: #{val}"
108
+ missing_count += 1
102
109
  os_device[val] = true
103
110
  end
104
111
  when "hw.product"
105
112
  if ! hw_prod[val]
106
113
  puts "HW PRODUCT MISSING: #{val}"
114
+ missing_count += 1
107
115
  hw_prod[val] = true
108
116
  end
109
117
  when "hw.family"
110
118
  if ! hw_family[val]
111
119
  puts "HW FAMILY MISSING: #{val}"
120
+ missing_count += 1
112
121
  hw_family[val] = true
113
122
  end
114
123
  when "hw.device"
115
124
  if ! hw_device[val]
116
125
  puts "HW DEVICE MISSING: #{val}"
126
+ missing_count += 1
117
127
  hw_device[val] = true
118
128
  end
119
129
  when "service.product", "service.component.product"
120
130
  if ! svc_prod[val]
121
131
  puts "SERVICE PRODUCT MISSING: #{val}"
132
+ missing_count += 1
122
133
  svc_prod[val] = true
123
134
  end
124
135
  when "service.family"
125
136
  if ! svc_family[val]
126
137
  puts "SERVICE FAMILY MISSING: #{val}"
138
+ missing_count += 1
127
139
  svc_family[val] = true
128
140
  end
129
141
  end
@@ -132,17 +144,20 @@ ARGV.each do |arg|
132
144
  end
133
145
  end
134
146
 
135
- exit if ! options.write
147
+ if options.write
148
+ # Write back the unique identifiers
149
+ write_identifiers(vendors, File.join(bdir, "vendor.txt"))
150
+ write_identifiers(fields, File.join(bdir, "fields.txt"))
151
+ write_identifiers(os_arch, File.join(bdir, "os_architecture.txt"))
152
+ write_identifiers(os_prod, File.join(bdir, "os_product.txt"))
153
+ write_identifiers(os_family, File.join(bdir, "os_family.txt"))
154
+ write_identifiers(os_device, File.join(bdir, "os_device.txt"))
155
+ write_identifiers(hw_prod, File.join(bdir, "hw_product.txt"))
156
+ write_identifiers(hw_family, File.join(bdir, "hw_family.txt"))
157
+ write_identifiers(hw_device, File.join(bdir, "hw_device.txt"))
158
+ write_identifiers(svc_prod, File.join(bdir, "service_product.txt"))
159
+ write_identifiers(svc_family, File.join(bdir, "service_family.txt"))
160
+ end
136
161
 
137
- # Write back the unique identifiers
138
- write_identifiers(vendors, File.join(bdir, "vendor.txt"))
139
- write_identifiers(fields, File.join(bdir, "fields.txt"))
140
- write_identifiers(os_arch, File.join(bdir, "os_architecture.txt"))
141
- write_identifiers(os_prod, File.join(bdir, "os_product.txt"))
142
- write_identifiers(os_family, File.join(bdir, "os_family.txt"))
143
- write_identifiers(os_device, File.join(bdir, "os_device.txt"))
144
- write_identifiers(hw_prod, File.join(bdir, "hw_product.txt"))
145
- write_identifiers(hw_family, File.join(bdir, "hw_family.txt"))
146
- write_identifiers(hw_device, File.join(bdir, "hw_device.txt"))
147
- write_identifiers(svc_prod, File.join(bdir, "service_product.txt"))
148
- write_identifiers(svc_family, File.join(bdir, "service_family.txt"))
162
+ exit_code = (missing_count > 0 ? 1 : 0)
163
+ exit(exit_code)
data/bin/recog_verify CHANGED
@@ -53,8 +53,7 @@ failures = 0
53
53
  ARGV.each do |arg|
54
54
  Dir.glob(arg).each do |file|
55
55
  ndb = Recog::DB.new(file)
56
- options.fingerprints = ndb.fingerprints
57
- verifier = Recog::VerifierFactory.build(options)
56
+ verifier = Recog::VerifierFactory.build(options, ndb)
58
57
  verified = verifier.verify
59
58
  failures += verifier.reporter.failure_count
60
59
  warnings += verifier.reporter.warning_count
data/cpe-remap.yaml CHANGED
@@ -140,6 +140,8 @@ mappings:
140
140
  parallels:
141
141
  products:
142
142
  plesk: parallels_plesk_panel
143
+ phoenix_contact:
144
+ vendor: phoenixcontact
143
145
  plesk:
144
146
  vendor: parallels
145
147
  proftpd_project:
@@ -159,6 +161,8 @@ mappings:
159
161
  jboss_eap: jboss_enterprise_application_platform
160
162
  jbossweb: jboss_web_framework_kit
161
163
  red_hat_directory_server: directory_server
164
+ rundeck:
165
+ vendor: pagerduty
162
166
  serv-u:
163
167
  vendor: solarwinds
164
168
  squid_cache:
@@ -208,6 +212,7 @@ mappings:
208
212
  apple:
209
213
  products:
210
214
  ios: iphone_os
215
+ mac_os: macos
211
216
  brocade:
212
217
  vendor: broadcom
213
218
  products:
@@ -277,10 +282,14 @@ mappings:
277
282
  ilom: integrated_lights_out_manager_firmware
278
283
  palo_alto_networks:
279
284
  vendor: paloaltonetworks
285
+ phoenix_contact:
286
+ vendor: phoenixcontact
280
287
  red_hat:
281
288
  vendor: redhat
282
289
  products:
283
290
  fedora_core_linux: fedora_core
291
+ software_house:
292
+ vendor: swhouse
284
293
  sun:
285
294
  products:
286
295
  solaris: sunos
@@ -337,6 +346,10 @@ mappings:
337
346
  vendor: dell
338
347
  products:
339
348
  k1000: kace_k1000_systems_management_appliance
349
+ phoenix_contact:
350
+ vendor: phoenixcontact
351
+ software_house:
352
+ vendor: swhouse
340
353
  tandberg:
341
354
  vendor: cisco
342
355
  ubiquiti:
@@ -2,17 +2,17 @@ Feature: Verify
2
2
  @no-clobber
3
3
  Scenario: No tests
4
4
  When I run `recog_verify no_tests.xml`
5
- Then it should pass with:
5
+ Then it should pass with exactly:
6
6
  """
7
- SUMMARY: Test completed with 0 successful, 0 warnings, and 0 failures
7
+ no_tests.xml: SUMMARY: Test completed with 0 successful, 0 warnings, and 0 failures
8
8
  """
9
9
 
10
10
  @no-clobber
11
11
  Scenario: Successful tests
12
12
  When I run `recog_verify successful_tests.xml`
13
- Then it should pass with:
13
+ Then it should pass with exactly:
14
14
  """
15
- SUMMARY: Test completed with 4 successful, 0 warnings, and 0 failures
15
+ successful_tests.xml: SUMMARY: Test completed with 4 successful, 0 warnings, and 0 failures
16
16
  """
17
17
 
18
18
  @no-clobber
@@ -20,18 +20,18 @@ Feature: Verify
20
20
  When I run `recog_verify tests_with_warnings.xml`
21
21
  Then it should fail with:
22
22
  """
23
- WARN: 'Pure-FTPd' has no test cases
24
- WARN: 'Pure-FTPd' is missing an example that checks for parameter 'pureftpd.config' messsage which is derived from a capture group
25
- SUMMARY: Test completed with 1 successful, 2 warnings, and 0 failures
23
+ tests_with_warnings.xml: WARN: 'Pure-FTPd' has no test cases
24
+ tests_with_warnings.xml: WARN: 'Pure-FTPd' is missing an example that checks for parameter 'pureftpd.config' which is derived from a capture group
25
+ tests_with_warnings.xml: SUMMARY: Test completed with 1 successful, 2 warnings, and 0 failures
26
26
  """
27
27
  And the exit status should be 2
28
28
 
29
29
  @no-clobber
30
30
  Scenario: Tests with warnings, warnings disabled
31
31
  When I run `recog_verify --no-warnings tests_with_warnings.xml`
32
- Then it should pass with:
32
+ Then it should pass with exactly:
33
33
  """
34
- SUMMARY: Test completed with 1 successful, 0 warnings, and 0 failures
34
+ tests_with_warnings.xml: SUMMARY: Test completed with 1 successful, 0 warnings, and 0 failures
35
35
  """
36
36
 
37
37
  @no-clobber
@@ -39,10 +39,10 @@ Feature: Verify
39
39
  When I run `recog_verify tests_with_failures.xml`
40
40
  Then it should fail with:
41
41
  """
42
- FAIL: 'foo test' failed to match "bar" with (?-mix:^foo$)'
43
- FAIL: '' failed to match "This almost matches" with (?-mix:^This matches$)'
44
- FAIL: 'bar test's os.name is a non-zero pos but specifies a value of 'Bar'
45
- FAIL: 'bar test' failed to find expected capture group os.version '5.0'. Result was 1.0
46
- SUMMARY: Test completed with 0 successful, 0 warnings, and 4 failures
42
+ tests_with_failures.xml: FAIL: 'foo test' failed to match "bar" with (?-mix:^foo$)'
43
+ tests_with_failures.xml: FAIL: '' failed to match "This almost matches" with (?-mix:^This matches$)'
44
+ tests_with_failures.xml: FAIL: 'bar test's os.name is a non-zero pos but specifies a value of 'Bar'
45
+ tests_with_failures.xml: FAIL: 'bar test' failed to find expected capture group os.version '5.0'. Result was 1.0
46
+ tests_with_failures.xml: SUMMARY: Test completed with 0 successful, 0 warnings, and 4 failures
47
47
  """
48
48
  And the exit status should be 4
@@ -2,15 +2,15 @@ agilent.serial
2
2
  apache.info
3
3
  apache.variant
4
4
  apache.variant.version
5
+ aptinex.model
5
6
  chromecast.capabilities
6
7
  chromecast.generation
7
- chromecast.serial_number
8
8
  cisco.imc_model
9
- cisco.serial_number
9
+ cisco.model
10
10
  cookie
11
11
  dell.service_tag
12
+ digi.serial_number
12
13
  extron.model
13
- fortinet.serial_number
14
14
  host.domain
15
15
  host.ip
16
16
  host.mac
@@ -32,6 +32,7 @@ imail.eval
32
32
  jetty.info
33
33
  junction.cookie
34
34
  junction.name
35
+ lantronix.serial_number
35
36
  lenovo.machine_model
36
37
  lenovo.machine_type
37
38
  linux.kernel.version
@@ -67,7 +68,6 @@ proftpd.server.name
67
68
  pureftpd.config
68
69
  python.version
69
70
  qpopper.version
70
- ruckus.serial_number
71
71
  securetransport.build
72
72
  sendmail.config.version
73
73
  sendmail.hpux.phne.version
@@ -100,5 +100,6 @@ system.time.millis
100
100
  tandberg.model
101
101
  thttpd.mx-patch
102
102
  tomcat.info
103
+ unify.model
103
104
  wd2go.device_id
104
105
  zmailer.ident
@@ -31,6 +31,7 @@ KVM
31
31
  Laptop
32
32
  Light Bulb
33
33
  Lights Out Management
34
+ Media Player
34
35
  Media Receiver
35
36
  Media Server
36
37
  Mobile Phone
@@ -42,6 +43,7 @@ Network Audio
42
43
  Network Management Device
43
44
  PLC
44
45
  Power Device
46
+ Power Meter
45
47
  Power Relay
46
48
  Powerline
47
49
  Print Server
@@ -53,6 +55,7 @@ SIP Device
53
55
  SIP Gateway
54
56
  Scanner
55
57
  Security Appliance
58
+ Sensor
56
59
  Smart TV
57
60
  Storage
58
61
  Storage Appliance
@@ -62,6 +65,8 @@ Tablet
62
65
  Tape Library
63
66
  Telecom
64
67
  Test Instrument
68
+ Thin Client
69
+ UPS
65
70
  VPN
66
71
  Video Conference
67
72
  Video Conferencing
@@ -73,6 +78,7 @@ VoIP Switch
73
78
  Voice Appliance
74
79
  WAP
75
80
  WLAN Repeater
81
+ Web Cam
76
82
  Whiteboard
77
83
  Wireless Controller
78
84
  Wireless Presenter
@@ -9,6 +9,7 @@ Data ONTAP
9
9
  DiskStation
10
10
  Document Centre
11
11
  EDR
12
+ Eurotherm
12
13
  Extended Systems ExtendNet
13
14
  FRITZ!Box
14
15
  FRITZ!Fon
@@ -30,6 +31,7 @@ ILOM
30
31
  IMDVR
31
32
  ION
32
33
  JetDirect
34
+ LX Series
33
35
  LaserJet
34
36
  LinkCom Xpress
35
37
  MGate
@@ -51,6 +53,7 @@ Netscaler
51
53
  Network Audio
52
54
  Network Security Appliance
53
55
  Network Video Door Station
56
+ OpenScape Desk Phone
54
57
  Optra
55
58
  Orbi
56
59
  POWER System
@@ -91,6 +94,7 @@ UniFi
91
94
  Unified Security Gateway
92
95
  VDX
93
96
  VSX
97
+ VVX
94
98
  Vigor
95
99
  VoIP
96
100
  WD2GO
@@ -111,3 +115,7 @@ iPad Air
111
115
  iPad Pro
112
116
  iPad mini
113
117
  iPhone
118
+ iPod
119
+ iSTAR Door Controllers
120
+ imageClass
121
+ imageRunner