recog 2.3.18 → 2.3.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. checksums.yaml +4 -4
  2. data/.github/dependabot.yml +8 -0
  3. data/.github/workflows/ci.yml +26 -0
  4. data/.github/workflows/verify.yml +89 -0
  5. data/CONTRIBUTING.md +6 -0
  6. data/README.md +17 -0
  7. data/bin/recog_standardize +33 -12
  8. data/bin/recog_verify +1 -2
  9. data/cpe-remap.yaml +355 -200
  10. data/features/verify.feature +14 -14
  11. data/identifiers/README.md +24 -10
  12. data/identifiers/fields.txt +105 -0
  13. data/identifiers/hw_device.txt +8 -0
  14. data/identifiers/hw_family.txt +19 -0
  15. data/identifiers/hw_product.txt +122 -0
  16. data/identifiers/os_device.txt +2 -1
  17. data/identifiers/os_family.txt +3 -0
  18. data/identifiers/os_product.txt +46 -8
  19. data/identifiers/service_family.txt +10 -1
  20. data/identifiers/service_product.txt +90 -2
  21. data/identifiers/vendor.txt +104 -0
  22. data/lib/recog/db.rb +2 -1
  23. data/lib/recog/fingerprint.rb +18 -5
  24. data/lib/recog/nizer.rb +1 -82
  25. data/lib/recog/verifier.rb +5 -5
  26. data/lib/recog/verifier_factory.rb +3 -3
  27. data/lib/recog/verify_reporter.rb +14 -4
  28. data/lib/recog/version.rb +1 -1
  29. data/requirements.txt +1 -1
  30. data/spec/lib/fingerprint_self_test_spec.rb +1 -0
  31. data/spec/lib/recog/verify_reporter_spec.rb +69 -0
  32. data/tools/dev/hooks/pre-commit +21 -0
  33. data/update_cpes.py +19 -6
  34. data/xml/apache_modules.xml +60 -0
  35. data/xml/apache_os.xml +38 -38
  36. data/xml/dhcp_vendor_class.xml +206 -0
  37. data/xml/dns_versionbind.xml +11 -1
  38. data/xml/favicons.xml +270 -45
  39. data/xml/ftp_banners.xml +89 -64
  40. data/xml/h323_callresp.xml +99 -99
  41. data/xml/hp_pjl_id.xml +3 -3
  42. data/xml/html_title.xml +1051 -62
  43. data/xml/http_cookies.xml +294 -85
  44. data/xml/http_servers.xml +551 -122
  45. data/xml/http_wwwauth.xml +139 -43
  46. data/xml/imap_banners.xml +8 -8
  47. data/xml/ldap_searchresult.xml +1 -0
  48. data/xml/mdns_device-info_txt.xml +720 -27
  49. data/xml/mysql_banners.xml +3 -2
  50. data/xml/nntp_banners.xml +4 -4
  51. data/xml/ntp_banners.xml +79 -65
  52. data/xml/operating_system.xml +6 -6
  53. data/xml/pop_banners.xml +11 -11
  54. data/xml/rsh_resp.xml +3 -3
  55. data/xml/rtsp_servers.xml +7 -0
  56. data/xml/sip_banners.xml +374 -9
  57. data/xml/sip_user_agents.xml +377 -5
  58. data/xml/smb_native_lm.xml +32 -1
  59. data/xml/smb_native_os.xml +160 -33
  60. data/xml/smtp_banners.xml +168 -129
  61. data/xml/smtp_ehlo.xml +1 -1
  62. data/xml/smtp_expn.xml +1 -0
  63. data/xml/smtp_help.xml +10 -10
  64. data/xml/smtp_noop.xml +2 -2
  65. data/xml/smtp_vrfy.xml +1 -0
  66. data/xml/snmp_sysdescr.xml +508 -214
  67. data/xml/snmp_sysobjid.xml +25 -25
  68. data/xml/ssh_banners.xml +145 -29
  69. data/xml/telnet_banners.xml +240 -61
  70. data/xml/tls_jarm.xml +162 -0
  71. data/xml/x509_issuers.xml +237 -2
  72. data/xml/x509_subjects.xml +369 -49
  73. metadata +10 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 8494784d36925a1996a74aeca8ffd899bf049dd9c03a959c067caadc4e9ecea4
4
- data.tar.gz: '050028f6d74692c1c14d7b4e3f8a8d7a2ef5a258f916b6bb50b911ceb132aa7e'
3
+ metadata.gz: 5e33013be558344280c798bb77321b809289a604f55a9f8447bba1cdd2b93151
4
+ data.tar.gz: 4088c7af5a4dc2250562a610f2bbc6307436fefabe1203f1bd00fd2a9f9c0e3d
5
5
  SHA512:
6
- metadata.gz: 640f3433ec61e3c320a380a761e569c4db81dcc29899a7f5232e5b800b2209c35bcfcc5fa990d56a42d4867908866a5cdaff44417bc4b81258127cc451795e55
7
- data.tar.gz: c4ab3c69bcca4c2b1b47724ced99e7308b62dab88df94e3b031f3f45ad7debc56a4a8e5642aec66e370663419ae3c6e9d186ff2be42f41ed22e67c9ac00dac6e
6
+ metadata.gz: abca30fbb5b218e69a2178c0b9b7337af15f1611d67076031ee5d489140505c558fe8981513d8d446bddc91b4166f304b88fed9eb23e22de0d4a5e10f2bfb668
7
+ data.tar.gz: 57c7e248435b5d52860cd2117a814176673c0e8cb193f854880b3244c91d9b745ee67ca1bc8f2f67f1f3046d26857ca198c81a0e129f9d50b8cc28349070d250
@@ -0,0 +1,8 @@
1
+ version: 2
2
+ updates:
3
+ - package-ecosystem: bundler
4
+ directory: "/"
5
+ schedule:
6
+ interval: daily
7
+ time: "11:00"
8
+ open-pull-requests-limit: 10
@@ -0,0 +1,26 @@
1
+ name: CI
2
+
3
+ on: [push, pull_request]
4
+
5
+ jobs:
6
+ test:
7
+ name: 'Ruby: ${{ matrix.ruby-version }}'
8
+ runs-on: ubuntu-latest
9
+ strategy:
10
+ fail-fast: false
11
+ matrix:
12
+ ruby-version: ['2.5', '2.6', '2.7', '3.0', 'jruby-9.1.17.0', 'jruby']
13
+
14
+ steps:
15
+ - uses: actions/checkout@v2
16
+ - name: Set up Ruby
17
+ uses: ruby/setup-ruby@v1
18
+ with:
19
+ ruby-version: ${{ matrix.ruby-version }}
20
+ bundler-cache: true # runs 'bundle install' and caches installed gems automatically
21
+ - name: Run tests
22
+ run: |
23
+ bundle exec rake --version
24
+ bundle exec rake tests
25
+ env:
26
+ JRUBY_OPTS: --server -J-Xms512m -J-Xmx2G
@@ -0,0 +1,89 @@
1
+ name: Verify
2
+
3
+ on:
4
+ push:
5
+ branches:
6
+ - master
7
+ paths:
8
+ - 'xml/**.xml'
9
+ pull_request:
10
+ paths:
11
+ - 'xml/**.xml'
12
+
13
+ jobs:
14
+ standardize:
15
+ name: 'Standardize'
16
+ runs-on: ubuntu-latest
17
+ strategy:
18
+ fail-fast: false
19
+
20
+ steps:
21
+ - uses: actions/checkout@v2
22
+ - uses: ruby/setup-ruby@v1
23
+ with:
24
+ bundler-cache: true # runs 'bundle install' and caches installed gems automatically
25
+ - name: Run recog standardize
26
+ run: bundle exec bin/recog_standardize xml/*.xml
27
+ ruby-verify:
28
+ name: 'Ruby Verify'
29
+ runs-on: ubuntu-latest
30
+ strategy:
31
+ fail-fast: false
32
+
33
+ steps:
34
+ - name: Checkout Ruby implementation
35
+ uses: actions/checkout@v2
36
+ - uses: ruby/setup-ruby@v1
37
+ with:
38
+ bundler-cache: true # runs 'bundle install' and caches installed gems automatically
39
+ - name: Run recog verify
40
+ run: bundle exec recog_verify --no-warnings xml/*.xml
41
+ java-verify:
42
+ name: 'Java Verify'
43
+ runs-on: ubuntu-latest
44
+ strategy:
45
+ fail-fast: false
46
+
47
+ steps:
48
+ - name: Checkout Java implementation
49
+ uses: actions/checkout@v2
50
+ with:
51
+ repository: rapid7/recog-java
52
+ - name: Checkout recog content
53
+ uses: actions/checkout@v2
54
+ with:
55
+ path: recog-content
56
+ - uses: actions/setup-java@v2
57
+ with:
58
+ distribution: zulu
59
+ java-version: '17'
60
+ - name: Cache Maven packages
61
+ uses: actions/cache@v2
62
+ with:
63
+ path: ~/.m2
64
+ key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
65
+ restore-keys: ${{ runner.os }}-m2
66
+ - name: Build with Maven
67
+ run: mvn --batch-mode --no-transfer-progress install -Dmaven.antrun.skip=true -DskipTests
68
+ - name: Run recog verify
69
+ run: mvn --batch-mode --no-transfer-progress --projects recog-verify exec:java -Dexec.mainClass="com.rapid7.recog.verify.RecogVerifier" -Dexec.args="--no-warnings recog-content/xml/*.xml"
70
+ go-verify:
71
+ name: 'Go Verify'
72
+ runs-on: ubuntu-latest
73
+ strategy:
74
+ fail-fast: false
75
+
76
+ steps:
77
+ - name: Checkout Go implementation
78
+ uses: actions/checkout@v2
79
+ with:
80
+ repository: RumbleDiscovery/recog-go
81
+ - name: Checkout recog content
82
+ uses: actions/checkout@v2
83
+ with:
84
+ path: recog-content
85
+ - uses: actions/setup-go@v2
86
+ with:
87
+ go-version: '^1.17.1'
88
+ - name: Run recog verify
89
+ run: go run cmd/recog_verify/main.go recog-content/xml/
data/CONTRIBUTING.md CHANGED
@@ -74,6 +74,12 @@ Generally, this should only need to be done once, or if you need to start over.
74
74
  git fetch --all
75
75
  ```
76
76
 
77
+ 1. Set up git hooks to help identify potential issues with your contributions:
78
+
79
+ ```bash
80
+ ln -sf ../../tools/dev/hooks/pre-commit .git/hooks/pre-commit
81
+ ```
82
+
77
83
  [^back to top](#contributing-to-recog)
78
84
 
79
85
  ### Branch and Improve
data/README.md CHANGED
@@ -76,6 +76,23 @@ The `example` string can be base64 encoded to permit the use of unprintable char
76
76
  </example>
77
77
  ````
78
78
 
79
+ Additionally, examples can be placed in a directory with the same base name as the XML file, in the same directory as the XML file:
80
+
81
+ ```
82
+ xml/services.xml
83
+ xml/services/file1
84
+ xml/services/file2
85
+ ...
86
+ ```
87
+
88
+ They can then be loaded using the `_filename` attribute:
89
+
90
+ ```xml
91
+ <example _filename="file1"/>
92
+ ```
93
+
94
+ This is useful for long examples.
95
+
79
96
  [^back to top](#recog-a-recognition-framework)
80
97
 
81
98
  ## Contributing
@@ -50,6 +50,7 @@ end
50
50
 
51
51
  # Load the unique identifiers
52
52
  vendors = load_identifiers(File.join(bdir, "vendor.txt"))
53
+ fields = load_identifiers(File.join(bdir, "fields.txt"))
53
54
  os_arch = load_identifiers(File.join(bdir, "os_architecture.txt"))
54
55
  os_prod = load_identifiers(File.join(bdir, "os_product.txt"))
55
56
  os_family = load_identifiers(File.join(bdir, "os_family.txt"))
@@ -60,6 +61,7 @@ hw_device = load_identifiers(File.join(bdir, "hw_device.txt"))
60
61
  svc_prod = load_identifiers(File.join(bdir, "service_product.txt"))
61
62
  svc_family = load_identifiers(File.join(bdir, "service_family.txt"))
62
63
 
64
+ missing_count = 0
63
65
 
64
66
  ARGV.each do |arg|
65
67
  Dir.glob(arg).each do |file|
@@ -67,6 +69,11 @@ ARGV.each do |arg|
67
69
  ndb.fingerprints.each do |f|
68
70
  f.params.each do |k,v|
69
71
  paramIndex, val = v
72
+ if ! fields[k]
73
+ puts "FIELD MISSING: #{k}"
74
+ missing_count += 1
75
+ fields[k] = true
76
+ end
70
77
  next if paramIndex != 0
71
78
  next if val.index("{") != nil
72
79
  next if val.strip == ""
@@ -74,51 +81,61 @@ ARGV.each do |arg|
74
81
  when "os.vendor", "service.vendor", "service.component.vendor", "hw.vendor"
75
82
  if ! vendors[val]
76
83
  puts "VENDOR MISSING: #{val}"
84
+ missing_count += 1
77
85
  vendors[val] = true
78
86
  end
79
87
  when "os.arch"
80
88
  if ! os_arch[val]
81
89
  puts "OS ARCH MISSING: #{val}"
90
+ missing_count += 1
82
91
  os_arch[val] = true
83
92
  end
84
93
  when "os.product"
85
94
  if ! os_prod[val]
86
95
  puts "OS PRODUCT MISSING: #{val}"
96
+ missing_count += 1
87
97
  os_prod[val] = true
88
98
  end
89
99
  when "os.family"
90
100
  if ! os_family[val]
91
101
  puts "OS FAMILY MISSING: #{val}"
102
+ missing_count += 1
92
103
  os_family[val] = true
93
104
  end
94
105
  when "os.device"
95
106
  if ! os_device[val]
96
107
  puts "OS DEVICE MISSING: #{val}"
108
+ missing_count += 1
97
109
  os_device[val] = true
98
110
  end
99
111
  when "hw.product"
100
112
  if ! hw_prod[val]
101
113
  puts "HW PRODUCT MISSING: #{val}"
114
+ missing_count += 1
102
115
  hw_prod[val] = true
103
116
  end
104
117
  when "hw.family"
105
118
  if ! hw_family[val]
106
119
  puts "HW FAMILY MISSING: #{val}"
120
+ missing_count += 1
107
121
  hw_family[val] = true
108
122
  end
109
123
  when "hw.device"
110
124
  if ! hw_device[val]
111
125
  puts "HW DEVICE MISSING: #{val}"
126
+ missing_count += 1
112
127
  hw_device[val] = true
113
128
  end
114
129
  when "service.product", "service.component.product"
115
130
  if ! svc_prod[val]
116
131
  puts "SERVICE PRODUCT MISSING: #{val}"
132
+ missing_count += 1
117
133
  svc_prod[val] = true
118
134
  end
119
135
  when "service.family"
120
136
  if ! svc_family[val]
121
137
  puts "SERVICE FAMILY MISSING: #{val}"
138
+ missing_count += 1
122
139
  svc_family[val] = true
123
140
  end
124
141
  end
@@ -127,16 +144,20 @@ ARGV.each do |arg|
127
144
  end
128
145
  end
129
146
 
130
- exit if ! options.write
147
+ if options.write
148
+ # Write back the unique identifiers
149
+ write_identifiers(vendors, File.join(bdir, "vendor.txt"))
150
+ write_identifiers(fields, File.join(bdir, "fields.txt"))
151
+ write_identifiers(os_arch, File.join(bdir, "os_architecture.txt"))
152
+ write_identifiers(os_prod, File.join(bdir, "os_product.txt"))
153
+ write_identifiers(os_family, File.join(bdir, "os_family.txt"))
154
+ write_identifiers(os_device, File.join(bdir, "os_device.txt"))
155
+ write_identifiers(hw_prod, File.join(bdir, "hw_product.txt"))
156
+ write_identifiers(hw_family, File.join(bdir, "hw_family.txt"))
157
+ write_identifiers(hw_device, File.join(bdir, "hw_device.txt"))
158
+ write_identifiers(svc_prod, File.join(bdir, "service_product.txt"))
159
+ write_identifiers(svc_family, File.join(bdir, "service_family.txt"))
160
+ end
131
161
 
132
- # Write back the unique identifiers
133
- write_identifiers(vendors, File.join(bdir, "vendor.txt"))
134
- write_identifiers(os_arch, File.join(bdir, "os_architecture.txt"))
135
- write_identifiers(os_prod, File.join(bdir, "os_product.txt"))
136
- write_identifiers(os_family, File.join(bdir, "os_family.txt"))
137
- write_identifiers(os_device, File.join(bdir, "os_device.txt"))
138
- write_identifiers(hw_prod, File.join(bdir, "hw_product.txt"))
139
- write_identifiers(hw_family, File.join(bdir, "hw_family.txt"))
140
- write_identifiers(hw_device, File.join(bdir, "hw_device.txt"))
141
- write_identifiers(svc_prod, File.join(bdir, "service_product.txt"))
142
- write_identifiers(svc_family, File.join(bdir, "service_family.txt"))
162
+ exit_code = (missing_count > 0 ? 1 : 0)
163
+ exit(exit_code)
data/bin/recog_verify CHANGED
@@ -53,8 +53,7 @@ failures = 0
53
53
  ARGV.each do |arg|
54
54
  Dir.glob(arg).each do |file|
55
55
  ndb = Recog::DB.new(file)
56
- options.fingerprints = ndb.fingerprints
57
- verifier = Recog::VerifierFactory.build(options)
56
+ verifier = Recog::VerifierFactory.build(options, ndb)
58
57
  verified = verifier.verify
59
58
  failures += verifier.reporter.failure_count
60
59
  warnings += verifier.reporter.warning_count