recog 2.3.18 → 2.3.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/dependabot.yml +8 -0
- data/.github/workflows/ci.yml +26 -0
- data/.github/workflows/verify.yml +89 -0
- data/CONTRIBUTING.md +6 -0
- data/README.md +17 -0
- data/bin/recog_standardize +33 -12
- data/bin/recog_verify +1 -2
- data/cpe-remap.yaml +355 -200
- data/features/verify.feature +14 -14
- data/identifiers/README.md +24 -10
- data/identifiers/fields.txt +105 -0
- data/identifiers/hw_device.txt +8 -0
- data/identifiers/hw_family.txt +19 -0
- data/identifiers/hw_product.txt +122 -0
- data/identifiers/os_device.txt +2 -1
- data/identifiers/os_family.txt +3 -0
- data/identifiers/os_product.txt +46 -8
- data/identifiers/service_family.txt +10 -1
- data/identifiers/service_product.txt +90 -2
- data/identifiers/vendor.txt +104 -0
- data/lib/recog/db.rb +2 -1
- data/lib/recog/fingerprint.rb +18 -5
- data/lib/recog/nizer.rb +1 -82
- data/lib/recog/verifier.rb +5 -5
- data/lib/recog/verifier_factory.rb +3 -3
- data/lib/recog/verify_reporter.rb +14 -4
- data/lib/recog/version.rb +1 -1
- data/requirements.txt +1 -1
- data/spec/lib/fingerprint_self_test_spec.rb +1 -0
- data/spec/lib/recog/verify_reporter_spec.rb +69 -0
- data/tools/dev/hooks/pre-commit +21 -0
- data/update_cpes.py +19 -6
- data/xml/apache_modules.xml +60 -0
- data/xml/apache_os.xml +38 -38
- data/xml/dhcp_vendor_class.xml +206 -0
- data/xml/dns_versionbind.xml +11 -1
- data/xml/favicons.xml +270 -45
- data/xml/ftp_banners.xml +89 -64
- data/xml/h323_callresp.xml +99 -99
- data/xml/hp_pjl_id.xml +3 -3
- data/xml/html_title.xml +1051 -62
- data/xml/http_cookies.xml +294 -85
- data/xml/http_servers.xml +551 -122
- data/xml/http_wwwauth.xml +139 -43
- data/xml/imap_banners.xml +8 -8
- data/xml/ldap_searchresult.xml +1 -0
- data/xml/mdns_device-info_txt.xml +720 -27
- data/xml/mysql_banners.xml +3 -2
- data/xml/nntp_banners.xml +4 -4
- data/xml/ntp_banners.xml +79 -65
- data/xml/operating_system.xml +6 -6
- data/xml/pop_banners.xml +11 -11
- data/xml/rsh_resp.xml +3 -3
- data/xml/rtsp_servers.xml +7 -0
- data/xml/sip_banners.xml +374 -9
- data/xml/sip_user_agents.xml +377 -5
- data/xml/smb_native_lm.xml +32 -1
- data/xml/smb_native_os.xml +160 -33
- data/xml/smtp_banners.xml +168 -129
- data/xml/smtp_ehlo.xml +1 -1
- data/xml/smtp_expn.xml +1 -0
- data/xml/smtp_help.xml +10 -10
- data/xml/smtp_noop.xml +2 -2
- data/xml/smtp_vrfy.xml +1 -0
- data/xml/snmp_sysdescr.xml +508 -214
- data/xml/snmp_sysobjid.xml +25 -25
- data/xml/ssh_banners.xml +145 -29
- data/xml/telnet_banners.xml +240 -61
- data/xml/tls_jarm.xml +162 -0
- data/xml/x509_issuers.xml +237 -2
- data/xml/x509_subjects.xml +369 -49
- metadata +10 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5e33013be558344280c798bb77321b809289a604f55a9f8447bba1cdd2b93151
|
4
|
+
data.tar.gz: 4088c7af5a4dc2250562a610f2bbc6307436fefabe1203f1bd00fd2a9f9c0e3d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: abca30fbb5b218e69a2178c0b9b7337af15f1611d67076031ee5d489140505c558fe8981513d8d446bddc91b4166f304b88fed9eb23e22de0d4a5e10f2bfb668
|
7
|
+
data.tar.gz: 57c7e248435b5d52860cd2117a814176673c0e8cb193f854880b3244c91d9b745ee67ca1bc8f2f67f1f3046d26857ca198c81a0e129f9d50b8cc28349070d250
|
@@ -0,0 +1,26 @@
|
|
1
|
+
name: CI
|
2
|
+
|
3
|
+
on: [push, pull_request]
|
4
|
+
|
5
|
+
jobs:
|
6
|
+
test:
|
7
|
+
name: 'Ruby: ${{ matrix.ruby-version }}'
|
8
|
+
runs-on: ubuntu-latest
|
9
|
+
strategy:
|
10
|
+
fail-fast: false
|
11
|
+
matrix:
|
12
|
+
ruby-version: ['2.5', '2.6', '2.7', '3.0', 'jruby-9.1.17.0', 'jruby']
|
13
|
+
|
14
|
+
steps:
|
15
|
+
- uses: actions/checkout@v2
|
16
|
+
- name: Set up Ruby
|
17
|
+
uses: ruby/setup-ruby@v1
|
18
|
+
with:
|
19
|
+
ruby-version: ${{ matrix.ruby-version }}
|
20
|
+
bundler-cache: true # runs 'bundle install' and caches installed gems automatically
|
21
|
+
- name: Run tests
|
22
|
+
run: |
|
23
|
+
bundle exec rake --version
|
24
|
+
bundle exec rake tests
|
25
|
+
env:
|
26
|
+
JRUBY_OPTS: --server -J-Xms512m -J-Xmx2G
|
@@ -0,0 +1,89 @@
|
|
1
|
+
name: Verify
|
2
|
+
|
3
|
+
on:
|
4
|
+
push:
|
5
|
+
branches:
|
6
|
+
- master
|
7
|
+
paths:
|
8
|
+
- 'xml/**.xml'
|
9
|
+
pull_request:
|
10
|
+
paths:
|
11
|
+
- 'xml/**.xml'
|
12
|
+
|
13
|
+
jobs:
|
14
|
+
standardize:
|
15
|
+
name: 'Standardize'
|
16
|
+
runs-on: ubuntu-latest
|
17
|
+
strategy:
|
18
|
+
fail-fast: false
|
19
|
+
|
20
|
+
steps:
|
21
|
+
- uses: actions/checkout@v2
|
22
|
+
- uses: ruby/setup-ruby@v1
|
23
|
+
with:
|
24
|
+
bundler-cache: true # runs 'bundle install' and caches installed gems automatically
|
25
|
+
- name: Run recog standardize
|
26
|
+
run: bundle exec bin/recog_standardize xml/*.xml
|
27
|
+
ruby-verify:
|
28
|
+
name: 'Ruby Verify'
|
29
|
+
runs-on: ubuntu-latest
|
30
|
+
strategy:
|
31
|
+
fail-fast: false
|
32
|
+
|
33
|
+
steps:
|
34
|
+
- name: Checkout Ruby implementation
|
35
|
+
uses: actions/checkout@v2
|
36
|
+
- uses: ruby/setup-ruby@v1
|
37
|
+
with:
|
38
|
+
bundler-cache: true # runs 'bundle install' and caches installed gems automatically
|
39
|
+
- name: Run recog verify
|
40
|
+
run: bundle exec recog_verify --no-warnings xml/*.xml
|
41
|
+
java-verify:
|
42
|
+
name: 'Java Verify'
|
43
|
+
runs-on: ubuntu-latest
|
44
|
+
strategy:
|
45
|
+
fail-fast: false
|
46
|
+
|
47
|
+
steps:
|
48
|
+
- name: Checkout Java implementation
|
49
|
+
uses: actions/checkout@v2
|
50
|
+
with:
|
51
|
+
repository: rapid7/recog-java
|
52
|
+
- name: Checkout recog content
|
53
|
+
uses: actions/checkout@v2
|
54
|
+
with:
|
55
|
+
path: recog-content
|
56
|
+
- uses: actions/setup-java@v2
|
57
|
+
with:
|
58
|
+
distribution: zulu
|
59
|
+
java-version: '17'
|
60
|
+
- name: Cache Maven packages
|
61
|
+
uses: actions/cache@v2
|
62
|
+
with:
|
63
|
+
path: ~/.m2
|
64
|
+
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
|
65
|
+
restore-keys: ${{ runner.os }}-m2
|
66
|
+
- name: Build with Maven
|
67
|
+
run: mvn --batch-mode --no-transfer-progress install -Dmaven.antrun.skip=true -DskipTests
|
68
|
+
- name: Run recog verify
|
69
|
+
run: mvn --batch-mode --no-transfer-progress --projects recog-verify exec:java -Dexec.mainClass="com.rapid7.recog.verify.RecogVerifier" -Dexec.args="--no-warnings recog-content/xml/*.xml"
|
70
|
+
go-verify:
|
71
|
+
name: 'Go Verify'
|
72
|
+
runs-on: ubuntu-latest
|
73
|
+
strategy:
|
74
|
+
fail-fast: false
|
75
|
+
|
76
|
+
steps:
|
77
|
+
- name: Checkout Go implementation
|
78
|
+
uses: actions/checkout@v2
|
79
|
+
with:
|
80
|
+
repository: RumbleDiscovery/recog-go
|
81
|
+
- name: Checkout recog content
|
82
|
+
uses: actions/checkout@v2
|
83
|
+
with:
|
84
|
+
path: recog-content
|
85
|
+
- uses: actions/setup-go@v2
|
86
|
+
with:
|
87
|
+
go-version: '^1.17.1'
|
88
|
+
- name: Run recog verify
|
89
|
+
run: go run cmd/recog_verify/main.go recog-content/xml/
|
data/CONTRIBUTING.md
CHANGED
@@ -74,6 +74,12 @@ Generally, this should only need to be done once, or if you need to start over.
|
|
74
74
|
git fetch --all
|
75
75
|
```
|
76
76
|
|
77
|
+
1. Set up git hooks to help identify potential issues with your contributions:
|
78
|
+
|
79
|
+
```bash
|
80
|
+
ln -sf ../../tools/dev/hooks/pre-commit .git/hooks/pre-commit
|
81
|
+
```
|
82
|
+
|
77
83
|
[^back to top](#contributing-to-recog)
|
78
84
|
|
79
85
|
### Branch and Improve
|
data/README.md
CHANGED
@@ -76,6 +76,23 @@ The `example` string can be base64 encoded to permit the use of unprintable char
|
|
76
76
|
</example>
|
77
77
|
````
|
78
78
|
|
79
|
+
Additionally, examples can be placed in a directory with the same base name as the XML file, in the same directory as the XML file:
|
80
|
+
|
81
|
+
```
|
82
|
+
xml/services.xml
|
83
|
+
xml/services/file1
|
84
|
+
xml/services/file2
|
85
|
+
...
|
86
|
+
```
|
87
|
+
|
88
|
+
They can then be loaded using the `_filename` attribute:
|
89
|
+
|
90
|
+
```xml
|
91
|
+
<example _filename="file1"/>
|
92
|
+
```
|
93
|
+
|
94
|
+
This is useful for long examples.
|
95
|
+
|
79
96
|
[^back to top](#recog-a-recognition-framework)
|
80
97
|
|
81
98
|
## Contributing
|
data/bin/recog_standardize
CHANGED
@@ -50,6 +50,7 @@ end
|
|
50
50
|
|
51
51
|
# Load the unique identifiers
|
52
52
|
vendors = load_identifiers(File.join(bdir, "vendor.txt"))
|
53
|
+
fields = load_identifiers(File.join(bdir, "fields.txt"))
|
53
54
|
os_arch = load_identifiers(File.join(bdir, "os_architecture.txt"))
|
54
55
|
os_prod = load_identifiers(File.join(bdir, "os_product.txt"))
|
55
56
|
os_family = load_identifiers(File.join(bdir, "os_family.txt"))
|
@@ -60,6 +61,7 @@ hw_device = load_identifiers(File.join(bdir, "hw_device.txt"))
|
|
60
61
|
svc_prod = load_identifiers(File.join(bdir, "service_product.txt"))
|
61
62
|
svc_family = load_identifiers(File.join(bdir, "service_family.txt"))
|
62
63
|
|
64
|
+
missing_count = 0
|
63
65
|
|
64
66
|
ARGV.each do |arg|
|
65
67
|
Dir.glob(arg).each do |file|
|
@@ -67,6 +69,11 @@ ARGV.each do |arg|
|
|
67
69
|
ndb.fingerprints.each do |f|
|
68
70
|
f.params.each do |k,v|
|
69
71
|
paramIndex, val = v
|
72
|
+
if ! fields[k]
|
73
|
+
puts "FIELD MISSING: #{k}"
|
74
|
+
missing_count += 1
|
75
|
+
fields[k] = true
|
76
|
+
end
|
70
77
|
next if paramIndex != 0
|
71
78
|
next if val.index("{") != nil
|
72
79
|
next if val.strip == ""
|
@@ -74,51 +81,61 @@ ARGV.each do |arg|
|
|
74
81
|
when "os.vendor", "service.vendor", "service.component.vendor", "hw.vendor"
|
75
82
|
if ! vendors[val]
|
76
83
|
puts "VENDOR MISSING: #{val}"
|
84
|
+
missing_count += 1
|
77
85
|
vendors[val] = true
|
78
86
|
end
|
79
87
|
when "os.arch"
|
80
88
|
if ! os_arch[val]
|
81
89
|
puts "OS ARCH MISSING: #{val}"
|
90
|
+
missing_count += 1
|
82
91
|
os_arch[val] = true
|
83
92
|
end
|
84
93
|
when "os.product"
|
85
94
|
if ! os_prod[val]
|
86
95
|
puts "OS PRODUCT MISSING: #{val}"
|
96
|
+
missing_count += 1
|
87
97
|
os_prod[val] = true
|
88
98
|
end
|
89
99
|
when "os.family"
|
90
100
|
if ! os_family[val]
|
91
101
|
puts "OS FAMILY MISSING: #{val}"
|
102
|
+
missing_count += 1
|
92
103
|
os_family[val] = true
|
93
104
|
end
|
94
105
|
when "os.device"
|
95
106
|
if ! os_device[val]
|
96
107
|
puts "OS DEVICE MISSING: #{val}"
|
108
|
+
missing_count += 1
|
97
109
|
os_device[val] = true
|
98
110
|
end
|
99
111
|
when "hw.product"
|
100
112
|
if ! hw_prod[val]
|
101
113
|
puts "HW PRODUCT MISSING: #{val}"
|
114
|
+
missing_count += 1
|
102
115
|
hw_prod[val] = true
|
103
116
|
end
|
104
117
|
when "hw.family"
|
105
118
|
if ! hw_family[val]
|
106
119
|
puts "HW FAMILY MISSING: #{val}"
|
120
|
+
missing_count += 1
|
107
121
|
hw_family[val] = true
|
108
122
|
end
|
109
123
|
when "hw.device"
|
110
124
|
if ! hw_device[val]
|
111
125
|
puts "HW DEVICE MISSING: #{val}"
|
126
|
+
missing_count += 1
|
112
127
|
hw_device[val] = true
|
113
128
|
end
|
114
129
|
when "service.product", "service.component.product"
|
115
130
|
if ! svc_prod[val]
|
116
131
|
puts "SERVICE PRODUCT MISSING: #{val}"
|
132
|
+
missing_count += 1
|
117
133
|
svc_prod[val] = true
|
118
134
|
end
|
119
135
|
when "service.family"
|
120
136
|
if ! svc_family[val]
|
121
137
|
puts "SERVICE FAMILY MISSING: #{val}"
|
138
|
+
missing_count += 1
|
122
139
|
svc_family[val] = true
|
123
140
|
end
|
124
141
|
end
|
@@ -127,16 +144,20 @@ ARGV.each do |arg|
|
|
127
144
|
end
|
128
145
|
end
|
129
146
|
|
130
|
-
|
147
|
+
if options.write
|
148
|
+
# Write back the unique identifiers
|
149
|
+
write_identifiers(vendors, File.join(bdir, "vendor.txt"))
|
150
|
+
write_identifiers(fields, File.join(bdir, "fields.txt"))
|
151
|
+
write_identifiers(os_arch, File.join(bdir, "os_architecture.txt"))
|
152
|
+
write_identifiers(os_prod, File.join(bdir, "os_product.txt"))
|
153
|
+
write_identifiers(os_family, File.join(bdir, "os_family.txt"))
|
154
|
+
write_identifiers(os_device, File.join(bdir, "os_device.txt"))
|
155
|
+
write_identifiers(hw_prod, File.join(bdir, "hw_product.txt"))
|
156
|
+
write_identifiers(hw_family, File.join(bdir, "hw_family.txt"))
|
157
|
+
write_identifiers(hw_device, File.join(bdir, "hw_device.txt"))
|
158
|
+
write_identifiers(svc_prod, File.join(bdir, "service_product.txt"))
|
159
|
+
write_identifiers(svc_family, File.join(bdir, "service_family.txt"))
|
160
|
+
end
|
131
161
|
|
132
|
-
|
133
|
-
|
134
|
-
write_identifiers(os_arch, File.join(bdir, "os_architecture.txt"))
|
135
|
-
write_identifiers(os_prod, File.join(bdir, "os_product.txt"))
|
136
|
-
write_identifiers(os_family, File.join(bdir, "os_family.txt"))
|
137
|
-
write_identifiers(os_device, File.join(bdir, "os_device.txt"))
|
138
|
-
write_identifiers(hw_prod, File.join(bdir, "hw_product.txt"))
|
139
|
-
write_identifiers(hw_family, File.join(bdir, "hw_family.txt"))
|
140
|
-
write_identifiers(hw_device, File.join(bdir, "hw_device.txt"))
|
141
|
-
write_identifiers(svc_prod, File.join(bdir, "service_product.txt"))
|
142
|
-
write_identifiers(svc_family, File.join(bdir, "service_family.txt"))
|
162
|
+
exit_code = (missing_count > 0 ? 1 : 0)
|
163
|
+
exit(exit_code)
|
data/bin/recog_verify
CHANGED
@@ -53,8 +53,7 @@ failures = 0
|
|
53
53
|
ARGV.each do |arg|
|
54
54
|
Dir.glob(arg).each do |file|
|
55
55
|
ndb = Recog::DB.new(file)
|
56
|
-
|
57
|
-
verifier = Recog::VerifierFactory.build(options)
|
56
|
+
verifier = Recog::VerifierFactory.build(options, ndb)
|
58
57
|
verified = verifier.verify
|
59
58
|
failures += verifier.reporter.failure_count
|
60
59
|
warnings += verifier.reporter.warning_count
|