recog 2.1.23 → 2.1.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CONTRIBUTING.md +32 -1
- data/lib/recog/fingerprint.rb +30 -0
- data/lib/recog/version.rb +1 -1
- data/requirements.txt +2 -0
- data/spec/lib/recog/fingerprint_spec.rb +15 -9
- data/xml/http_servers.xml +2 -5
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1311937d249f7d775e823684af469e6fdb63d219
|
4
|
+
data.tar.gz: a521aa3b679c5107feabd73b17db70ff22618c75
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d6fb453205539af744e318a6dd74dc42a9ff7730bbc0ee04b2d1ccae4e5c13f9b1c3935730a75e7d27ec726d6a7eb4e1e645668fc8e1e1c95f402e8184cf7635
|
7
|
+
data.tar.gz: 80bbb58d47f7758f9aaf80ed0191a9e100476fc036f586c1ab9cd5ed85e538d17458a73ef2cf4a4d07e3e3dca81d5df1dd831180da2e00d7dcd4b1500ffd26d4
|
data/CONTRIBUTING.md
CHANGED
@@ -67,7 +67,7 @@ git rebase upstream/master
|
|
67
67
|
git checkout -b FOO
|
68
68
|
```
|
69
69
|
|
70
|
-
Now, make your changes, commit as necessary with useful commit messages.
|
70
|
+
Now, make your changes, commit as necessary with useful commit messages.
|
71
71
|
|
72
72
|
Please note that changes to [lib/recog/version.rb](https://github.com/rapid7/recog/blob/master/lib/recog/version.rb) in PRs are almost never necessary.
|
73
73
|
|
@@ -83,6 +83,37 @@ Finally, submit the PR. Navigate to ```https://github.com/<your-github-username
|
|
83
83
|
|
84
84
|
When your PR is submitted, it will be automatically subjected to the full run of tests in [Travis](https://travis-ci.org/rapid7/recog/), however you are encourage to perform testing _before_ submitting the PR. To do this, simply run `rake tests`.
|
85
85
|
|
86
|
+
## Updating CPEs
|
87
|
+
|
88
|
+
There exists some automation to update the CPEs that might be asserted with
|
89
|
+
some recog fingerprints. This should be run periodically to ensure that all
|
90
|
+
fingerprints that could have CPEs do, etc.
|
91
|
+
|
92
|
+
First, setup a python3 venv:
|
93
|
+
|
94
|
+
```
|
95
|
+
python3 -m venv venv
|
96
|
+
source venv/bin/activate
|
97
|
+
pip install -r requirements.txt
|
98
|
+
```
|
99
|
+
|
100
|
+
Download the latest CPE 2.3 dictionary:
|
101
|
+
|
102
|
+
```
|
103
|
+
wget https://nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml.gz
|
104
|
+
````
|
105
|
+
|
106
|
+
Run the CPE automation against every XML file, using GNU `parallel` to speed things up:
|
107
|
+
|
108
|
+
```
|
109
|
+
ls xml/*.xml | parallel --gnu "./update_cpes.py {} official-cpe-dictionary_v2.3.xml cpe-remap.yaml && xmllint --format --noblanks {} > {}.bak && mv {}.bak {} || echo {}" 2> errors.txt
|
110
|
+
```
|
111
|
+
|
112
|
+
Any mismatched fingerprints will be listed in `errors.txt` for eventual
|
113
|
+
maintenance. The `cpe-remap.yaml` file can be used to map between
|
114
|
+
vendor/product/etc differences between Recog and CPE, or to work around bugs in
|
115
|
+
either.
|
116
|
+
|
86
117
|
## Landing PRs
|
87
118
|
|
88
119
|
(Note: this portion is a work-in-progress. Please update it as things change)
|
data/lib/recog/fingerprint.rb
CHANGED
@@ -95,6 +95,36 @@ class Fingerprint
|
|
95
95
|
|
96
96
|
result['fingerprint_db'] = @match_key if @match_key
|
97
97
|
|
98
|
+
result.each_pair do |k,v|
|
99
|
+
# skip any nil result values, which is allowed but woud jam up the match below
|
100
|
+
next if v.nil?
|
101
|
+
# if this key's value uses interpolation of the form "foo{some.thing}",
|
102
|
+
# if some.thing was "bar" then this keys value would be set to "foobar".
|
103
|
+
if /\{(?<replace>[^\s{}]+)\}/ =~ v
|
104
|
+
if result[replace]
|
105
|
+
if /\{(?<bad_replace>[^\s{}]+)\}/ =~ result[replace]
|
106
|
+
raise "Invalid recursive use of #{bad_replace} in #{replace}"
|
107
|
+
end
|
108
|
+
result[k] = v.gsub(/\{#{replace}\}/, result[replace])
|
109
|
+
else
|
110
|
+
# if the value uses an interpolated value that does not exist, in general this could be
|
111
|
+
# very bad, but over time we have allowed the use of regexes with
|
112
|
+
# optional captures that are then used for parts of the asserted
|
113
|
+
# fingerprints. This is frequently done for optional version
|
114
|
+
# strings. If the key in question is cpe23 and the interpolated
|
115
|
+
# value we are trying to replace is version related, use the CPE
|
116
|
+
# standard of '-' for the version, otherwise raise and exception as
|
117
|
+
# this code currently does not handle interpolation of undefined
|
118
|
+
# values in other cases.
|
119
|
+
if k =~ /\.cpe23$/ and replace =~ /\.version$/
|
120
|
+
result[k] = v.gsub(/\{#{replace}\}/, '-')
|
121
|
+
else
|
122
|
+
raise "Invalid use of nil interpolated value #{replace} in non-cpe23 fingerprint param #{k}"
|
123
|
+
end
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
98
128
|
return result
|
99
129
|
end
|
100
130
|
|
data/lib/recog/version.rb
CHANGED
data/requirements.txt
ADDED
@@ -2,16 +2,22 @@ require 'nokogiri'
|
|
2
2
|
require 'recog/fingerprint'
|
3
3
|
|
4
4
|
describe Recog::Fingerprint do
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
5
|
+
context "whitespace" do
|
6
|
+
let(:xml) do
|
7
|
+
path = File.expand_path(File.join('spec', 'data', 'whitespaced_fingerprint.xml'))
|
8
|
+
doc = Nokogiri::XML(IO.read(path))
|
9
|
+
doc.xpath("//fingerprint").first
|
10
|
+
end
|
11
|
+
subject { Recog::Fingerprint.new(xml) }
|
11
12
|
|
12
|
-
|
13
|
-
|
14
|
-
|
13
|
+
describe "#name" do
|
14
|
+
it "properly squashes whitespace" do
|
15
|
+
expect(subject.name).to eq('I love whitespace!')
|
16
|
+
end
|
15
17
|
end
|
16
18
|
end
|
19
|
+
|
20
|
+
skip "value interpolation" do
|
21
|
+
# TODO
|
22
|
+
end
|
17
23
|
end
|
data/xml/http_servers.xml
CHANGED
@@ -3,11 +3,8 @@
|
|
3
3
|
<!-- HTTP Server headers are matched against these patterns to fingerprint HTTP servers. -->
|
4
4
|
<fingerprint pattern="^Stronghold/(\d\.\d) Apache/([012][\d.]*)\s*(.*)$">
|
5
5
|
<description>Red Hat Stronghold Enterprise Apache</description>
|
6
|
-
<example>Stronghold/3.0 Apache/1.3.19 RedHat/3014c</example>
|
7
|
-
<example
|
8
|
-
<example>Stronghold/3.0 Apache/1.3.22 RedHat/3017c (Unix) PHP/4.3.3 mod_ssl/2.8.7 OpenSSL/0.9.6 mod_perl/1.25</example>
|
9
|
-
<example>Stronghold/4.0 Apache/1.3.22</example>
|
10
|
-
<example>Stronghold/4.0 Apache/1.3.22 (Unix) mod_ssl/2.8.7 OpenSSL/0.9.6c mod_perl/1.26</example>
|
6
|
+
<example service.version="1.3.19" service.cpe23="cpe:/a:apache:http_server:1.3.19" service.component.cpe23="cpe:/a:redhat:stronghold:3.0">Stronghold/3.0 Apache/1.3.19 RedHat/3014c</example>
|
7
|
+
<example service.version="1.3.22" service.cpe23="cpe:/a:apache:http_server:1.3.22" service.component.cpe23="cpe:/a:redhat:stronghold:4.0" apache.info="(Unix) mod_ssl/2.8.7 OpenSSL/0.9.6c mod_perl/1.26">Stronghold/4.0 Apache/1.3.22 (Unix) mod_ssl/2.8.7 OpenSSL/0.9.6c mod_perl/1.26</example>
|
11
8
|
<param pos="0" name="service.vendor" value="Apache"/>
|
12
9
|
<param pos="0" name="service.product" value="HTTPD"/>
|
13
10
|
<param pos="0" name="service.family" value="Apache"/>
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: recog
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.1.
|
4
|
+
version: 2.1.24
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Rapid7 Research
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-10-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rspec
|
@@ -164,6 +164,7 @@ files:
|
|
164
164
|
- misc/convert_mysql_err
|
165
165
|
- misc/order.xsl
|
166
166
|
- recog.gemspec
|
167
|
+
- requirements.txt
|
167
168
|
- spec/data/best_os_match_1.yml
|
168
169
|
- spec/data/best_os_match_2.yml
|
169
170
|
- spec/data/best_service_match_1.yml
|