recog 2.1.23 → 2.1.24

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 2c3b5a929f55c25d63a5ac8d27bb69b8f79c66ce
4
- data.tar.gz: d184b08644cfc547bacdb21e332cb344f02a8fba
3
+ metadata.gz: 1311937d249f7d775e823684af469e6fdb63d219
4
+ data.tar.gz: a521aa3b679c5107feabd73b17db70ff22618c75
5
5
  SHA512:
6
- metadata.gz: 052ed55f73da7ab43cf25dd43e729ae3a3c14ebbe1b1b254aa9fc49304967076a92040b7fcbb9edd030cf22f8f0fa3ecbc3c0b6bc705d74ba7dd0d0c9f7feba1
7
- data.tar.gz: 78c0558d39dd7a888335691eebe2faae2a0ead1678671bd84a9326d24ba9ff21090c139aba6d8e263c5e80c911a55c278374e1cb089e5e0716ef3e65974fa0ec
6
+ metadata.gz: d6fb453205539af744e318a6dd74dc42a9ff7730bbc0ee04b2d1ccae4e5c13f9b1c3935730a75e7d27ec726d6a7eb4e1e645668fc8e1e1c95f402e8184cf7635
7
+ data.tar.gz: 80bbb58d47f7758f9aaf80ed0191a9e100476fc036f586c1ab9cd5ed85e538d17458a73ef2cf4a4d07e3e3dca81d5df1dd831180da2e00d7dcd4b1500ffd26d4
data/CONTRIBUTING.md CHANGED
@@ -67,7 +67,7 @@ git rebase upstream/master
67
67
  git checkout -b FOO
68
68
  ```
69
69
 
70
- Now, make your changes, commit as necessary with useful commit messages.
70
+ Now, make your changes, commit as necessary with useful commit messages.
71
71
 
72
72
  Please note that changes to [lib/recog/version.rb](https://github.com/rapid7/recog/blob/master/lib/recog/version.rb) in PRs are almost never necessary.
73
73
 
@@ -83,6 +83,37 @@ Finally, submit the PR. Navigate to ```https://github.com/<your-github-username
83
83
 
84
84
  When your PR is submitted, it will be automatically subjected to the full run of tests in [Travis](https://travis-ci.org/rapid7/recog/), however you are encourage to perform testing _before_ submitting the PR. To do this, simply run `rake tests`.
85
85
 
86
+ ## Updating CPEs
87
+
88
+ There exists some automation to update the CPEs that might be asserted with
89
+ some recog fingerprints. This should be run periodically to ensure that all
90
+ fingerprints that could have CPEs do, etc.
91
+
92
+ First, setup a python3 venv:
93
+
94
+ ```
95
+ python3 -m venv venv
96
+ source venv/bin/activate
97
+ pip install -r requirements.txt
98
+ ```
99
+
100
+ Download the latest CPE 2.3 dictionary:
101
+
102
+ ```
103
+ wget https://nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml.gz
104
+ ````
105
+
106
+ Run the CPE automation against every XML file, using GNU `parallel` to speed things up:
107
+
108
+ ```
109
+ ls xml/*.xml | parallel --gnu "./update_cpes.py {} official-cpe-dictionary_v2.3.xml cpe-remap.yaml && xmllint --format --noblanks {} > {}.bak && mv {}.bak {} || echo {}" 2> errors.txt
110
+ ```
111
+
112
+ Any mismatched fingerprints will be listed in `errors.txt` for eventual
113
+ maintenance. The `cpe-remap.yaml` file can be used to map between
114
+ vendor/product/etc differences between Recog and CPE, or to work around bugs in
115
+ either.
116
+
86
117
  ## Landing PRs
87
118
 
88
119
  (Note: this portion is a work-in-progress. Please update it as things change)
@@ -95,6 +95,36 @@ class Fingerprint
95
95
 
96
96
  result['fingerprint_db'] = @match_key if @match_key
97
97
 
98
+ result.each_pair do |k,v|
99
+ # skip any nil result values, which is allowed but woud jam up the match below
100
+ next if v.nil?
101
+ # if this key's value uses interpolation of the form "foo{some.thing}",
102
+ # if some.thing was "bar" then this keys value would be set to "foobar".
103
+ if /\{(?<replace>[^\s{}]+)\}/ =~ v
104
+ if result[replace]
105
+ if /\{(?<bad_replace>[^\s{}]+)\}/ =~ result[replace]
106
+ raise "Invalid recursive use of #{bad_replace} in #{replace}"
107
+ end
108
+ result[k] = v.gsub(/\{#{replace}\}/, result[replace])
109
+ else
110
+ # if the value uses an interpolated value that does not exist, in general this could be
111
+ # very bad, but over time we have allowed the use of regexes with
112
+ # optional captures that are then used for parts of the asserted
113
+ # fingerprints. This is frequently done for optional version
114
+ # strings. If the key in question is cpe23 and the interpolated
115
+ # value we are trying to replace is version related, use the CPE
116
+ # standard of '-' for the version, otherwise raise and exception as
117
+ # this code currently does not handle interpolation of undefined
118
+ # values in other cases.
119
+ if k =~ /\.cpe23$/ and replace =~ /\.version$/
120
+ result[k] = v.gsub(/\{#{replace}\}/, '-')
121
+ else
122
+ raise "Invalid use of nil interpolated value #{replace} in non-cpe23 fingerprint param #{k}"
123
+ end
124
+ end
125
+ end
126
+ end
127
+
98
128
  return result
99
129
  end
100
130
 
data/lib/recog/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Recog
2
- VERSION = '2.1.23'
2
+ VERSION = '2.1.24'
3
3
  end
data/requirements.txt ADDED
@@ -0,0 +1,2 @@
1
+ lxml==4.2.4
2
+ pyyaml
@@ -2,16 +2,22 @@ require 'nokogiri'
2
2
  require 'recog/fingerprint'
3
3
 
4
4
  describe Recog::Fingerprint do
5
- let(:xml) do
6
- path = File.expand_path(File.join('spec', 'data', 'whitespaced_fingerprint.xml'))
7
- doc = Nokogiri::XML(IO.read(path))
8
- doc.xpath("//fingerprint").first
9
- end
10
- subject { Recog::Fingerprint.new(xml) }
5
+ context "whitespace" do
6
+ let(:xml) do
7
+ path = File.expand_path(File.join('spec', 'data', 'whitespaced_fingerprint.xml'))
8
+ doc = Nokogiri::XML(IO.read(path))
9
+ doc.xpath("//fingerprint").first
10
+ end
11
+ subject { Recog::Fingerprint.new(xml) }
11
12
 
12
- describe "#name" do
13
- it "properly squashes whitespace" do
14
- expect(subject.name).to eq('I love whitespace!')
13
+ describe "#name" do
14
+ it "properly squashes whitespace" do
15
+ expect(subject.name).to eq('I love whitespace!')
16
+ end
15
17
  end
16
18
  end
19
+
20
+ skip "value interpolation" do
21
+ # TODO
22
+ end
17
23
  end
data/xml/http_servers.xml CHANGED
@@ -3,11 +3,8 @@
3
3
  <!-- HTTP Server headers are matched against these patterns to fingerprint HTTP servers. -->
4
4
  <fingerprint pattern="^Stronghold/(\d\.\d) Apache/([012][\d.]*)\s*(.*)$">
5
5
  <description>Red Hat Stronghold Enterprise Apache</description>
6
- <example>Stronghold/3.0 Apache/1.3.19 RedHat/3014c</example>
7
- <example>Stronghold/3.0 Apache/1.3.22 RedHat/3017c (Unix) PHP/4.1.2 mod_ssl/2.8.7 OpenSSL/0.9.6</example>
8
- <example>Stronghold/3.0 Apache/1.3.22 RedHat/3017c (Unix) PHP/4.3.3 mod_ssl/2.8.7 OpenSSL/0.9.6 mod_perl/1.25</example>
9
- <example>Stronghold/4.0 Apache/1.3.22</example>
10
- <example>Stronghold/4.0 Apache/1.3.22 (Unix) mod_ssl/2.8.7 OpenSSL/0.9.6c mod_perl/1.26</example>
6
+ <example service.version="1.3.19" service.cpe23="cpe:/a:apache:http_server:1.3.19" service.component.cpe23="cpe:/a:redhat:stronghold:3.0">Stronghold/3.0 Apache/1.3.19 RedHat/3014c</example>
7
+ <example service.version="1.3.22" service.cpe23="cpe:/a:apache:http_server:1.3.22" service.component.cpe23="cpe:/a:redhat:stronghold:4.0" apache.info="(Unix) mod_ssl/2.8.7 OpenSSL/0.9.6c mod_perl/1.26">Stronghold/4.0 Apache/1.3.22 (Unix) mod_ssl/2.8.7 OpenSSL/0.9.6c mod_perl/1.26</example>
11
8
  <param pos="0" name="service.vendor" value="Apache"/>
12
9
  <param pos="0" name="service.product" value="HTTPD"/>
13
10
  <param pos="0" name="service.family" value="Apache"/>
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: recog
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.1.23
4
+ version: 2.1.24
5
5
  platform: ruby
6
6
  authors:
7
7
  - Rapid7 Research
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-09-20 00:00:00.000000000 Z
11
+ date: 2018-10-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rspec
@@ -164,6 +164,7 @@ files:
164
164
  - misc/convert_mysql_err
165
165
  - misc/order.xsl
166
166
  - recog.gemspec
167
+ - requirements.txt
167
168
  - spec/data/best_os_match_1.yml
168
169
  - spec/data/best_os_match_2.yml
169
170
  - spec/data/best_service_match_1.yml