recog 2.1.23 → 2.1.24

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2c3b5a929f55c25d63a5ac8d27bb69b8f79c66ce
-  data.tar.gz: d184b08644cfc547bacdb21e332cb344f02a8fba
+  metadata.gz: 1311937d249f7d775e823684af469e6fdb63d219
+  data.tar.gz: a521aa3b679c5107feabd73b17db70ff22618c75
 SHA512:
-  metadata.gz: 052ed55f73da7ab43cf25dd43e729ae3a3c14ebbe1b1b254aa9fc49304967076a92040b7fcbb9edd030cf22f8f0fa3ecbc3c0b6bc705d74ba7dd0d0c9f7feba1
-  data.tar.gz: 78c0558d39dd7a888335691eebe2faae2a0ead1678671bd84a9326d24ba9ff21090c139aba6d8e263c5e80c911a55c278374e1cb089e5e0716ef3e65974fa0ec
+  metadata.gz: d6fb453205539af744e318a6dd74dc42a9ff7730bbc0ee04b2d1ccae4e5c13f9b1c3935730a75e7d27ec726d6a7eb4e1e645668fc8e1e1c95f402e8184cf7635
+  data.tar.gz: 80bbb58d47f7758f9aaf80ed0191a9e100476fc036f586c1ab9cd5ed85e538d17458a73ef2cf4a4d07e3e3dca81d5df1dd831180da2e00d7dcd4b1500ffd26d4

data/CONTRIBUTING.md

@@ -67,7 +67,7 @@ git rebase upstream/master
 git checkout -b FOO
 ```
 
-Now, make your changes, commit as necessary with useful commit messages. 
+Now, make your changes, commit as necessary with useful commit messages.
 
 Please note that changes to [lib/recog/version.rb](https://github.com/rapid7/recog/blob/master/lib/recog/version.rb) in PRs are almost never necessary.
 
@@ -83,6 +83,37 @@ Finally, submit the PR.  Navigate to ```https://github.com/<your-github-username
 
 When your PR is submitted, it will be automatically subjected to the full run of tests in [Travis](https://travis-ci.org/rapid7/recog/), however you are encourage to perform testing _before_ submitting the PR.  To do this, simply run `rake tests`.
 
+## Updating CPEs
+
+There exists some automation to update the CPEs that might be asserted with
+some recog fingerprints.  This should be run periodically to ensure that all
+fingerprints that could have CPEs do, etc.
+
+First, setup a python3 venv:
+
+  ```
+  python3 -m venv venv
+  source venv/bin/activate
+  pip install -r requirements.txt
+  ```
+
+Download the latest CPE 2.3 dictionary:
+
+  ```
+  wget https://nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml.gz
+  ````
+
+Run the CPE automation against every XML file, using GNU `parallel` to speed things up:
+
+  ```
+  ls xml/*.xml | parallel --gnu "./update_cpes.py {} official-cpe-dictionary_v2.3.xml cpe-remap.yaml && xmllint --format --noblanks {} > {}.bak &&  mv {}.bak {} || echo {}" 2> errors.txt
+  ```
+
+Any mismatched fingerprints will be listed in `errors.txt` for eventual
+maintenance.  The `cpe-remap.yaml` file can be used to map between
+vendor/product/etc differences between Recog and CPE, or to work around bugs in
+either.
+
 ## Landing PRs
 
 (Note: this portion is a work-in-progress.  Please update it as things change)

data/lib/recog/fingerprint.rb

@@ -95,6 +95,36 @@ class Fingerprint
 
     result['fingerprint_db'] = @match_key if @match_key
 
+    result.each_pair do |k,v|
+      # skip any nil result values, which is allowed but woud jam up the match below
+      next if v.nil?
+      # if this key's value uses interpolation of the form "foo{some.thing}",
+      # if some.thing was "bar" then this keys value would be set to "foobar".
+      if /\{(?<replace>[^\s{}]+)\}/ =~ v
+        if result[replace]
+          if /\{(?<bad_replace>[^\s{}]+)\}/ =~ result[replace]
+            raise "Invalid recursive use of #{bad_replace} in #{replace}"
+          end
+          result[k] = v.gsub(/\{#{replace}\}/, result[replace])
+        else
+          # if the value uses an interpolated value that does not exist, in general this could be
+          # very bad, but over time we have allowed the use of regexes with
+          # optional captures that are then used for parts of the asserted
+          # fingerprints.  This is frequently done for optional version
+          # strings.  If the key in question is cpe23 and the interpolated
+          # value we are trying to replace is version related, use the CPE
+          # standard of '-' for the version, otherwise raise and exception as
+          # this code currently does not handle interpolation of undefined
+          # values in other cases.
+          if k =~ /\.cpe23$/ and replace =~ /\.version$/
+            result[k] = v.gsub(/\{#{replace}\}/, '-')
+          else
+            raise "Invalid use of nil interpolated value #{replace} in non-cpe23 fingerprint param #{k}"
+          end
+        end
+      end
+    end
+
     return result
   end
 

data/lib/recog/version.rb

@@ -1,3 +1,3 @@
 module Recog
-  VERSION = '2.1.23'
+  VERSION = '2.1.24'
 end

data/requirements.txt

@@ -0,0 +1,2 @@
+lxml==4.2.4
+pyyaml

data/spec/lib/recog/fingerprint_spec.rb

@@ -2,16 +2,22 @@ require 'nokogiri'
 require 'recog/fingerprint'
 
 describe Recog::Fingerprint do
-  let(:xml) do
-    path = File.expand_path(File.join('spec', 'data', 'whitespaced_fingerprint.xml'))
-    doc = Nokogiri::XML(IO.read(path))
-    doc.xpath("//fingerprint").first
-  end
-  subject { Recog::Fingerprint.new(xml) }
+  context "whitespace" do
+    let(:xml) do
+      path = File.expand_path(File.join('spec', 'data', 'whitespaced_fingerprint.xml'))
+      doc = Nokogiri::XML(IO.read(path))
+      doc.xpath("//fingerprint").first
+    end
+    subject { Recog::Fingerprint.new(xml) }
 
-  describe "#name" do
-    it "properly squashes whitespace" do
-      expect(subject.name).to eq('I love whitespace!')
+    describe "#name" do
+      it "properly squashes whitespace" do
+        expect(subject.name).to eq('I love whitespace!')
+      end
     end
   end
+
+  skip  "value interpolation" do
+    # TODO
+  end
 end

data/xml/http_servers.xml

@@ -3,11 +3,8 @@
   <!-- HTTP Server headers are matched against these patterns to fingerprint HTTP servers. -->
   <fingerprint pattern="^Stronghold/(\d\.\d) Apache/([012][\d.]*)\s*(.*)$">
     <description>Red Hat Stronghold Enterprise Apache</description>
-    <example>Stronghold/3.0 Apache/1.3.19 RedHat/3014c</example>
-    <example>Stronghold/3.0 Apache/1.3.22 RedHat/3017c (Unix) PHP/4.1.2 mod_ssl/2.8.7 OpenSSL/0.9.6</example>
-    <example>Stronghold/3.0 Apache/1.3.22 RedHat/3017c (Unix) PHP/4.3.3 mod_ssl/2.8.7 OpenSSL/0.9.6 mod_perl/1.25</example>
-    <example>Stronghold/4.0 Apache/1.3.22</example>
-    <example>Stronghold/4.0 Apache/1.3.22 (Unix) mod_ssl/2.8.7 OpenSSL/0.9.6c mod_perl/1.26</example>
+    <example service.version="1.3.19" service.cpe23="cpe:/a:apache:http_server:1.3.19" service.component.cpe23="cpe:/a:redhat:stronghold:3.0">Stronghold/3.0 Apache/1.3.19 RedHat/3014c</example>
+    <example service.version="1.3.22" service.cpe23="cpe:/a:apache:http_server:1.3.22" service.component.cpe23="cpe:/a:redhat:stronghold:4.0" apache.info="(Unix) mod_ssl/2.8.7 OpenSSL/0.9.6c mod_perl/1.26">Stronghold/4.0 Apache/1.3.22 (Unix) mod_ssl/2.8.7 OpenSSL/0.9.6c mod_perl/1.26</example>
     <param pos="0" name="service.vendor" value="Apache"/>
     <param pos="0" name="service.product" value="HTTPD"/>
     <param pos="0" name="service.family" value="Apache"/>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: recog
 version: !ruby/object:Gem::Version
-  version: 2.1.23
+  version: 2.1.24
 platform: ruby
 authors:
 - Rapid7 Research
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-09-20 00:00:00.000000000 Z
+date: 2018-10-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -164,6 +164,7 @@ files:
 - misc/convert_mysql_err
 - misc/order.xsl
 - recog.gemspec
+- requirements.txt
 - spec/data/best_os_match_1.yml
 - spec/data/best_os_match_2.yml
 - spec/data/best_service_match_1.yml