recog 1.0.16 → 1.0.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5967636c56e64027e73963a64574380b85c6af88
-  data.tar.gz: a06812e81e6c16b3fc8237d50040ad61d27a7adf
+  metadata.gz: ba7d8b2d9765280fc41fea934102dbff6bff69b6
+  data.tar.gz: d86c1221b484f17a0d826de292648b6bf5737381
 SHA512:
-  metadata.gz: f9a3360742a7156517f2069c5e7dc12223504c0cfd935db0fde50fbe2ec5f5ce2a6c4c968f26b3b34276ee707835f53461c2e04fb0468b77d49d360ce8dc8fd6
-  data.tar.gz: 5e0677ff0170f1c4fbac6a0a9a9d2513f5332a652a6bec2f21fbd92a3ca8ec803c585d3415d64cd59124065655516fd04e832728e8d4869e691a17e0faa567a1
+  metadata.gz: 56446fa6f827bdcf28976878dc4697333da0eb48be67884c875a8b848b6e0d44e429e675becb3a2e9556338bf9ab9af674d38dea77603f8f399a1be05adb4446
+  data.tar.gz: 7c1e4e4751dfbf8b8082aa9928340be690f98f3cdbf1141bd5570c3a35b57a2142dc64b604a35726f9443e520c4d53e27080fa4b07891c52d7cac59aa80356b7

data/.travis.yml

@@ -4,7 +4,9 @@ rvm:
   - 2.1.5
   - 1.9.3
   - jruby
-matrix:
-  allow_failures:
-    - rvm: jruby
-script: bundle exec rspec spec features
+before_install:
+  - "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc"
+  - rake --version
+before_script:
+  - bundle exec rake --version
+script: bundle exec rake tests

data/CONTRIBUTING.md

@@ -100,23 +100,23 @@ In short:
 
     ```
     git checkout -b upstream-master --track upstream/master
-    git merge -S --no-ff --edit landing-PR
+    git merge -S --no-ff --edit landing-PR # merge the PR into upstream-master
+    # re-test if/as necessary
     git push upstream upstream-master:master --dry-run # confirm you are pushing what you expect
-    git push upstream upstream-master:master
+    git push upstream upstream-master:master # push upstream-master to upstream:master
     ```
+6. If applicable, release a new version (see next section)
 
 ## Releasing New Versions
 
-When Recog's critical parts are modified, for example its fingerprints or
-underlying supporting code, a new version should eventually be released.  These
-new releases can then be optionally included in projects such as Metasploit or
-products such as Rapid7's in a controlled manner.
+When Recog's critical parts are modified, for example its fingerprints or underlying supporting code, a new version _must_ eventually be released.  These new releases can then be optionally included in projects such as Metasploit or products such as Rapid7's Nexpose in a controlled manner.  Releases for non-functional updates such as updates to documentation are not necessary.
 
-For now, in general any time Recog is modified you should release a version of
-the Gem and the Github release, described below.  Eventually this process may
-change.
+When a new version of Recog is released, you _must_ do so in two different places:
 
-### Release New Gem
+   * Gem -- for systems that use Gemspec or similar tools to control what version of Recog to include
+   * Github -- for systems that use Git tags to control what version of Recog to include
+
+### Gem Release
 
 1. Get an account on [Rubygems](https://rubygems.org)
 2. Contact one of the Recog project contributors (listed [here under OWNERS](https://rubygems.org/gems/recog) and have them add you to the Recog gem.  They'll need to run:

data/Gemfile

@@ -5,6 +5,7 @@ gemspec
 gem 'nokogiri'
 
 group :test do
+  gem 'rake'
   gem 'rspec', '>= 2.99'
   gem 'cucumber', '~> 1.3.8'
   gem 'aruba', '~> 0.5.3'

data/README.md

@@ -30,7 +30,7 @@ A fingerprint file consists of an XML document like the following:
     04:
     05: <fingerprint pattern="^RomSShell_([\d\.]+)$">
     06:  <description>Allegro RomSShell SSH</description>
-    07:  <example>RomSShell_4.62</example>
+    07:  <example service.version="4.62">RomSShell_4.62</example>
     08:  <param pos="0" name="service.vendor" value="Allegro"/>
     09:  <param pos="0" name="service.product" value="RomSShell"/>
     10:  <param pos="1" name="service.version"/>
@@ -38,17 +38,21 @@ A fingerprint file consists of an XML document like the following:
     12:
     13: </fingerprints>
 
-The first line should always consist of the XML version declaration. The first element should always be a <fingerpints/> block with a `matches` attribute indicating what this fingerprint file is supposed to match. The `matches` attribute is normally in the form of protocol.field.
+The first line should always consist of the XML version declaration. The first element should always be a `fingerpints` block with a `matches` attribute indicating what data this fingerprint file is supposed to match. The `matches` attribute is normally in the form of `protocol.field`.
 
-Inside of the <fingerprints/> element there should be one or more <fingerprint/> elements. Every fingerprint should contain a `pattern` attribute, which contains the regular expression to be used against the match key.
+Inside of the `fingerprints` element there should be one or more `fingerprint` elements. Every `fingerprint` must contain a `pattern` attribute, which contains the regular expression to be used to match against the data.  An optional `flags` attribute can be specified to control how the regular expression is to be interpreted.  See [the Recog documentation for `FLAG_MAP`](http://www.rubydoc.info/gems/recog/Recog/Fingerprint/RegexpFactory#FLAG_MAP-constant) for more information.
 
-Inside of the fingerprint, a <description/> element should contain a human-readable string describing this fingerprint.
+Inside of the fingerprint, a `description` element should contain a human-readable string describing this fingerprint.
 
-The <example/> element should contain a successful match for the fingerprint's `pattern`. Multiple <example/> elements are preferred, as these elements are used for the built-in regression testing suite.
+At least one `example` element should be present, however multiple `example` elements are preferred.  These elements are used as part of the test coverage present in rspec which validates that the provided data matches the specified regular expression.  Additionally, if the fingerprint is using the `param` elements to extract field values from the data (described next), you can add these expected extractions as attributes for the `example` elements.  In the example above, this:
 
-the <param/> elements contain a `pos` attribute, which indicates what capture field from the `pattern` should be extracted, or `0` for a static string. The `name` attribute is the key that will be reported in the case of a successful match and the `value` will either be a static string for `pos` values of `0` or missing and taken from the captured field.
+    07:  <example service.version="4.62">RomSShell_4.62</example>
 
-Once a fingerprint has been added, the <examples/> entries can be tested by executing `bin/recog_verify` against the fingerprint file:
+tests that `RomSShell_4.62` matches the provided regular expression and that the value of `service.version` is 4.62.
+
+The `param` elements contain a `pos` attribute, which indicates what capture field from the `pattern` should be extracted, or `0` for a static string. The `name` attribute is the key that will be reported in the case of a successful match and the `value` will either be a static string for `pos` values of `0` or missing and taken from the captured field.
+
+Once a fingerprint has been added, the `example` entries can be tested by executing `bin/recog_verify` against the fingerprint file:
 
     $ bin/recog_verify xml/ssh_banners.xml
 

data/Rakefile

@@ -18,5 +18,5 @@ Cucumber::Rake::Task.new(:features) do |t|
     t.cucumber_opts = "features --format pretty"
 end
 
-task :default => [ :spec, :features, :yard ]
-
+task :default => [ :tests, :yard ]
+task :tests => [ :spec, :features ]

data/features/support/env.rb

@@ -2,4 +2,5 @@ require 'aruba/cucumber'
 
 Before do
   @dirs = ["features/xml"]
+  @aruba_timeout_seconds = 30
 end

data/lib/recog/fingerprint/regexp_factory.rb

@@ -9,11 +9,22 @@ module Recog
     #
     module RegexpFactory
 
-      # Map strings as used in Recog XML to Fixnum values used by Regexp
+      # Currently, only options relating to case insensitivity and
+      # multiline/newline are supported.  Because Recog's data is used by tools
+      # written in different languages like Ruby and Java, we currently support
+      # specifying them in a variety of ways.  This map controls how they can
+      # be specified.
+      #
+      # TODO: consider supporting only a simpler variant and require that tools
+      # that use Recog data translate accordingly
       FLAG_MAP = {
+        # multiline variations
         'REG_DOT_NEWLINE'   => Regexp::MULTILINE,
         'REG_LINE_ANY_CRLF' => Regexp::MULTILINE,
+        'MULTILINE'         => Regexp::MULTILINE,
+        # case variations
         'REG_ICASE'         => Regexp::IGNORECASE,
+        'IGNORECASE'        => Regexp::IGNORECASE
       }
 
       # @return [Regexp]
@@ -29,6 +40,10 @@ module Recog
       # @param flags [Array<String>]
       # @return [Fixnum] Flags for creating a regular expression object
       def self.build_options(flags)
+        unsupported_flags = flags.select { |flag| !FLAG_MAP.key?(flag) }
+        unless unsupported_flags.empty?
+          fail "Unsupported regular expression flags found: #{unsupported_flags.join(',')}. Must be one of: #{FLAG_MAP.keys.join(',')}"
+        end
         flags.reduce(Regexp::NOENCODING) do |sum, flag|
           sum |= (FLAG_MAP[flag] || 0)
         end

data/lib/recog/version.rb

@@ -1,3 +1,3 @@
 module Recog
-  VERSION = '1.0.16'
+  VERSION = '1.0.17'
 end

data/spec/lib/fingerprint_self_test_spec.rb

@@ -30,7 +30,7 @@ describe Recog::DB do
 
           it 'uses capturing regular expressions properly' do
             # the list of index-based captures that the fingerprint is expecting
-            expected_capture_positions = fp.params.values.map(&:first).map(&:to_i).select { |i| i > 0 }
+            expected_capture_positions = fp.params.values.map(&:first).map(&:to_i).select { |position| position > 0 }
             if fp.params.empty? && expected_capture_positions.size > 0
               fail "Non-asserting fingerprint with regex #{fp.regex} captures #{expected_capture_positions.size} time(s); 0 are needed"
             else

data/spec/lib/recog/fingerprint/regexp_factory.rb

@@ -6,8 +6,8 @@ describe Recog::Fingerprint::RegexpFactory do
   describe 'FLAG_MAP' do
     subject { described_class::FLAG_MAP }
 
-    it "should have three flags" do
-      expect(subject.size).to be 3
+    it "should have the right number of flags" do
+      expect(subject.size).to be 5
     end
   end
 
@@ -56,6 +56,11 @@ describe Recog::Fingerprint::RegexpFactory do
       end
     end
 
+    context 'with invalid flags' do
+      let(:flags) { %w(SYN ACK FIN) } # oh, wrong flags!
+      specify 'raises and lists supported/unsupported flags' do
+        expect { subject }.to raise_error(/SYN,ACK,FIN. Must be one of: .+/)
+      end
+    end
   end
 end
-

data/xml/http_cookies.xml

@@ -375,5 +375,42 @@ servers.
       <param pos="0" name="service.vendor" value="Oracle"/>
       <param pos="0" name="service.family" value="OracleAS"/>
       <param pos="0" name="service.product" value="OracleAS Portal"/>
-   </fingerprint>
+    </fingerprint>
+
+    <fingerprint pattern="^Compaq-HMMD=[^;]+;.*$">
+      <description>HP System Management Homepage (SMH)</description>
+      <example>Compaq-HMMD=0001-c01fffff-487a-394a-aab0-ffffffffffff-ffffffffffffffff; path=/</example>
+      <example>Compaq-HMMD=0001-c01fffff-487a-394a-aab0-ffffffffffff-ffffffffffffffff; path=/; Secure</example>
+      <param pos="0" name="service.vendor" value="HP"/>
+      <param pos="0" name="service.family" value="SMH"/>
+      <param pos="0" name="service.product" value="SMH"/>
+    </fingerprint>
+
+    <!--
+        Ignore various cookies that are very generic cookies for session IDs
+        that are not necessarily indicative of any particular
+        product/device/etc.  If a future fingerprint comes along that utilizes
+        a similar cookie name, you must ensure that it is located prior to
+        these and this is enforced by rspec.
+    -->
+
+    <fingerprint pattern="^JSESSIONID(?:\.[^=]+)?=[^;]+;.*$" flags="REG_ICASE">
+      <description>Ignore simple JSESSIONID and related cookies</description>
+      <example>JSESSIONID=6ooov35i4l3n36qtaf8csvg0;Path=/</example>
+      <example>jsessionid=6nkp66iogcdc92720%2Dc6e4%2D4989%2Db7b2%2D5021624cfdff;Path=/;secure</example>
+      <example>JSESSIONID.c00a9623=v216643eijh19p9duve5srgf;Path=/;HttpOnly</example>
+    </fingerprint>
+
+    <fingerprint pattern="^_?SESSION_?ID\s*=\s*[^;]+;.*$" flags="REG_ICASE">
+      <description>Ignore simple SESSIONID and related cookies</description>
+      <example>sessionId=7dba3249cfcd4b59854055311099a294; path=/;</example>
+      <example>_session_id=7fe933db0fea13e9c872103ba2d142db; path=/; HttpOnly</example>
+      <example>sessionId =0VrS6Ro6uC5QPXKgNdqGvyUgUFtUOVwv6OWAEWcWQ3jLRtAk2TVAgAApN9yTWVz;postId=; path=/;</example>
+      <example>_session_id=18b3e173aa11db0533fd01752e81f583; path=/; HttpOnly</example>
+    </fingerprint>
+
+    <fingerprint pattern="^sid=[^;]+;.*$" flags="REG_ICASE">
+      <description>Ignore simple SID and related cookies</description>
+      <example>sid=sfd10bf73-654458f687aa3c68b3874915f651e0ca;path=/;"</example>
+    </fingerprint>
 </fingerprints>

data/xml/http_servers.xml

@@ -1876,6 +1876,19 @@
       <param pos="1" name="service.version"/>
    </fingerprint>
 
+   <fingerprint pattern="^eHTTP[/ ]v?(\d+\.\d+)" flags="REG_ICASE">
+      <example service.version="1.1">EHTTP/1.1</example>
+      <example service.version="2.0">eHTTP v2.0</example>
+      <description>HTTP Server present on seemingly only HP ProCurve network devices</description>
+      <param pos="0" name="service.vendor" value="HP"/>
+      <param pos="0" name="service.product" value="HTTP"/>
+      <param pos="0" name="service.family" value="ProCurve"/>
+      <param pos="1" name="service.version"/>
+      <param pos="0" name="os.vendor" value="HP"/>
+      <param pos="0" name="os.family" value="ProCurve"/>
+      <param pos="0" name="os.certainty" value="0.75"/>
+   </fingerprint>
+
    <fingerprint pattern = "^com.hp.openview.Coda (\d\.\d.\d)$">
       <description>HP Openview Coda</description>
       <example>com.hp.openview.Coda 0.0.1</example>

data/xml/http_wwwauth.xml

@@ -302,8 +302,99 @@
       <param pos="0" name="os.vendor" value="TP-LINK"/>
       <param pos="0" name="os.device" value="WAP"/>
       <param pos="1" name="os.product"/>
+    </fingerprint>
+
+    <fingerprint pattern="^(?:Basic|Digest) .*realm=&quot;Broadcom Management Service&quot;.*$" flags="REG_ICASE">
+      <description>Supposedly part of Broadcom Advanced Control Suite 3 (BACS3) or something similar</description>
+      <example>Digest qop="auth", realm="Broadcom Management Service", nonce="AAAAAAAAAAAAAP//DwHpMwYy1zc=", algorithm="MD5"</example>
+      <param pos="0" name="service.vendor" value="Broadcom"/>
+      <param pos="0" name="service.product" value="Management Service"/>
+    </fingerprint>
+
+    <fingerprint pattern="^(?:Basic|Digest) .*realm=&quot;SWAT&quot;.*$">
+      <description>Samba Web Administration Tool (SWAT)</description>
+      <example>Basic realm="SWAT"</example>
+      <param pos="0" name="service.vendor" value="Samba"/>
+      <param pos="0" name="service.family" value="Samba"/>
+      <param pos="0" name="service.product" value="SWAT"/>
+    </fingerprint>
+
+    <fingerprint pattern="^.*(?:Basic|Digest) realm=&quot;SPIP Configuration&quot;.*$">
+      <description>SPIP publishing system (www.spip.net)</description>
+      <example>Basic realm="SPIP Configuration", Digest realm="SPIP Configuration", nonce="116761147", algorithm="MD5"</example>
+      <param pos="0" name="service.vendor" value="SPIP"/>
+      <param pos="0" name="service.product" value="SPIP"/>
+    </fingerprint>
+
+    <fingerprint pattern="^.*(?:Basic|Digest) .*realm=&quot;HP ISEE @ ([^&quot;]+)&quot;.*$">
+      <description>HP Instant Support Enterprise Edition with a hostname</description>
+      <example host.name="blah">Basic realm="HP ISEE @ blah"</example>
+      <param pos="0" name="service.vendor" value="HP"/>
+      <param pos="0" name="service.product" value="ISEE"/>
+      <param pos="1" name="host.name"/>
+    </fingerprint>
+
+    <fingerprint pattern="^.*(?:Basic|Digest) .*realm=&quot;BIG-IP&quot;.*$">
+      <description>Generic F5 Big-IP</description>
+      <example>Basic realm="BIG-IP"</example>
+      <param pos="0" name="os.vendor" value="F5"/>
+      <param pos="0" name="os.product" value="BIG-IP"/>
+    </fingerprint>
+
+   <!-- HP ProCurve -->
+
+    <fingerprint pattern="^(?:Basic|Digest) realm=&quot;(?:HP|ProCurve) (J[3]\d{3}A)&quot;$" flags="REG_ICASE">
+      <description>HP ProCurve Hubs</description>
+      <example os.product="J3295A">Basic realm="HP J3295A"</example>
+      <param pos="0" name="os.vendor" value="HP"/>
+      <param pos="0" name="os.family" value="ProCurve"/>
+      <param pos="0" name="os.device" value="Hub"/>
+      <param pos="1" name="os.product"/>
+   </fingerprint>
+
+   <fingerprint pattern="^(?:Basic|Digest) realm=&quot;(?:HP|ProCurve) (J[489]\d{3}A)&quot;$" flags="REG_ICASE">
+      <description>HP ProCurve Switches</description>
+      <example os.product="J4110A">Basic realm="HP J4110A"</example>
+      <example os.product="J8164A">Basic realm="ProCurve J8164A"</example>
+      <example os.product="J8165A">Basic realm="HP J8165A"</example>
+      <example os.product="J9021A">Basic realm="HP J9021A"</example>
+      <param pos="0" name="os.vendor" value="HP"/>
+      <param pos="0" name="os.family" value="ProCurve"/>
+      <param pos="0" name="os.device" value="Switch"/>
+      <param pos="1" name="os.product"/>
    </fingerprint>
 
+    <!-- a variety of headers we currently just ignore -->
+
+    <fingerprint pattern="^NTLM$" flags="REG_ICASE">
+      <example>NTLM</example>
+      <example>Ntlm</example>
+      <description>Ignore NTLM-only</description>
+    </fingerprint>
+
+    <fingerprint pattern="^Negotiate$">
+      <description>Ignore Negotiate-only</description>
+      <example>Negotiate</example>
+    </fingerprint>
+
+    <fingerprint pattern="^(?:Basic|Digest) .*realm=&quot;null&quot;">
+      <description>Ignore null</description>
+      <example>Basic realm="null"</example>
+    </fingerprint>
+
+    <fingerprint pattern="^(?:Basic|Digest) .*realm=&quot;(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)&quot;.*$">
+      <description>Ignore realms with an IPv4 address</description>
+      <example>Basic realm="192.168.0.1"</example>
+      <example>Digest qop="auth", realm="172.16.0.1", nonce="AAAAAAAAAAAAAP//DwHpM0IvM78=", algorithm="MD5"</example>
+    </fingerprint>
+
+    <fingerprint pattern="^(?:Basic|Digest) .*realm=&quot;config&quot;.*$">
+      <description>Ignore generic 'config' realms</description>
+      <example>Digest realm="config", nonce="1155041914", algorithm="MD5", qop="auth"</example>
+    </fingerprint>
+
+
+
    <!--
    Temporarily disable this version-less fingerprint because it overrode the
    one in http_servers.xml (see NEX-1255).

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: recog
 version: !ruby/object:Gem::Version
-  version: 1.0.16
+  version: 1.0.17
 platform: ruby
 authors:
 - Rapid7 Research
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-10 00:00:00.000000000 Z
+date: 2015-02-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec