recog 0.02 → 1.0.0

data/lib/recog/fingerprint/regexp_factory.rb

@@ -0,0 +1,39 @@
+
+module Recog
+  class Fingerprint
+
+    #
+    # @example
+    #   r = RegexpFactory.build("^Apache[ -]Coyote/(\d\.\d)$", "REG_ICASE")
+    #   r.match("Apache-Coyote/1.1")
+    #
+    module RegexpFactory
+
+      # Map strings as used in Recog XML to Fixnum values used by Regexp
+      FLAG_MAP = {
+        'REG_DOT_NEWLINE'   => Regexp::MULTILINE,
+        'REG_LINE_ANY_CRLF' => Regexp::MULTILINE,
+        'REG_ICASE'         => Regexp::IGNORECASE,
+      }
+
+      # @return [Regexp]
+      def self.build(pattern, flags)
+        options = build_options(flags)
+        Regexp.new(pattern, options)
+      end
+
+      # Convert string flag names as used in Recog XML into a Fixnum suitable for
+      # passing as the `options` parameter to `Regexp.new`
+      #
+      # @see FLAG_MAP
+      # @param flags [Array<String>]
+      # @return [Fixnum] Flags for creating a regular expression object
+      def self.build_options(flags)
+        flags.reduce(Regexp::NOENCODING) do |sum, flag|
+          sum |= (FLAG_MAP[flag] || 0)
+        end
+      end
+    end
+  end
+end
+

data/lib/recog/fingerprint/test.rb

@@ -0,0 +1,13 @@
+
+class Recog::Fingerprint::Test
+  attr_accessor :content
+  attr_accessor :attributes
+  def initialize(content, attributes=[])
+    @content = content
+    @attributes = attributes
+  end
+
+  def to_s
+    content
+  end
+end

data/lib/recog/matcher.rb

@@ -9,7 +9,7 @@ class Matcher
 
   def match_banners(banners_file)
     reporter.report do
-      
+
       fd = $stdin
       file_source = false
 
@@ -22,7 +22,7 @@ class Matcher
         reporter.increment_line_count
 
         line = line.to_s.unpack("C*").pack("C*").strip.gsub(/\\[rn]/, '')
-        found = nil 
+        found = nil
         fingerprints.each do |fp|
           m = line.match(fp.regex)
           if m
@@ -53,7 +53,7 @@ class Matcher
       end
 
       fd.close if file_source
-      
+
     end
   end
 end

data/lib/recog/nizer.rb

@@ -22,27 +22,20 @@ class Nizer
   @@db_manager = nil
 
   #
-  # Locate a database that corresponds with the match_key and attempt to
-  # find a matching fingerprinting, stopping a the first hit. Returns
-  # nil when no matching database or fingerprint is found.
+  # Locate a database that corresponds with the `match_key` and attempt to
+  # find a matching {Fingerprint fingerprint}, stopping at the first hit. Returns `nil`
+  # when no matching database or fingerprint is found.
   #
+  # @param match_key [String] Fingerprint DB name, e.g. 'smb.native_os'
+  # @return (see Fingerprint#match)
   def self.match(match_key, match_string)
     match_string = match_string.to_s.unpack("C*").pack("C*")
     @@db_manager ||= Recog::DBManager.new
     @@db_manager.databases.each do |db|
       next unless db.match_key == match_key
       db.fingerprints.each do |fprint|
-        m = fprint.regex.match(match_string)
-        next unless m
-        result = { 'matched' => fprint.name }
-        fprint.params.each_pair do |k,v|
-          if v[0] == 0
-            result[k] = v[1]
-          else
-            result[k] = m[ v[0] ]
-          end
-        end
-        return result  
+        m = fprint.match(match_string)
+        return m if m
       end
     end
     nil
@@ -69,7 +62,7 @@ class Nizer
       (HOST_ATTRIBUTES & m.keys).each do |ha|
         host_attrs[ha]        ||= {}
         host_attrs[ha][m[ha]] ||= 0
-        host_attrs[ha][m[ha]]  += 1        
+        host_attrs[ha][m[ha]]  += 1
       end
 
       next unless m.has_key?('os.product')
@@ -85,7 +78,7 @@ class Nizer
     # Select the best host attribute value by highest frequency
     #
     host_attrs.keys.each do |hk|
-      ranked_attr = host_attrs[hk].keys.sort do |a,b| 
+      ranked_attr = host_attrs[hk].keys.sort do |a,b|
         host_attrs[hk][b] <=> host_attrs[hk][a]
       end
       result[hk] = ranked_attr.first
@@ -101,7 +94,7 @@ class Nizer
     # matches within an os.product group. Multiple weak matches can
     # outweigh a single strong match by design.
     #
-    ranked_os = os_products.keys.sort do |a,b|    
+    ranked_os = os_products.keys.sort do |a,b|
       os_products[b].map{ |r| r['os.certainty'] }.inject(:+) <=>
       os_products[a].map{ |r| r['os.certainty'] }.inject(:+)
     end
@@ -151,11 +144,11 @@ class Nizer
     end
 
     #
-    # Select the best service name by combined certainty of all matches 
-    # within an service.product group. Multiple weak matches can 
+    # Select the best service name by combined certainty of all matches
+    # within an service.product group. Multiple weak matches can
     # outweigh a single strong match by design.
     #
-    ranked_service = service_products.keys.sort do |a,b|    
+    ranked_service = service_products.keys.sort do |a,b|
       service_products[b].map{ |r| r['service.certainty'] }.inject(:+) <=>
       service_products[a].map{ |r| r['service.certainty'] }.inject(:+)
     end
@@ -177,7 +170,7 @@ class Nizer
 
     result
   end
- 
+
 end
 end
 
@@ -259,5 +252,5 @@ Current key names:
   timeout
   tomcat.info
   zmailer.ident
-  
-=end
+
+=end

data/lib/recog/verifier.rb

@@ -12,33 +12,18 @@ class Verifier
       fingerprints.each do |fp|
         reporter.print_name fp
 
-        if fp.tests.length == 0
-          warning = "'#{fp.name}' has no test cases"
-          reporter.warning "WARN: #{warning}"
-        end
-
-        fp.tests.each do |test|
-          m = test.match(fp.regex)
-          unless m
-            failure = "'#{fp.name}' failed to match #{test.inspect} with #{fp.regex.to_s}'"
-            reporter.failure("FAIL: #{failure}")
-          else
-            info = { }
-            fp.params.each_pair do |k,v|
-              if v[0] == 0
-                info[k] = v[1]
-              else
-                info[k] = m[ v[0] ]
-                if m[ v[0] ].to_s.empty?
-                  warning = "'#{fp.name}' failed to match #{test.inspect} key '#{k}'' with #{fp.regex.to_s}'"
-                  reporter.warning "WARN: #{warning}"
-                end					
-              end
-            end
-
-            reporter.success(test)
+        fp.verify_tests do |status, message|
+          case status
+          when :warn
+            reporter.warning "WARN: #{message}"
+          when :fail
+            reporter.failure "FAIL: #{message}"
+          when :success
+            reporter.success(message)
           end
+
         end
+
       end
     end
   end

data/lib/recog/verifier_factory.rb

@@ -10,4 +10,4 @@ module VerifierFactory
     Verifier.new(options.fingerprints, reporter)
   end
 end
-end
+end

data/lib/recog/verify_reporter.rb

@@ -82,4 +82,4 @@ class VerifyReporter
     end
   end
 end
-end
+end

data/lib/recog/version.rb

@@ -1,3 +1,3 @@
 module Recog
-  VERSION = "0.02"
+  VERSION = "1.0.0"
 end

data/recog.gemspec

@@ -14,9 +14,9 @@ Gem::Specification.new do |s|
   s.homepage    = "https://www.github.com/rapid7/recog"
   s.summary     = %q{Network service fingerprint database, classes, and utilities}
   s.description = %q{
-    Recog is a framework for identifying products, services, operating systems, and hardware by matching 
-    fingerprints against data returned from various network probes. Recog makes it simply to extract useful 
-    information from web server banners, snmp system description fields, and a whole lot more. 
+    Recog is a framework for identifying products, services, operating systems, and hardware by matching
+    fingerprints against data returned from various network probes. Recog makes it simply to extract useful
+    information from web server banners, snmp system description fields, and a whole lot more.
   }.gsub(/\s+/, ' ').strip
 
   s.files         = `git ls-files`.split("\n")
@@ -27,8 +27,17 @@ Gem::Specification.new do |s|
   # ---- Dependencies ----
 
   s.add_development_dependency 'rspec'
+  s.add_development_dependency 'yard'
+  if RUBY_PLATFORM =~ /java/
+    # markdown formatting for yard
+    s.add_development_dependency 'kramdown'
+  else
+    # markdown formatting for yard
+    s.add_development_dependency 'redcarpet'
+  end
   s.add_development_dependency 'cucumber'
   s.add_development_dependency 'aruba'
+  s.add_development_dependency 'simplecov'
 
   s.add_runtime_dependency 'nokogiri'
 end

data/spec/data/test_fingerprints.xml

@@ -21,4 +21,16 @@
      <example>HP LaserJet 2200</example>
      <param pos="0" name="service.vendor" value="HP"/>
    </fingerprint>
+
+   <fingerprint pattern="^Windows XP (\d+) (Service Pack \d+)$">
+      <description>Windows XP</description>
+      <example os.build="2600" os.version="Service Pack 1"
+        >Windows XP 2600 Service Pack 1</example>
+      <param pos="0" name="os.certainty" value="1.0"/>
+      <param pos="0" name="os.vendor" value="Microsoft"/>
+      <param pos="0" name="os.product" value="Windows XP"/>
+      <param pos="1" name="os.build"/>
+      <param pos="2" name="os.version"/>
+   </fingerprint>
+
 </fingerprints>

data/spec/lib/fingerprint_self_test_spec.rb

@@ -1,4 +1,4 @@
-require_relative '../../lib/recog/db'
+require 'recog/db'
 
 describe Recog::DB do
   Dir[File.expand_path File.join('xml', '*.xml')].each do |xml_file_name|
@@ -16,6 +16,10 @@ describe Recog::DB do
 
         context "#{fp.regex}" do
 
+          if fp.name.nil? || fp.name.empty?
+            skip "has a name"
+          end
+
           # Not yet enforced
           # it "has a name" do
           #   expect(fp.name).not_to be_nil
@@ -28,13 +32,13 @@ describe Recog::DB do
           end
 
           # Not yet enforced
-          # it "has a test cases" do
+          # it "has test cases" do
           #  expect(fp.tests.length).not_to equal(0)
           # end
 
           fp.tests.each do |example|
-            it "passes self-test #{example.gsub(/\s+/, ' ')[0,32]}..." do
-              expect(fp.regex.match(example)).to_not eq(nil)
+            it "passes self-test #{example.content.gsub(/\s+/, ' ')[0,32]}..." do
+              expect(fp.match(example.content)).to_not be_nil
             end
           end
 

data/spec/lib/{db_spec.rb → recog/db_spec.rb}

@@ -1,12 +1,16 @@
-require_relative '../../lib/recog/db'
+require 'recog/db'
 
 describe Recog::DB do
   let(:xml_file) { File.expand_path File.join('spec', 'data', 'test_fingerprints.xml') }
   subject { Recog::DB.new(xml_file) }
 
   describe "#fingerprints" do
+    subject(:fingerprints) { described_class.new(xml_file).fingerprints }
+
+    it { is_expected.to be_a(Enumerable) }
+
     context "with only a pattern" do
-      let(:entry) { subject.fingerprints[0] }
+      subject(:entry) { described_class.new(xml_file).fingerprints[0] }
 
       it "has a blank name with no description" do
         expect(entry.name).to be_empty
@@ -26,7 +30,7 @@ describe Recog::DB do
     end
 
     context "with params" do
-      let(:entry) { subject.fingerprints[1] }
+      subject(:entry) { described_class.new(xml_file).fingerprints[1] }
 
       it "has a name" do
         expect(entry.name).to eq('PalmOS')
@@ -46,14 +50,22 @@ describe Recog::DB do
     end
 
     context "with pattern flags" do
-      let(:entry) { subject.fingerprints[2] }
+      subject(:entry) { described_class.new(xml_file).fingerprints[2] }
 
       it "has a name and only uses the first value" do
         expect(entry.name).to eq('HP Designjet printer')
       end
 
-      it "has a pattern" do
+      it 'creates a Regexp with expected flags' do
+        if RUBY_PLATFORM =~ /java/i
+          pending "Bug in jruby"
+        end
+        expect(entry.regex).to be_a(Regexp)
         expect(entry.regex.options).to eq(Regexp::NOENCODING | Regexp::IGNORECASE)
+      end
+
+      it "has a pattern" do
+        expect(entry.regex).to be_a(Regexp)
         expect(entry.regex.source).to eq("(designjet \\S+)")
       end
 
@@ -67,7 +79,7 @@ describe Recog::DB do
     end
 
     context "with test" do
-      let(:entry) { subject.fingerprints[3] }
+      subject(:entry) { described_class.new(xml_file).fingerprints[3] }
 
       it "has a name" do
         expect(entry.name).to eq('HP JetDirect Printer')
@@ -82,7 +94,7 @@ describe Recog::DB do
       end
 
       it "has no tests" do
-        expect(entry.tests).to match_array(["HP LaserJet 4100 Series", "HP LaserJet 2200"])
+        expect(entry.tests.map(&:content)).to match_array(["HP LaserJet 4100 Series", "HP LaserJet 2200"])
       end
     end
   end

data/spec/lib/recog/fingerprint/regexp_factory.rb

@@ -0,0 +1,61 @@
+
+require 'recog/fingerprint/regexp_factory'
+
+describe Recog::Fingerprint::RegexpFactory do
+
+  describe 'FLAG_MAP' do
+    subject { described_class::FLAG_MAP }
+
+    it "should have three flags" do
+      expect(subject.size).to be 3
+    end
+  end
+
+  describe '.build' do
+    subject { described_class.build(pattern, options) }
+
+    let(:pattern) { 'Apache/(\d+)' }
+    let(:options) { [ 'REG_ICASE' ] }
+
+    it { is_expected.to be_a(Regexp) }
+    it { is_expected.to match('Apache/2') }
+
+  end
+
+  describe '.build_options' do
+    subject { described_class.build_options(flags) }
+
+    let(:flags) { [ ] }
+    it { is_expected.to be_a(Fixnum) }
+
+    specify "sets NOENCODING" do
+      expect(subject & Regexp::NOENCODING).to_not be_zero
+    end
+
+    context 'with REG_ICASE' do
+      let(:flags) { [ 'REG_ICASE' ] }
+      specify "sets NOENCODING & IGNORECASE" do
+        expect(subject & Regexp::NOENCODING).to_not be_zero
+        expect(subject & Regexp::IGNORECASE).to_not be_zero
+      end
+    end
+
+    context 'with REG_DOT_NEWLINE' do
+      let(:flags) { [ 'REG_DOT_NEWLINE' ] }
+      specify "sets NOENCODING & MULTILINE" do
+        expect(subject & Regexp::NOENCODING).to_not be_zero
+        expect(subject & Regexp::MULTILINE).to_not be_zero
+      end
+    end
+
+    context 'with REG_LINE_ANY_CRLF' do
+      let(:flags) { [ 'REG_LINE_ANY_CRLF' ] }
+      specify "sets NOENCODING & MULTILINE" do
+        expect(subject & Regexp::NOENCODING).to_not be_zero
+        expect(subject & Regexp::MULTILINE).to_not be_zero
+      end
+    end
+
+  end
+end
+