recog-intrigue 2.3.7

data/lib/recog/db_manager.rb

@@ -0,0 +1,31 @@
+module Recog
+class DBManager
+  require 'nokogiri'
+  require_relative 'db'
+
+  attr_accessor :path, :databases
+
+  DefaultDatabasePath = File.expand_path( File.join( File.dirname(__FILE__), "..", "..", "xml") )
+
+  def initialize(path = DefaultDatabasePath)
+    self.path = path
+    reload
+  end
+
+  def load_databases
+    if File.directory?(self.path)
+      Dir[self.path + "/*.xml"].each do |dbxml|
+        self.databases << DB.new(dbxml)
+      end
+    else
+      self.databases << DB.new(self.path)
+    end
+  end
+
+  def reload
+    self.databases = []
+    load_databases
+  end
+
+end
+end

data/lib/recog/fingerprint.rb

@@ -0,0 +1,280 @@
+module Recog
+
+# A fingerprint that can be {#match matched} against a particular kind of
+# fingerprintable data, e.g. an HTTP `Server` header
+class Fingerprint
+  require_relative 'fingerprint/regexp_factory'
+  require_relative 'fingerprint/test'
+
+  # A human readable name describing this fingerprint
+  # @return (see #parse_description)
+  attr_reader :name
+
+  # Regular expression pulled from the {DB} xml file.
+  #
+  # @see #create_regexp
+  # @return [Regexp] the Regexp to try when calling {#match}
+  attr_reader :regex
+
+  # Collection of indexes for capture groups created by {#match}
+  #
+  # @return (see #parse_params)
+  attr_reader :params
+
+  # Collection of example strings that should {#match} our {#regex}
+  #
+  # @return (see #parse_examples)
+  attr_reader :tests
+
+  # @param xml [Nokogiri::XML::Element]
+  # @param match_key [String] See Recog::DB
+  # @param protocol [String] Protocol such as ftp, mssql, http, etc.
+  def initialize(xml, match_key=nil, protocol=nil)
+    @match_key = match_key
+    @protocol = protocol
+    @name   = parse_description(xml)
+    @regex  = create_regexp(xml)
+    @params = {}
+    @tests = []
+
+    @protocol.downcase! if @protocol
+    parse_examples(xml)
+    parse_params(xml)
+  end
+
+  def output_diag_data(message, data, exception)
+    STDERR.puts message
+    STDERR.puts exception.inspect
+    STDERR.puts "Length:   #{data.length}"
+    STDERR.puts "Encoding: #{data.encoding}"
+    STDERR.puts "Problematic data:\n#{data}"
+    STDERR.puts "Raw bytes:\n#{data.pretty_inspect}\n"
+  end
+
+  # Attempt to match the given string.
+  #
+  # @param match_string [String]
+  # @return [Hash,nil] Keys will be host, service, and os attributes
+  def match(match_string)
+    # match_string.force_encoding('BINARY') if match_string
+    begin
+      match_data = @regex.match(match_string)
+    rescue Encoding::CompatibilityError => e
+      begin
+        # Replace invalid UTF-8 characters with spaces, just as DAP does.
+        encoded_str = match_string.encode("UTF-8", :invalid => :replace, :undef => :replace, :replace => '')
+        match_data = @regex.match(encoded_str)
+      rescue Exception => e
+        output_diag_data('Exception while re-encoding match_string to UTF-8', match_string, e)
+      end
+    rescue Exception => e
+      output_diag_data('Exception while running regex against match_string', match_string, e)
+    end
+    return if match_data.nil?
+
+    result = { 'matched' => @name }
+    replacements = {}
+    @params.each_pair do |k,v|
+      pos = v[0]
+      if pos == 0
+        # A match offset of 0 means this param has a hardcoded value
+        result[k] = v[1]
+        # if this value uses interpolation, note it for handling later
+        v[1].scan(/\{([^\s{}]+)\}/).flatten.each do |replacement|
+          replacements[k] ||= Set[]
+          replacements[k] << replacement
+        end
+      else
+        # A match offset other than 0 means the value should come from
+        # the corresponding match result index
+        result[k] = match_data[ pos ]
+      end
+    end
+
+    # Use the protocol specified in the XML database if there isn't one
+    # provided as part of this fingerprint.
+    if @protocol
+      unless result['service.protocol']
+        result['service.protocol'] = @protocol
+      end
+    end
+
+    result['fingerprint_db'] = @match_key if @match_key
+
+    # for everything identified as using interpolation, do so
+    replacements.each_pair do |replacement_k, replacement_vs|
+      replacement_vs.each do |replacement|
+        if result[replacement]
+          result[replacement_k] = result[replacement_k].gsub(/\{#{replacement}\}/, result[replacement])
+        else
+          # if the value uses an interpolated value that does not exist, in general this could be
+          # very bad, but over time we have allowed the use of regexes with
+          # optional captures that are then used for parts of the asserted
+          # fingerprints.  This is frequently done for optional version
+          # strings.  If the key in question is cpe23 and the interpolated
+          # value we are trying to replace is version related, use the CPE
+          # standard of '-' for the version, otherwise raise and exception as
+          # this code currently does not handle interpolation of undefined
+          # values in other cases.
+          if replacement_k =~ /\.cpe23$/ and replacement =~ /\.version$/
+            result[replacement_k] = result[replacement_k].gsub(/\{#{replacement}\}/, '-')
+          else
+            raise "Invalid use of nil interpolated non-version value #{replacement} in non-cpe23 fingerprint param #{replacement_k}"
+          end
+        end
+      end
+    end
+
+    return result
+  end
+
+  # Ensure all the {#params} are valid
+  #
+  # @yieldparam status [Symbol] One of `:warn`, `:fail`, or `:success` to
+  #   indicate whether a param is valid
+  # @yieldparam message [String] A human-readable string explaining the
+  #   `status`
+  def verify_params(&block)
+    return if params.empty?
+    params.each do |param_name, pos_value|
+      pos, value = pos_value
+      if pos > 0 && !value.to_s.empty?
+        yield :fail, "'#{@name}'s #{param_name} is a non-zero pos but specifies a value of '#{value}'"
+      elsif pos == 0 && value.to_s.empty?
+        yield :fail, "'#{@name}'s #{param_name} is not a capture (pos=0) but doesn't specify a value"
+      end
+    end
+  end
+
+  # Ensure all the {#tests} actually match the fingerprint and return the
+  # expected capture groups.
+  #
+  # @yieldparam status [Symbol] One of `:warn`, `:fail`, or `:success` to
+  #   indicate whether a test worked
+  # @yieldparam message [String] A human-readable string explaining the
+  #   `status`
+  def verify_tests(&block)
+
+    # look for the presence of test cases
+    if tests.size == 0
+      yield :warn, "'#{@name}' has no test cases"
+    end
+
+    # make sure each test case passes
+    tests.each do |test|
+      result = match(test.content)
+      if result.nil?
+        yield :fail, "'#{@name}' failed to match #{test.content.inspect} with #{@regex}'"
+        next
+      end
+
+      message = test
+      status = :success
+      # Ensure that all the attributes as provided by the example were parsed
+      # out correctly and match the capture group values we expect.
+      test.attributes.each do |k, v|
+        next if k == '_encoding'
+        if !result.has_key?(k) || result[k] != v
+          message = "'#{@name}' failed to find expected capture group #{k} '#{v}'. Result was #{result[k]}"
+          status = :fail
+          break
+        end
+      end
+      yield status, message
+    end
+
+    # make sure there are capture groups for all params that use them
+    verify_tests_have_capture_groups(&block)
+  end
+
+  # For fingerprints that specify parameters that are defined by
+  # capture groups, ensure that each parameter has at least one test
+  # that defines an attribute to test for the correct capture of that
+  # parameter.
+  #
+  # @yieldparam status [Symbol] One of `:warn`, `:fail`, or `:success` to
+  #   indicate whether a test worked
+  # @yieldparam message [String] A human-readable string explaining the
+  #   `status`
+  def verify_tests_have_capture_groups(&block)
+    capture_group_used = {}
+    if !params.empty?
+      # get a list of parameters that are defined by capture groups
+      params.each do |param_name, pos_value|
+        pos, value = pos_value
+        if pos > 0 && value.to_s.empty?
+          capture_group_used[param_name] = false
+        end
+      end
+    end
+
+    # match up the fingerprint parameters with test attributes
+    tests.each do |test|
+      test.attributes.each do |k,v|
+        if capture_group_used.has_key?(k)
+          capture_group_used[k] = true
+        end
+      end
+    end
+
+    # alert on untested parameters
+    capture_group_used.each do |param_name, param_used|
+      if !param_used
+        message = "'#{@name}' is missing an example that checks for parameter '#{param_name}' " +
+                  "messsage which is derived from a capture group"
+        yield :warn, message
+      end
+    end
+  end
+
+  private
+
+  # @param xml [Nokogiri::XML::Element]
+  # @return [Regexp]
+  def create_regexp(xml)
+    pattern = xml['pattern']
+    flags   = xml['flags'].to_s.split(',')
+    RegexpFactory.build(pattern, flags)
+  end
+
+  # @param xml [Nokogiri::XML::Element]
+  # @return [String] Contents of the source XML's `description` tag
+  def parse_description(xml)
+    element = xml.xpath('description')
+    element.empty? ? '' : element.first.content.to_s.gsub(/\s+/, ' ').strip
+  end
+
+  # @param xml [Nokogiri::XML::Element]
+  # @return [void]
+  def parse_examples(xml)
+    elements = xml.xpath('example')
+
+    elements.each do |elem|
+      # convert nokogiri Attributes into a hash of name => value
+      attrs = elem.attributes.values.reduce({}) { |a,e| a.merge(e.name => e.value) }
+      @tests << Test.new(elem.content, attrs)
+    end
+
+    nil
+  end
+
+  # @param xml [Nokogiri::XML::Element]
+  # @return [Hash<String,Array>] Keys are things like `"os.name"`, values are a two
+  #   element Array. The first element is an index for the capture group that returns
+  #   that thing. If the index is 0, the second element is a static value for
+  #   that thing; otherwise it is undefined.
+  def parse_params(xml)
+    @params = {}.tap do |h|
+      xml.xpath('param').each do |param|
+        name  = param['name']
+        pos   = param['pos'].to_i
+        value = param['value'].to_s
+        h[name] = [pos, value]
+      end
+    end
+
+    nil
+  end
+
+end
+end

data/lib/recog/fingerprint/regexp_factory.rb

@@ -0,0 +1,56 @@
+
+module Recog
+  class Fingerprint
+
+    #
+    # @example
+    #   r = RegexpFactory.build("^Apache[ -]Coyote/(\d\.\d)$", "REG_ICASE")
+    #   r.match("Apache-Coyote/1.1")
+    #
+    module RegexpFactory
+
+      # Currently, only options relating to case insensitivity and
+      # multiline/newline are supported.  Because Recog's data is used by tools
+      # written in different languages like Ruby and Java, we currently support
+      # specifying them in a variety of ways.  This map controls how they can
+      # be specified.
+      #
+      # TODO: consider supporting only a simpler variant and require that tools
+      # that use Recog data translate accordingly
+      FLAG_MAP = {
+        # multiline variations
+        'REG_DOT_NEWLINE'   => Regexp::MULTILINE,
+        'REG_LINE_ANY_CRLF' => Regexp::MULTILINE,
+        'REG_MULTILINE'     => Regexp::MULTILINE,
+        # case variations
+        'REG_ICASE'         => Regexp::IGNORECASE,
+        'IGNORECASE'        => Regexp::IGNORECASE
+      }
+
+      DEFAULT_FLAGS = 0
+
+      # @return [Regexp]
+      def self.build(pattern, flags)
+        options = build_options(flags)
+        Regexp.new(pattern, options)
+      end
+
+      # Convert string flag names as used in Recog XML into a Fixnum suitable for
+      # passing as the `options` parameter to `Regexp.new`
+      #
+      # @see FLAG_MAP
+      # @param flags [Array<String>]
+      # @return [Fixnum] Flags for creating a regular expression object
+      def self.build_options(flags)
+        unsupported_flags = flags.select { |flag| !FLAG_MAP.key?(flag) }
+        unless unsupported_flags.empty?
+          fail "Unsupported regular expression flags found: #{unsupported_flags.join(',')}. Must be one of: #{FLAG_MAP.keys.join(',')}"
+        end
+        flags.reduce(DEFAULT_FLAGS) do |sum, flag|
+          sum |= (FLAG_MAP[flag] || 0)
+        end
+      end
+    end
+  end
+end
+

data/lib/recog/fingerprint/test.rb

@@ -0,0 +1,18 @@
+
+class Recog::Fingerprint::Test
+  attr_accessor :content
+  attr_accessor :attributes
+  def initialize(content, attributes=[])
+    @attributes = attributes
+
+    if @attributes['_encoding'] && @attributes['_encoding'] == 'base64'
+      @content = content.to_s.unpack('m*').first
+    else
+      @content = content
+    end
+  end
+
+  def to_s
+    content
+  end
+end

data/lib/recog/formatter.rb

@@ -0,0 +1,51 @@
+module Recog
+class Formatter
+  COLORS = {
+    :red    => 31,
+    :yellow => 33,
+    :green  => 32,
+    :white  => 15
+  }
+
+  attr_reader :options, :output
+
+  def initialize(options, output)
+    @options = options
+    @output  = output || StringIO.new
+  end
+
+  def status_message(text)
+    output.puts color(text, :white)
+  end
+
+  def success_message(text)
+    output.puts color(text, :green)
+  end
+
+  def warning_message(text)
+    output.puts color(text, :yellow)
+  end
+
+  def failure_message(text)
+    output.puts color(text, :red)
+  end
+
+  private
+
+  def color_enabled?
+    options.color
+  end
+
+  def color(text, color_code)
+    color_enabled? ? colorize(text, color_code) : text
+  end
+
+  def colorize(text, color_code)
+    "\e[#{color_code_for(color_code)}m#{text}\e[0m"
+  end
+
+  def color_code_for(code)
+    COLORS.fetch(code) { COLORS.fetch(:white) }
+  end
+end
+end

data/lib/recog/match_reporter.rb

@@ -0,0 +1,77 @@
+module Recog
+class MatchReporter
+  attr_reader :formatter
+  attr_reader :line_count, :match_count, :fail_count
+
+  def initialize(options, formatter)
+    @options = options
+    @formatter = formatter
+    reset_counts
+  end
+
+  def report
+    reset_counts
+    yield self
+    summarize unless @options.quiet
+  end
+
+  def stop?
+    return false unless @options.fail_fast
+    @fail_count >= @options.stop_after
+  end
+
+  def increment_line_count
+    @line_count += 1
+  end
+
+  def match(text)
+    @match_count += 1
+    formatter.success_message(text)
+  end
+
+  def failure(text)
+    @fail_count += 1
+    formatter.failure_message(text)
+  end
+
+  def print_summary
+    colorize_summary(summary_line)
+  end
+
+  private
+
+  def reset_counts
+    @line_count = @match_count = @fail_count = 0
+  end
+
+  def detail?
+    @options.detail
+  end
+
+  def summarize
+    if detail?
+      print_lines_processed
+      print_summary
+    end
+  end
+
+  def print_lines_processed
+    formatter.status_message("\nProcessed #{line_count} lines")
+  end
+
+  def summary_line
+    summary = "SUMMARY: "
+    summary << "#{match_count} matches"
+    summary << " and #{fail_count} failures"
+    summary
+  end
+
+  def colorize_summary(summary)
+    if @fail_count > 0
+      formatter.failure_message(summary)
+    else
+      formatter.success_message(summary)
+    end
+  end
+end
+end