recog 0.01

data/features/xml/successful_tests.xml

@@ -0,0 +1,33 @@
+<?xml version="1.0"?>
+<fingerprints>
+   <fingerprint pattern="^Cisco-SIPGateway/IOS-([\d\.x]+)$">
+      <description>Cisco SIPGateway</description>
+      <example>Cisco-SIPGateway/IOS-12.x</example>
+      <param pos="0" name="os.vendor" value="Cisco"/>
+      <param pos="0" name="os.product" value="IOS"/>
+      <param pos="1" name="os.version"/>
+   </fingerprint>
+   <fingerprint pattern="^Microsoft Exchange Server 2007 IMAP4 service ready$">
+      <!--  Microsoft Exchange Server 2007 IMAP4 service ready
+        -->
+      <description>Microsoft Exchange Server 2007</description>
+      <param pos="0" name="service.vendor" value="Microsoft"/>
+      <param pos="0" name="service.family" value="Exchange Server"/>
+      <param pos="0" name="service.product" value="Exchange 2007 Server"/>
+      <param pos="0" name="os.vendor" value="Microsoft"/>
+      <param pos="0" name="os.device" value="General"/>
+      <param pos="0" name="os.family" value="Windows"/>
+      <param pos="0" name="os.product" value="Windows"/>
+    </fingerprint>
+    <fingerprint pattern="^The Microsoft Exchange IMAP4 service is ready\.?$">
+      <example>The Microsoft Exchange IMAP4 service is ready.</example>
+      <description>Microsoft Exchange Server</description>
+      <param pos="0" name="service.vendor" value="Microsoft"/>
+      <param pos="0" name="service.family" value="Exchange Server"/>
+      <param pos="0" name="service.product" value="Exchange Server"/>
+      <param pos="0" name="os.vendor" value="Microsoft"/>
+      <param pos="0" name="os.device" value="General"/>
+      <param pos="0" name="os.family" value="Windows"/>
+      <param pos="0" name="os.product" value="Windows"/>
+   </fingerprint>
+</fingerprints>

data/features/xml/tests_with_failures.xml

@@ -0,0 +1,10 @@
+<?xml version="1.0"?>
+<fingerprints>
+   <fingerprint pattern="^foo$">
+     <description>foo test</description>
+     <example>bar</example>
+   </fingerprint>
+   <fingerprint pattern="^This matches$">
+     <example>This almost matches</example>
+   </fingerprint>
+</fingerprints>

data/features/xml/tests_with_warnings.xml

@@ -0,0 +1,10 @@
+<?xml version="1.0"?>
+<fingerprints>
+  <fingerprint pattern="^-{10} Welcome to Pure-FTPd (.*)-{10}$">
+    <example>---------- Welcome to Pure-FTPd ----------</example>
+    <description>Pure-FTPd</description>
+    <param pos="1" name="pureftpd.config"/>
+    <param pos="0" name="service.family" value="Pure-FTPd"/>
+    <param pos="0" name="service.product" value="Pure-FTPd"/>
+  </fingerprint>
+</fingerprints>

data/lib/recog.rb

@@ -0,0 +1,3 @@
+require 'recog/version'
+require 'recog/db_manager'
+require 'recog/nizer'

data/lib/recog/db.rb

@@ -0,0 +1,38 @@
+module Recog
+class DB
+  require 'nokogiri'
+  require 'recog/fingerprint'
+
+  attr_accessor :path, :fingerprints, :match_key
+
+  def initialize(path)
+    self.path = path
+    parse_fingerprints
+  end
+
+  def parse_fingerprints
+    self.fingerprints = []
+    xml = nil
+    
+    File.open(self.path, "rb") do |fd|
+      xml = Nokogiri::XML( fd.read(fd.stat.size))
+    end
+
+    xml.xpath("/fingerprints").each do |fbase|
+      if fbase['matches']
+        self.match_key = fbase['matches'].to_s
+      end
+    end
+
+    unless self.match_key
+      self.match_key = File.basename(self.path).sub(/\.xml$/, '')
+    end
+
+    xml.xpath("/fingerprints/fingerprint").each do |fprint|
+      fingerprints << Fingerprint.new(fprint)
+    end
+
+    xml = nil
+  end
+end
+end

data/lib/recog/db_manager.rb

@@ -0,0 +1,27 @@
+module Recog
+class DBManager
+  require 'nokogiri'
+  require 'recog/db'
+
+  attr_accessor :path, :databases
+
+  DefaultDatabasePath = File.expand_path( File.join( File.dirname(__FILE__), "..", "..", "xml") )
+
+  def initialize(path = DefaultDatabasePath)
+    self.path = path
+    reload
+  end
+
+  def load_databases
+    Dir[self.path + "/*.xml"].each do |dbxml|
+      self.databases << DB.new(dbxml)
+    end
+  end
+
+  def reload
+    self.databases = []
+    load_databases
+  end
+
+end
+end

data/lib/recog/fingerprint.rb

@@ -0,0 +1,60 @@
+module Recog
+class Fingerprint
+  attr_reader :name, :regex, :params, :tests
+
+  def initialize(xml)
+    @name   = description(xml)
+    @regex  = create_regexp(xml)
+    @params = parse_params(xml)
+    @tests  = examples(xml)
+  end
+
+  private
+
+  def description(xml)
+    element = xml.xpath('description')
+    element.empty? ? '' : element.first.content 
+  end
+
+  def create_regexp(xml)
+    pattern = xml['pattern']
+    flags   = xml['flags'].to_s.split(',')
+    RegexpFactory.build(pattern, flags)
+  end
+
+  def parse_params(xml)
+    {}.tap do |h|
+      xml.xpath('param').each do |e|
+        name  = e['name']
+        pos   = e['pos'].to_i
+        value = e['value'].to_s
+        h[name] = [pos, value]
+      end
+    end
+  end
+
+  def examples(xml)
+    xml.xpath('example').collect(&:content)
+  end
+
+  module RegexpFactory
+    def self.build(pattern, flags)
+      options = build_options(flags)
+      Regexp.new(pattern, options)
+    end
+
+    def self.build_options(flags)
+      rflags = Regexp::NOENCODING
+      flags.each do |flag|
+        case flag
+        when 'REG_DOT_NEWLINE', 'REG_LINE_ANY_CRLF'
+          rflags |= Regexp::MULTILINE
+        when 'REG_ICASE'
+          rflags |= Regexp::IGNORECASE 
+        end
+      end
+      rflags
+    end
+  end
+end
+end

data/lib/recog/formatter.rb

@@ -0,0 +1,51 @@
+module Recog
+class Formatter
+  COLORS = {
+    :red    => 31,
+    :yellow => 33,
+    :green  => 32,
+    :white  => 15
+  }
+
+  attr_reader :options, :output
+
+  def initialize(options, output)
+    @options = options
+    @output  = output || StringIO.new
+  end
+
+  def status_message(text)
+    output.puts color(text, :white)
+  end
+
+  def success_message(text)
+    output.puts color(text, :green)
+  end
+
+  def warning_message(text)
+    output.puts color(text, :yellow)
+  end
+
+  def failure_message(text)
+    output.puts color(text, :red)
+  end
+
+  private
+
+  def color_enabled?
+    options.color
+  end
+
+  def color(text, color_code)
+    color_enabled? ? colorize(text, color_code) : text
+  end
+
+  def colorize(text, color_code)
+    "\e[#{color_code_for(color_code)}m#{text}\e[0m"
+  end
+
+  def color_code_for(code)
+    COLORS.fetch(code) { COLORS.fetch(:white) }
+  end
+end
+end

data/lib/recog/match_reporter.rb

@@ -0,0 +1,77 @@
+module Recog
+class MatchReporter
+  attr_reader :formatter
+  attr_reader :line_count, :match_count, :fail_count
+
+  def initialize(options, formatter)
+    @options = options
+    @formatter = formatter
+    reset_counts
+  end
+
+  def report
+    reset_counts
+    yield self
+    summarize
+  end
+
+  def stop?
+    return false unless @options.fail_fast
+    @fail_count >= @options.stop_after
+  end
+
+  def increment_line_count
+    @line_count += 1
+  end
+
+  def match(text)
+    @match_count += 1
+    formatter.success_message(text)
+  end
+
+  def failure(text)
+    @fail_count += 1
+    formatter.failure_message(text)
+  end
+
+  def print_summary
+    colorize_summary(summary_line)
+  end
+
+  private
+
+  def reset_counts
+    @line_count = @match_count = @fail_count = 0
+  end
+
+  def detail?
+    @options.detail
+  end
+
+  def summarize
+    if detail?
+      print_lines_processed
+      print_summary
+    end
+  end
+
+  def print_lines_processed
+    formatter.status_message("\nProcessed #{line_count} lines")
+  end
+
+  def summary_line
+    summary = "SUMMARY: "
+    summary << "#{match_count} matches"
+    summary << " and #{fail_count} failures"
+    summary
+  end
+
+  def colorize_summary(summary)
+    if @fail_count > 0
+      formatter.failure_message(summary)
+    else
+      formatter.success_message(summary)
+    end
+  end
+end
+end

data/lib/recog/matcher.rb

@@ -0,0 +1,60 @@
+module Recog
+class Matcher
+  attr_reader :fingerprints, :reporter
+
+  def initialize(fingerprints, reporter)
+    @fingerprints = fingerprints
+    @reporter = reporter
+  end
+
+  def match_banners(banners_file)
+    reporter.report do
+      
+      fd = $stdin
+      file_source = false
+
+      if banners_file and banners_file != "-"
+        fd = File.open(banners_file, "rb")
+        file_source = true
+      end
+
+      fd.each_line do |line|
+        reporter.increment_line_count
+
+        line = line.to_s.unpack("C*").pack("C*").strip.gsub(/\\[rn]/, '')
+        found = nil 
+        fingerprints.each do |fp|
+          m = line.match(fp.regex)
+          if m
+            found = [fp, m]
+            break
+          end
+        end
+
+        if found
+          info = { }
+          fp, m = found
+          fp.params.each_pair do |k,v|
+            if v[0] == 0
+              info[k] = v[1]
+            else
+              info[k] = m[ v[0] ]
+            end
+          end
+          info['data'] = line
+          reporter.match "MATCH: #{info.inspect}"
+        else
+          reporter.failure "FAIL: #{line}"
+        end
+
+        if reporter.stop?
+          break
+        end
+      end
+
+      fd.close if file_source
+      
+    end
+  end
+end
+end

data/lib/recog/matcher_factory.rb

@@ -0,0 +1,14 @@
+
+require 'recog/matcher'
+require 'recog/formatter'
+require 'recog/match_reporter'
+
+module Recog
+module MatcherFactory
+  def self.build(options)
+    formatter = Formatter.new(options, $stdout)
+    reporter  = MatchReporter.new(options, formatter)
+    Matcher.new(options.fingerprints, reporter)
+  end
+end
+end

data/lib/recog/nizer.rb

@@ -0,0 +1,263 @@
+module Recog
+class Nizer
+
+  # Default certainty ratings where none are specified in the fingerprint itself
+  DEFAULT_OS_CERTAINTY      = 0.85      # Most frequent weights are 0.9, 1.0, and 0.5
+  DEFAULT_SERVICE_CERTAINTY = 0.85      # Most frequent weight is 0.85
+
+  # Non-weighted host attributes that can be extracted from fingerprint matches
+  HOST_ATTRIBUTES = %W{
+    host.domain
+    host.id
+    host.ip
+    host.mac
+    host.name
+    host.time
+    hw.device
+    hw.family
+    hw.product
+    hw.vendor
+  }
+
+  @@db_manager = nil
+
+  #
+  # Locate a database that corresponds with the match_key and attempt to
+  # find a matching fingerprinting, stopping a the first hit. Returns
+  # nil when no matching database or fingerprint is found.
+  #
+  def self.match(match_key, match_string)
+    match_string = match_string.to_s.unpack("C*").pack("C*")
+    @@db_manager ||= Recog::DBManager.new
+    @@db_manager.databases.each do |db|
+      next unless db.match_key == match_key
+      db.fingerprints.each do |fprint|
+        m = fprint.regex.match(match_string)
+        next unless m
+        result = { 'matched' => fprint.name }
+        fprint.params.each_pair do |k,v|
+          if v[0] == 0
+            result[k] = v[1]
+          else
+            result[k] = m[ v[0] ]
+          end
+        end
+        return result  
+      end
+    end
+    nil
+  end
+
+  #
+  # Consider an array of match outputs, choose the best result, taking into
+  # account the granularity of OS vs Version vs SP vs Language. Only consider
+  # fields relevant to the host (OS, name, mac address, etc).
+  #
+  def self.best_os_match(matches)
+
+    # The result hash we return to the caller
+    result = {}
+
+    # Certain attributes should be evaluated separately
+    host_attrs  = {}
+
+    # Bucket matches into matched OS product names
+    os_products = {}
+
+    matches.each do |m|
+      # Count how many times each host attribute value is asserted
+      (HOST_ATTRIBUTES & m.keys).each do |ha|
+        host_attrs[ha]        ||= {}
+        host_attrs[ha][m[ha]] ||= 0
+        host_attrs[ha][m[ha]]  += 1        
+      end
+
+      next unless m.has_key?('os.product')
+
+      # Group matches by OS product and normalize certainty
+      cm = m.dup
+      cm['os.certainty'] = ( m['os.certainty'] || DEFAULT_OS_CERTAINTY ).to_f
+      os_products[ cm['os.product'] ] ||= []
+      os_products[ cm['os.product'] ]  << cm
+    end
+
+    #
+    # Select the best host attribute value by highest frequency
+    #
+    host_attrs.keys.each do |hk|
+      ranked_attr = host_attrs[hk].keys.sort do |a,b| 
+        host_attrs[hk][b] <=> host_attrs[hk][a]
+      end
+      result[hk] = ranked_attr.first
+    end
+
+    # Unable to guess the OS without OS matches
+    unless os_products.keys.length > 0
+      return result
+    end
+
+    #
+    # Select the best operating system name by combined certainty of all
+    # matches within an os.product group. Multiple weak matches can
+    # outweigh a single strong match by design.
+    #
+    ranked_os = os_products.keys.sort do |a,b|    
+      os_products[b].map{ |r| r['os.certainty'] }.inject(:+) <=>
+      os_products[a].map{ |r| r['os.certainty'] }.inject(:+)
+    end
+
+    # Within the best match group, try to fill in missing attributes
+    os_name = ranked_os.first
+
+    # Find the best match within the winning group
+    ranked_os_matches = os_products[os_name].sort do |a,b|
+      b['os.certainty'] <=> a['os.certainty']
+    end
+
+    # Fill in missing result values in descending order of best match
+    ranked_os_matches.each do |rm|
+      rm.each_pair do |k,v|
+        result[k] ||= v
+      end
+    end
+
+    result
+  end
+
+  #
+  # Consider an array of match outputs, choose the best result, taking into
+  # account the granularity of service. Only consider fields relevant to the
+  # service.
+  #
+  def self.best_service_match(matches)
+
+    # The result hash we return to the caller
+    result = {}
+
+    # Bucket matches into matched service product names
+    service_products = {}
+
+    matches.select{ |m| m.has_key?('service.product') }.each do |m|
+      # Group matches by product and normalize certainty
+      cm = m.dup
+      cm['service.certainty'] = ( m['service.certainty'] || DEFAULT_SERVICE_CERTAINTY ).to_f
+      service_products[ cm['service.product'] ] ||= []
+      service_products[ cm['service.product'] ]  << cm
+    end
+
+    # Unable to guess the service without service matches
+    unless service_products.keys.length > 0
+      return result
+    end
+
+    #
+    # Select the best service name by combined certainty of all matches 
+    # within an service.product group. Multiple weak matches can 
+    # outweigh a single strong match by design.
+    #
+    ranked_service = service_products.keys.sort do |a,b|    
+      service_products[b].map{ |r| r['service.certainty'] }.inject(:+) <=>
+      service_products[a].map{ |r| r['service.certainty'] }.inject(:+)
+    end
+
+    # Within the best match group, try to fill in missing attributes
+    service_name = ranked_service.first
+
+    # Find the best match within the winning group
+    ranked_service_matches = service_products[service_name].sort do |a,b|
+      b['service.certainty'] <=> a['service.certainty']
+    end
+
+    # Fill in missing service values in descending order of best match
+    ranked_service_matches.each do |rm|
+      rm.keys.select{ |k| k.index('service.') == 0 }.each do |k|
+        result[k] ||= rm[k]
+      end
+    end
+
+    result
+  end
+ 
+end
+end
+
+=begin
+
+Current key names:
+
+  apache.info
+  apache.variant
+  apache.variant.version
+  cookie
+  host.domain
+  host.id
+  host.ip
+  host.mac
+  host.name
+  host.time
+  hw.device
+  hw.family
+  hw.product
+  hw.vendor
+  imail.eval
+  jetty.info
+  junction.cookie
+  junction.name
+  linux.kernel.version
+  loadbalancer.poolname
+  mdaemon.unregistered
+  mercur.os.info
+  metainfo.version
+  metainfo.version.version
+  ms.nttp.version
+  notes.build.version
+  notes.intl
+  ntmail.id
+  openssh.comment
+  openssh.cvepatch
+  os.arch
+  os.build
+  os.certainty
+  os.device
+  os.edition
+  os.family
+  os.product
+  os.vendor
+  os.version
+  os.version.version
+  os.version.version.version
+  postfix.os.info
+  postoffice.build
+  postoffice.id
+  proftpd.server.name
+  pureftpd.config
+  qpopper.version
+  sendmail.config.version
+  sendmail.hpux.phne.version
+  sendmail.vendor.version
+  service.certainty
+  service.component.family
+  service.component.product
+  service.component.vendor
+  service.component.version
+  service.family
+  service.product
+  service.vendor
+  service.version
+  service.version.version
+  service.version.version.version
+  service.version.version.version.version
+  service.version.version.version.version.version
+  siemens.model
+  snmp.fpmib.oid.1
+  snmp.fpmib.oid.2
+  system.time
+  system.time.format
+  system.time.micros
+  system.time.millis
+  thttpd.mx-patch
+  timeout
+  tomcat.info
+  zmailer.ident
+  
+=end