recog-intrigue 2.3.7

data/lib/recog/verifier.rb

@@ -0,0 +1,39 @@
+module Recog
+class Verifier
+  attr_reader :fingerprints, :reporter
+
+  def initialize(fingerprints, reporter)
+    @fingerprints = fingerprints
+    @reporter = reporter
+  end
+
+  def verify
+    reporter.report(fingerprints.count) do
+      fingerprints.each do |fp|
+        reporter.print_name fp
+
+        fp.verify_params do |status, message|
+          case status
+          when :warn
+            reporter.warning "WARN: #{message}"
+          when :fail
+            reporter.failure "FAIL: #{message}"
+          when :success
+            reporter.success(message)
+          end
+        end
+        fp.verify_tests do |status, message|
+          case status
+          when :warn
+            reporter.warning "WARN: #{message}"
+          when :fail
+            reporter.failure "FAIL: #{message}"
+          when :success
+            reporter.success(message)
+          end
+        end
+      end
+    end
+  end
+end
+end

data/lib/recog/verifier_factory.rb

@@ -0,0 +1,13 @@
+require 'verifier'
+require 'formatter'
+require 'verify_reporter'
+
+module Recog
+module VerifierFactory
+  def self.build(options)
+    formatter = Formatter.new(options, $stdout)
+    reporter  = VerifyReporter.new(options, formatter)
+    Verifier.new(options.fingerprints, reporter)
+  end
+end
+end

data/lib/recog/verify_reporter.rb

@@ -0,0 +1,86 @@
+module Recog
+class VerifyReporter
+  attr_reader :formatter
+  attr_reader :success_count, :warning_count, :failure_count
+
+  def initialize(options, formatter)
+    @options = options
+    @formatter = formatter
+    reset_counts
+  end
+
+  def report(fingerprint_count)
+    reset_counts
+    yield self
+    summarize(fingerprint_count) unless @options.quiet
+  end
+
+  def success(text)
+    @success_count += 1
+    formatter.success_message("#{padding}#{text}") if detail?
+  end
+
+  def warning(text)
+    return unless @options.warnings
+    @warning_count += 1
+    formatter.warning_message("#{padding}#{text}")
+  end
+
+  def failure(text)
+    @failure_count += 1
+    formatter.failure_message("#{padding}#{text}")
+  end
+
+  def print_name(fingerprint)
+    if detail? && fingerprint.tests.any?
+      name = fingerprint.name.empty? ? '[unnamed]' : fingerprint.name
+      formatter.status_message("\n#{name}")
+    end
+  end
+
+  def summarize(fingerprint_count)
+    print_fingerprint_count(fingerprint_count) if detail?
+    print_summary
+  end
+
+  def print_fingerprint_count(count)
+    formatter.status_message("\nVerified #{count} fingerprints:")
+  end
+
+  def print_summary
+    colorize_summary(summary_line)
+  end
+
+  private
+
+  def reset_counts
+    @success_count = @failure_count = @warning_count = 0
+  end
+
+  def detail?
+    @options.detail
+  end
+
+  def padding
+    '   ' if @options.detail
+  end
+
+  def summary_line
+    summary = "SUMMARY: Test completed with "
+    summary << "#{@success_count} successful"
+    summary << ", #{@warning_count} warnings"
+    summary << ", and #{@failure_count} failures"
+    summary
+  end
+
+  def colorize_summary(summary)
+    if @failure_count > 0
+      formatter.failure_message(summary)
+    elsif @warning_count > 0
+      formatter.warning_message(summary)
+    else
+      formatter.success_message(summary)
+    end
+  end
+end
+end

data/lib/recog/version.rb

@@ -0,0 +1,3 @@
+module Recog
+  VERSION = '2.3.7'
+end

data/misc/convert_mysql_err

@@ -0,0 +1,61 @@
+#!/usr/bin/env ruby
+
+# Takes the MySQL error messages from sql/share/errmsg-utf8.txt, locates the
+# provided error message type (for example, ER_HOST_NOT_PRIVILEGED), then
+# creates XML snippets for each locale to be used in Recog.  Note that this
+# cannot be used as-is to generate mysql_errors.xml, or oftentimes even parts
+# -- it merely spits out XML snippets that you can start with; many will still
+# need to be modified by hand.
+
+require 'builder'
+require 'open-uri'
+require 'securerandom'
+
+def generate_recog(error_name, locale, error_message)
+  xml = Builder::XmlMarkup.new(target: STDOUT, indent: 2)
+  xml.fingerprint(pattern: error_message) do
+    xml.description "Oracle MySQL error #{error_name} (#{locale})"
+    xml.example(error_message)
+    xml.param(pos: 0, name: 'service.vendor', value: 'Oracle')
+    xml.param(pos: 0, name: 'service.family', value: 'MySQL')
+    xml.param(pos: 0, name: 'service.product', value: 'MySQL')
+  end
+end
+
+unless ARGV.size == 2
+  fail "Usage: #{$PROGRAM_NAME} <path/URI for errmsg-utf8.txt> <error name>"
+end
+
+path = ARGV.first
+error_name = ARGV.last
+
+lines = IO.readlines(open(path))
+
+fail "Nothing read from #{path}" if lines.empty?
+
+unless (error_start = lines.find_index { |line| line.strip =~ /^#{error_name}(?:\s+\S+)?$/ })
+  fail "Unable to find #{error_name} in #{path}"
+end
+
+locale_map = {}
+lines.slice(error_start + 1, lines.size).each do |line|
+  if /^\s+(?<locale>\S+)\s+"(?<error_message>.*)",?$/ =~ line
+    locale_map[locale] = error_message
+  else
+    break
+  end
+end
+
+# Many of the error messages contain format strings.  This can be problematic
+# in that they need to be removed or otherwise handled as part of the 'pattern'
+# attribute and appropriately filled in in any example elements.  So simply try
+# a rough count of the possible format strings and warn the user so that they
+# can deal with it.
+format_count = locale_map.values.map { |error_message| error_message.scan(/%/).size }.inject(&:+)
+unless format_count == 0
+  warn("#{format_count} possible format strings found -- you'll need to deal with this")
+end
+
+Hash[locale_map.sort].map do |locale, error_message|
+  generate_recog(error_name, locale, error_message)
+end

data/misc/order.xsl

@@ -0,0 +1,17 @@
+<?xml version="1.0"?>
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
+  <xsl:output encoding="UTF-8" indent="yes" method="xml"/>
+  <xsl:template match="@*|node()">
+    <xsl:copy>
+      <xsl:apply-templates select="@*|node()"/>
+    </xsl:copy>
+  </xsl:template>
+  <xsl:template match="fingerprints/fingerprint">
+    <xsl:copy>
+      <xsl:copy-of select="@*"/>
+      <xsl:apply-templates select="description"/>
+      <xsl:apply-templates select="example"/>
+      <xsl:apply-templates select="param"/>
+    </xsl:copy>
+  </xsl:template>
+</xsl:stylesheet>

data/recog-intrigue.gemspec

@@ -0,0 +1,45 @@
+# -*- encoding: utf-8 -*-
+$LOAD_PATH.push File.expand_path('../lib', __FILE__)
+require 'recog/version'
+
+Gem::Specification.new do |s|
+  s.name        = 'recog-intrigue'
+  s.version     = Recog::VERSION
+  s.required_ruby_version = '>= 2.1'
+  s.authors     = [
+      'Rapid7 Research',
+      'jcran'
+  ]
+  s.email       = [
+      'jcran@intrigue.io'
+  ]
+  s.homepage    = "https://www.github.com/rapid7/recog"
+  s.summary     = %q{Network service fingerprint database, classes, and utilities}
+  s.description = %q{
+    Recog is a framework for identifying products, services, operating systems, and hardware by matching
+    fingerprints against data returned from various network probes. Recog makes it simply to extract useful
+    information from web server banners, snmp system description fields, and a whole lot more.
+  }.gsub(/\s+/, ' ').strip
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ['lib']
+
+  # ---- Dependencies ----
+
+  s.add_development_dependency 'rspec'
+  s.add_development_dependency 'yard'
+  if RUBY_PLATFORM =~ /java/
+    # markdown formatting for yard
+    s.add_development_dependency 'kramdown'
+  else
+    # markdown formatting for yard
+    s.add_development_dependency 'redcarpet'
+  end
+  s.add_development_dependency 'cucumber'
+  s.add_development_dependency 'aruba'
+  s.add_development_dependency 'simplecov'
+
+  s.add_runtime_dependency 'nokogiri'
+end

data/requirements.txt

@@ -0,0 +1,2 @@
+lxml==4.2.4
+pyyaml

data/spec/data/best_os_match_1.yml

@@ -0,0 +1,17 @@
+---
+- os.product: Windows 2008
+  os.vendor: Microsoft
+  os.version: Service Pack 2
+  os.certainty: 0.5
+- os.product: Windows 2008
+  os.vendor: Microsoft
+  os.version: Service Pack 1
+  os.certainty: 0.4
+- os.product: Windows 2008
+  os.vendor: Microsoft
+  os.certainty: 0.3
+  os.language: English
+- os.product: Windows 2012
+  os.vendor: Microsoft
+  os.certainty: 0.4
+  os.language: Arabic

data/spec/data/best_os_match_2.yml

@@ -0,0 +1,17 @@
+---
+- os.product: Windows 2008
+  os.vendor: Microsoft
+  os.version: Service Pack 2
+  os.certainty: 1.0
+- os.product: Windows 2012
+  os.vendor: Microsoft
+  os.version: Service Pack 1
+  os.certainty: 0.7
+- os.product: Windows 2008
+  os.vendor: Microsoft
+  os.certainty: 0.3
+  os.language: English
+- os.product: Windows 2012
+  os.vendor: Microsoft
+  os.certainty: 0.8
+  os.language: Arabic

data/spec/data/best_service_match_1.yml

@@ -0,0 +1,17 @@
+---
+- service.product: IIS
+  service.vendor: Microsoft
+  service.version: 6.0
+  service.certainty: 1.0
+- service.product: Apache
+  service.vendor: Linux
+  service.version: 2.2.4
+  service.certainty: 0.5
+- service.product: IIS
+  service.vendor: Microsoft
+  service.certainty: 0.5
+  service.language: English
+- service.product: IIS
+  service.vendor: Microsoft
+  service.certainty: 0.4
+  service.language: Arabic

data/spec/data/smb_native_os.txt

@@ -0,0 +1,25 @@
+Windows Web Server 2008 R2 7601 Service Pack 1
+Windows Vista (TM) Home Premium 6002 Service Pack 2
+Windows Server (R) 2008 Standard 6002 Service Pack 2
+Windows Server (R) 2008 Enterprise without Hyper-V 6001 Service Pack 1
+Windows Server (R) 2008 Enterprise 6002 Service Pack 2
+Windows Server (R) 2008 Enterprise 6001 Service Pack 1
+Windows Server 2012 Standard 9200
+Windows Server 2012 R2 Standard 9600
+Windows Server 2008 R2 Standard 7601 Service Pack 1
+Windows Server 2008 R2 Standard 7600
+Windows Server 2008 R2 Enterprise 7601 Service Pack 1
+Windows Server 2008 R2 Enterprise 7600
+Windows Server 2008 HPC Edition 7600
+Windows Server 2003 R2 3790 Service Pack 2
+Windows Server 2003 3790 Service Pack 2
+Windows (R) Web Server 2008 6002 Service Pack 2
+Windows MultiPoint Server 2012 Premium 9200
+Windows 8 Enterprise 9200
+Windows 8.1 Enterprise 9600
+Windows 7 Ultimate 7601 Service Pack 1
+Windows 7 Ultimate 7600
+Windows 7 Starter 7601 Service Pack 1
+Windows 7 Home Premium 7600
+Windows 7 Enterprise 7601 Service Pack 1
+Windows 7 Enterprise 7600

data/spec/data/test_fingerprints.xml

@@ -0,0 +1,36 @@
+<?xml version="1.0"?>
+<fingerprints>
+   <fingerprint pattern=".*\(iSeries\).*">
+   </fingerprint>
+
+   <fingerprint pattern=".*\(PalmOS\).*">
+     <description>PalmOS</description>
+     <param pos="1" name="os.vendor" value="Palm"/>
+     <param pos="2" name="os.device" value="General"/>
+   </fingerprint>
+
+   <fingerprint pattern="(designjet \S+)" flags="REG_ICASE">
+     <description>HP Designjet printer</description>
+     <description>I should be ignored</description>
+     <param pos="0" name="service.vendor" value="HP"/>
+   </fingerprint>
+
+   <fingerprint pattern="laserjet (.*)(?: series)?" flags="REG_ICASE">
+     <description>HP JetDirect Printer</description>
+     <example>HP LaserJet 4100 Series</example>
+     <example>HP LaserJet 2200</example>
+     <param pos="0" name="service.vendor" value="HP"/>
+   </fingerprint>
+
+   <fingerprint pattern="^Windows XP (\d+) (Service Pack \d+)$">
+      <description>Windows XP</description>
+      <example os.build="2600" os.version="Service Pack 1"
+        >Windows XP 2600 Service Pack 1</example>
+      <param pos="0" name="os.certainty" value="1.0"/>
+      <param pos="0" name="os.vendor" value="Microsoft"/>
+      <param pos="0" name="os.product" value="Windows XP"/>
+      <param pos="1" name="os.build"/>
+      <param pos="2" name="os.version"/>
+   </fingerprint>
+
+</fingerprints>

data/spec/data/verification_fingerprints.xml

@@ -0,0 +1,86 @@
+<fingerprints>
+
+  <fingerprint pattern="^[dD]ovecot (?:\((.*)\) )?(?:DA )?ready\.(?: &lt;.+@(.+)&gt;)?$">
+    <description>Dovecot Secure POP Server - no params</description>
+    <example>Dovecot ready.</example>
+  </fingerprint>
+
+  <fingerprint pattern="^[dD]ovecot (?:\((.*)\) )?(?:DA )?ready\.(?: &lt;.+@(.+)&gt;)?$">
+    <description>Dovecot Secure POP Server - no params defined by capture group</description>
+    <example>Dovecot ready.</example>
+    <param pos="0" name="service.family" value="Dovecot"/>
+    <param pos="0" name="service.product" value="Dovecot"/>
+  </fingerprint>
+
+  <fingerprint pattern="^[dD]ovecot (?:\((.*)\) )?(?:DA )?ready\.(?: &lt;.+@(.+)&gt;)?$">
+    <description>Dovecot Secure POP Server - no example</description>
+    <param pos="0" name="service.family" value="Dovecot"/>
+    <param pos="0" name="service.product" value="Dovecot"/>
+  </fingerprint>
+
+  <fingerprint pattern="^[dD]ovecot (?:\((.*)\) )?(?:DA )?ready\.(?: &lt;.+@(.+)&gt;)?$">
+    <description>Dovecot Secure POP Server - one parameter, one example</description>
+    <example host.name="domain">Dovecot ready. &lt;abc11.1.1234abcd.abdcabcdabcd@domain&gt;</example>
+    <param pos="0" name="service.family" value="Dovecot"/>
+    <param pos="0" name="service.product" value="Dovecot"/>
+    <param pos="2" name="host.name"/>
+  </fingerprint>
+
+  <fingerprint pattern="^[dD]ovecot (?:\((.*)\) )?(?:DA )?ready\.(?: &lt;.+@(.+)&gt;)?$">
+    <description>Dovecot Secure POP Server - two paremeters, one example</description>
+    <example host.name="domain" os.vendor="Ubuntu">Dovecot (Ubuntu) ready. &lt;abc11.1.1234abcd.abdcabcdabcd@domain&gt;</example>
+    <param pos="0" name="service.family" value="Dovecot"/>
+    <param pos="0" name="service.product" value="Dovecot"/>
+    <param pos="1" name="os.vendor"/>
+    <param pos="2" name="host.name"/>
+  </fingerprint>
+
+  <fingerprint pattern="^[dD]ovecot (?:\((.*)\) )?(?:DA )?ready\.(?: &lt;.+@(.+)&gt;)?$">
+    <description>Dovecot Secure POP Server - two parameters, two examples</description>
+    <example host.name="domain">Dovecot ready. &lt;abc11.1.1234abcd.abdcabcdabcd@domain&gt;</example>
+    <example os.vendor="Ubuntu">Dovecot (Ubuntu) ready.</example>
+    <param pos="0" name="service.family" value="Dovecot"/>
+    <param pos="0" name="service.product" value="Dovecot"/>
+    <param pos="1" name="os.vendor"/>
+    <param pos="2" name="host.name"/>
+  </fingerprint>
+
+  <fingerprint pattern="^[dD]ovecot (?:\((.*)\) )?(?:DA )?ready\.(?: &lt;.+@(.+)&gt;)?$">
+    <description>Dovecot Secure POP Server - two parameters, one example, one missing param</description>
+    <example host.name="domain">Dovecot ready. &lt;abc11.1.1234abcd.abdcabcdabcd@domain&gt;</example>
+    <param pos="0" name="service.family" value="Dovecot"/>
+    <param pos="0" name="service.product" value="Dovecot"/>
+    <param pos="1" name="os.vendor"/>
+    <param pos="2" name="host.name"/>
+  </fingerprint>
+
+  <fingerprint pattern="^[dD]ovecot (?:\((.*)\) )?(?:DA )?ready\.(?: &lt;.+@(.+)&gt;)?$">
+    <description>Dovecot Secure POP Server - two parameters, one example, two missing params</description>
+    <example>Dovecot ready. &lt;abc11.1.1234abcd.abdcabcdabcd@domain&gt;</example>
+    <param pos="0" name="service.family" value="Dovecot"/>
+    <param pos="0" name="service.product" value="Dovecot"/>
+    <param pos="1" name="os.vendor"/>
+    <param pos="2" name="host.name"/>
+  </fingerprint>
+
+  <fingerprint pattern="^[dD]ovecot (?:\((.*)\) )?(?:DA )?ready\.(?: &lt;.+@(.+)&gt;)?$">
+    <description>Dovecot Secure POP Server - two parameters, two examples, one missing param</description>
+    <example host.name="domain">Dovecot ready. &lt;abc11.1.1234abcd.abdcabcdabcd@domain&gt;</example>
+    <example>Dovecot (Ubuntu) ready.</example>
+    <param pos="0" name="service.family" value="Dovecot"/>
+    <param pos="0" name="service.product" value="Dovecot"/>
+    <param pos="1" name="os.vendor"/>
+    <param pos="2" name="host.name"/>
+  </fingerprint>
+
+  <fingerprint pattern="^[dD]ovecot (?:\((.*)\) )?(?:DA )?ready\.(?: &lt;.+@(.+)&gt;)?$">
+    <description>Dovecot Secure POP Server - two parameters, two examples, two missing params</description>
+    <example>Dovecot ready. &lt;abc11.1.1234abcd.abdcabcdabcd@domain&gt;</example>
+    <example>Dovecot (Ubuntu) ready.</example>
+    <param pos="0" name="service.family" value="Dovecot"/>
+    <param pos="0" name="service.product" value="Dovecot"/>
+    <param pos="1" name="os.vendor"/>
+    <param pos="2" name="host.name"/>
+  </fingerprint>
+
+</fingerprints>