recog-intrigue 2.3.7

data/Rakefile

@@ -0,0 +1,22 @@
+require "bundler/gem_tasks"
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new do |t|
+    t.pattern = "spec/**/*_spec.rb"
+end
+
+require 'yard'
+require 'yard/rake/yardoc_task'
+YARD::Rake::YardocTask.new do |t|
+    t.files = ['lib/**/*.rb', '-', 'README.md']
+end
+
+require 'cucumber'
+require 'cucumber/rake/task'
+
+Cucumber::Rake::Task.new(:features) do |t|
+    t.cucumber_opts = "features --format pretty"
+end
+
+task :default => [ :tests, :yard ]
+task :tests => [ :spec, :features ]

data/bin/recog_export

@@ -0,0 +1,81 @@
+#!/usr/bin/env ruby
+
+$:.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", "lib")))
+require 'optparse'
+require 'ostruct'
+require 'recog'
+
+def squash_lines(str)
+  str.split(/\n/).join(' ').gsub(/\s+/, ' ')
+end
+
+def export_text(options)
+end
+
+def export_ruby(options)
+  $stdout.puts "# Recog fingerprint database export [ #{File.basename(options.xml_file)} ] on #{Time.now.to_s}"
+  $stdout.puts "fp_str   = '' # Set this value to the match string"
+  $stdout.puts "fp_match = {} # Match results are stored here"
+  $stdout.puts ""
+  $stdout.puts "case fp_str"
+  options.db.fingerprints.each do |fp|
+    puts "  # #{squash_lines fp.name}"
+    puts "  when /#{fp.regex.to_s}/"
+    fp.tests.each do |test|
+      puts "    # Example: #{squash_lines test}"
+    end
+    fp.params.each_pair do |k,v|
+      if v[0] == 0
+        puts "    fp_match[#{k.inspect}] = #{v[1].inspect}"
+      else
+        puts "    fp_match[#{k.inspect}] = $#{v[0].to_s}"
+      end
+    end
+    puts ""
+  end
+  $stdout.puts "end"
+end
+
+
+options = OpenStruct.new(etype: :ruby)
+
+option_parser = OptionParser.new do |opts|
+  opts.banner = "Usage: #{$0} [options] XML_FINGERPRINTS_FILE"
+  opts.separator "Exports an XML fingerprint database to another format."
+  opts.separator ""
+  opts.separator "Options"
+
+  opts.on("-t", "--type type", 
+          "Choose a type of export.",
+          "  [r]uby (default - export a ruby case statement with regular expressions)",
+          "  [t]ext (export a text description of the fingerprints)") do |etype|
+    case etype.downcase
+    when /^r/
+      options.etype = :ruby
+    when /^t/
+      options.etype = :text
+    end
+  end
+
+  opts.on("-h", "--help", "Show this message.") do
+    puts opts
+    exit
+  end
+end
+option_parser.parse!(ARGV)
+
+if ARGV.count != 1
+  puts option_parser
+  exit
+end
+
+options.xml_file = ARGV.shift
+options.db = Recog::DB.new(options.xml_file)
+
+case options.etype
+when :ruby
+  export_ruby(options)
+when :text
+  export_text(options)
+end
+

data/bin/recog_match

@@ -0,0 +1,55 @@
+#!/usr/bin/env ruby
+
+$:.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", "lib")))
+require 'optparse'
+require 'ostruct'
+require 'recog'
+require 'recog/matcher_factory'
+
+options = OpenStruct.new(color: false, detail: false, fail_fast: false, multi_match: false)
+
+option_parser = OptionParser.new do |opts|
+  opts.banner = "Usage: #{$0} [options] XML_FINGERPRINT_FILE [BANNERS_FILE]"
+  opts.separator "Identifies the matches and misses between the fingerprints and the banners file or STDIN"
+  opts.separator ""
+  opts.separator "Options"
+
+  opts.on("-f", "--format FORMATTER",
+          "Choose a formatter.",
+          "  [s]ummary (default - failure/match msgs)",
+          "  [d]etail  (msgs with total counts)") do |format|
+    if format.start_with? 'd'
+      options.detail = true
+    end
+  end
+
+  opts.on("--fail-fast [NUM]",
+          "Stop after number of failures (default: 10).") do |num|
+    options.fail_fast = true
+    options.stop_after = (num.to_i == 0) ? 10 : num.to_i
+  end
+
+  opts.on("-c", "--color", "Enable color in the output.") do
+    options.color = true
+  end
+
+  opts.on("--[no-]multi-match", "Enable or disable multiple matches (defaults to disabled)") do |o|
+    options.multi_match = o
+  end
+
+  opts.on("-h", "--help", "Show this message.") do
+    puts opts
+    exit
+  end
+end
+option_parser.parse!(ARGV)
+
+if ARGV.count != 1 && ARGV.count != 2
+  puts option_parser
+  exit(1)
+end
+
+ndb = Recog::DB.new(ARGV.shift)
+options.fingerprints = ndb.fingerprints
+matcher = Recog::MatcherFactory.build(options)
+matcher.match_banners(ARGV.shift || "-")

data/bin/recog_standardize

@@ -0,0 +1,118 @@
+#!/usr/bin/env ruby
+
+$:.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", "lib")))
+require 'optparse'
+require 'ostruct'
+require 'recog'
+
+def load_identifiers(path)
+  res = {}
+  File.readlines(path).map{|line| line.strip}.each do |ident|
+    res[ident] = true
+  end
+  return res
+end
+
+def write_identifiers(vals, path)
+  res = []
+  vals.each_pair do |k,v|
+    res = res.push(k)
+  end
+  res = res.sort.uniq
+  File.write(path, res.join("\n") + "\n")
+end
+
+bdir = File.expand_path(File.join(File.dirname(__FILE__), "..", "identifiers"))
+
+options = OpenStruct.new(write: false)
+option_parser = OptionParser.new do |opts|
+  opts.banner = "Usage: #{$0} [options] XML_FINGERPRINT_FILE1 ..."
+  opts.separator "Verifies that each fingerprint asserts known identifiers."
+  opts.separator ""
+  opts.separator "Options"
+
+  opts.on("-w", "--write") do
+      options.write = true
+  end
+
+  opts.on("-h", "--help", "Show this message.") do
+    puts opts
+    exit
+  end
+end
+option_parser.parse!(ARGV)
+
+if ARGV.empty?
+  $stderr.puts 'Missing XML fingerprint files'
+  puts option_parser
+  exit(1)
+end
+
+# Load the unique identifiers
+vendors = load_identifiers(File.join(bdir, "vendor.txt"))
+os_arch = load_identifiers(File.join(bdir, "os_architecture.txt"))
+os_prod = load_identifiers(File.join(bdir, "os_product.txt"))
+os_family = load_identifiers(File.join(bdir, "os_family.txt"))
+os_device = load_identifiers(File.join(bdir, "os_device.txt"))
+svc_prod = load_identifiers(File.join(bdir, "service_product.txt"))
+svc_family = load_identifiers(File.join(bdir, "service_family.txt"))
+
+ARGV.each do |arg|
+  Dir.glob(arg).each do |file|
+    ndb = Recog::DB.new(file)
+    ndb.fingerprints.each do |f|
+      f.params.each do |k,v|
+        paramIndex, val = v
+        next if paramIndex != 0
+        case k
+        when "os.vendor", "service.vendor", "service.component.vendor", "hw.vendor"
+          if ! vendors[val]
+            puts "VENDOR MISSING: #{val}"
+            vendors[val] = true
+          end
+        when "os.product"
+          if ! os_prod[val]
+            puts "OS PRODUCT MISSING: #{val}"
+            os_prod[val] = true
+          end
+        when "os.arch"
+          if ! os_arch[val]
+            puts "OS ARCH MISSING: #{val}"
+            os_arch[val] = true
+          end
+        when "os.family"
+          if ! os_family[val]
+            puts "OS FAMILY MISSING: #{val}"
+            os_family[val] = true
+          end
+        when "os.device"
+          if ! os_device[val]
+            puts "OS DEVICE MISSING: #{val}"
+            os_device[val] = true
+          end
+        when "service.product"
+          if ! svc_prod[val]
+            puts "SERVICE PRODUCT MISSING: #{val}"
+            svc_prod[val] = true
+          end            
+        when "service.family"
+          if ! svc_family[val]
+            puts "SERVICE FAMILY MISSING: #{val}"
+            svc_family[val] = true
+          end          
+        end
+      end
+    end
+  end
+end
+
+exit if ! options.write
+
+# Write back the unique identifiers
+write_identifiers(vendors, File.join(bdir, "vendor.txt"))
+write_identifiers(os_arch, File.join(bdir, "os_architecture.txt"))
+write_identifiers(os_prod, File.join(bdir, "os_product.txt"))
+write_identifiers(os_family, File.join(bdir, "os_family.txt"))
+write_identifiers(os_device, File.join(bdir, "os_device.txt"))
+write_identifiers(svc_prod, File.join(bdir, "service_product.txt"))
+write_identifiers(svc_family, File.join(bdir, "service_family.txt"))

data/bin/recog_verify

@@ -0,0 +1,64 @@
+#!/usr/bin/env ruby
+
+$:.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", "lib")))
+require 'optparse'
+require 'ostruct'
+require 'recog'
+require 'recog/verifier_factory'
+
+options = OpenStruct.new(color: false, detail: false, quiet: false, warnings: true)
+
+option_parser = OptionParser.new do |opts|
+  opts.banner = "Usage: #{$0} [options] XML_FINGERPRINT_FILE1 ..."
+  opts.separator "Verifies that each fingerprint passes its internal tests."
+  opts.separator ""
+  opts.separator "Options"
+
+  opts.on("-f", "--format FORMATTER",
+          "Choose a formatter.",
+          "  [s]ummary (default - failure/warning msgs and summary)",
+          "  [q]uiet (configured failure/warning msgs only)",
+          "  [d]etail  (fingerprint name with tests and expanded summary)") do |format|
+    if format.start_with? 'd'
+      options.detail = true
+    end
+    if format.start_with? 'q'
+      options.quiet = true
+    end
+  end
+
+  opts.on("-c", "--color", "Enable color in the output.") do
+    options.color = true
+  end
+
+  opts.on("--[no-]warnings", "Track warnings") do |o|
+    options.warnings = o
+  end
+
+  opts.on("-h", "--help", "Show this message.") do
+    puts opts
+    exit
+  end
+end
+option_parser.parse!(ARGV)
+
+if ARGV.empty?
+  $stderr.puts 'Missing XML fingerprint files'
+  puts option_parser
+  exit(1)
+end
+
+warnings = 0
+failures = 0
+ARGV.each do |arg|
+  Dir.glob(arg).each do |file|
+    ndb = Recog::DB.new(file)
+    options.fingerprints = ndb.fingerprints
+    verifier = Recog::VerifierFactory.build(options)
+    verified = verifier.verify
+    failures += verifier.reporter.failure_count
+    warnings += verifier.reporter.warning_count
+  end
+end
+
+exit failures + warnings

data/cpe-remap.yaml

@@ -0,0 +1,134 @@
+mappings:
+  apache:
+    vendor: apache
+    products:
+      httpd: http_server
+  apple:
+    products:
+      ios: iphone_os
+  alt-n:
+    vendor: altn
+  bea:
+    vendor: bea
+    products:
+      weblogic: weblogic_server
+  blue_coat:
+    vendor: bluecoat
+  centos:
+    vendor: centos
+    products:
+      linux: centos
+  check_point:
+    vendor: checkpoint
+  cisco:
+    vendor: cisco
+    products:
+      adaptive_security_appliance: adaptive_security_appliance_software
+      pix: pix_firewall_software
+      telepresence: telepresence_video_communication_server_software
+  debian:
+    vendor: debian
+    products:
+      linux: debian_linux
+  f5:
+    vendor: f5
+    products:
+      big-ip: big-ip_local_traffic_manager
+      big-ip_ltm: big-ip_local_traffic_manager
+  hp:
+    vendor: hp
+    products:
+      ilo: integrated_lights_out
+      lotus_domino: lotus_domino_server
+      tru64_unix: tru64
+  ibm:
+    vendor: ibm
+    products:
+      lotus_domino: lotus_domino_server
+  juniper:
+    vendor: juniper
+    products:
+      junos_os: junos
+  linux:
+    vendor: linux
+    products:
+      linux: linux_kernel
+  mailenable:
+    vendor: mailenable
+    products:
+      mail_server: mailenable
+  microsoft:
+    vendor: microsoft
+    products:
+      active_directory_controller: active_directory
+      exchange_server_5.5: exchange_server
+      exchange_2000_server: exchange_server
+      exchange_2003_server: exchange_server
+      exchange_2007_server: exchange_server
+      lightweight_directory_server: active_directory_lightweight_directory_service
+      windows_server_2003_datacenter_edition: windows_server_2003
+      windows_server_2003_r2: windows_server_2003
+      windows_2008_r2: windows_server_2008
+      windows_server_2008_datacenter_edition: windows_server_2008
+      windows_server_2008_r2: windows_server_2008
+      windows_server_2008_r2_datacenter_edition: windows_server_2008
+      windows_server_2012_r2: windows_server_2012
+      nt: windows_nt
+      windows_nt_desktop: windows_nt
+      windows_nt_server: windows_nt
+      windows_server_2000: windows_2000
+      windows_2000_server: windows_2000
+      windows_2000_datacenter_server: windows_2000
+      pws: personal_web_server
+  mod_ssl:
+    vendor: modssl
+  mod_wsgi:
+    vendor: modwsgi
+  mort_bay:
+    vendor: mortbay
+  net-snmp:
+    vendor: net-snmp
+    products:
+      snmp_agent: net-snmp
+  palo_alto_networks:
+    vendor: paloaltonetworks
+    products:
+      pa_firewall: pan-os
+  proftpd_project:
+    vendor: proftpd
+  realvnc_ltd.:
+    vendor: realvnc
+  red_hat:
+    vendor: redhat
+    products:
+      cygwin_x_server_project: cygwin
+      fedora_core_linux: fedora_core
+      jboss_as: jboss_wildfly_application_server
+      jboss_eap: jboss_enterprise_application_platform
+      jbossweb: jboss_web_framework_kit
+      red_hat_directory_server: directory_server
+  squid_cache:
+    vendor: squid-cache
+  sun:
+    vendor: sun
+    products:
+      solaris: sunos
+  ubuntu:
+    vendor: canonical
+    products:
+      linux: ubuntu_linux
+  vandyke_software:
+    vendor: vandyke
+  vmware:
+    vendor: vmware
+    products:
+      photon_linux: photon_os
+      zimbra: zimbra_desktop
+      vmware_esx_server: esx
+      vmware_esxi_server: esxi
+  wind_river:
+    vendor: windriver
+  x.org:
+    vendor: x.org
+    products:
+      x.org_x11: x11