intrigue-ident 0.2 → 0.9.9

data/lib/http/matchers.rb

@@ -0,0 +1,132 @@
+module Intrigue
+module Ident
+module Http
+module Matchers
+
+  require_relative "http"
+  require_relative 'content'
+  include Intrigue::Ident::Http
+  include Intrigue::Ident::Content::Http
+
+  def match_http_response_hash(check,hash)
+
+    # save off the generator string
+    generator_match = "#{hash[:response_body]}".match(/<meta name=\"?generator\"? content=\"?(.*?)\"?\/>/i)
+    generator_string = generator_match.captures.first.gsub("\"","") if generator_match
+
+    # save off the title string
+    title_match = "#{hash[:response_body]}".match(/<title>(.*?)<\/title>/i)
+    title_string = title_match.captures.first.strip if title_match
+
+    # grab the set cookie header
+    set_cookie_header = "#{(hash[:response_headers]||[]).select{|x| x =~ /^set-cookie:(.*)/i}.first}".gsub("set-cookie:","").strip
+
+    data = hash.merge({
+      "details" =>  {
+        "hidden_response_data" => "#{hash[:response_body]}",
+        "start_url" => "#{hash[:start_url]}",
+        "final_url" => "#{hash[:final_url]}",
+        "headers" => hash[:response_headers], # this is a hash and we need an array!
+        "cookies" => set_cookie_header,
+        "generator" => generator_string,
+        "title" => title_string
+      }
+    })
+
+  match_uri_hash(check,data)
+  end
+
+  def match_browser_response_hash(check,browser_response_hash)
+    data = {
+      "details" =>  {
+        "hidden_response_data_rendered" => "#{browser_response_hash[:rendered]}",
+        "start_url" => "#{browser_response_hash[:start_url]}",
+        "final_url" => "#{browser_response_hash[:final_url]}",
+        "headers" => [],
+        "cookies" => "",
+        "generator" => "",
+        "title" => "#{browser_response_hash[:title]}",
+      }
+    }
+
+    match_uri_hash(check,data)
+  end
+
+  # Matches a text http response
+  def match_http_response_text(check,http_response_text)
+
+    # first convert to intrigue uri format
+
+    # grab headers
+    header_part = http_response_text.split(/\n\n/).first
+    body_part = http_response_text.split(/\n\n/).last
+
+    headers = header_part.split("\n");
+    body = body_part
+
+    # TODO - fix to only grab content!!!!
+    cookies = headers.select{|x| x =~ /^set-cookie:(.*)/i }
+
+    ### grab the page attributes
+    match = body.match(/<title>(.*?)<\/title>/i)
+    title = match.captures.first if match
+
+    match = response.body.match(/<meta name=\"?generator\"? content=\"?(.*?)\"?\/?>/i)
+    generator = match.captures.first.gsub("\"","") if match
+
+    # rest is a response
+    # save title
+    # save Cookies
+    # save scripts ?
+    data = {
+      "details" =>  {
+        "hidden_response_data" => body,
+        "headers" => headers,
+        "cookies" => cookies,
+        "generator" => generator,
+        "title" => title
+      }
+    }
+
+    match_uri_hash(check,data)
+  end
+
+  def match_uri_hash(check, data)
+    return nil unless check && data
+
+    # data[:body] => page body
+    # data[:headers] => block of text with headers, one per line
+    # data[:cookies] => set_cookie header
+    # data[:title] => parsed page title
+    # data[:generator] => parsed meta generator tag
+    # data[:body_md5] => md5 hash of the body
+    # if type "content", do the content check
+
+    if check[:type] == "fingerprint"
+      if check[:match_type] == :content_body
+        match = _construct_match_response(check,data) if _body(data) =~ check[:match_content]
+      elsif check[:match_type] == :content_body_raw
+        match = _construct_match_response(check,data) if _body_raw(data) =~ check[:match_content]
+      elsif check[:match_type] == :content_dom
+        match = _construct_match_response(check,data) if _body_rendered(data) =~ check[:match_content]
+      elsif check[:match_type] == :content_headers
+        match = _construct_match_response(check,data) if _headers(data) =~ check[:match_content]
+      elsif check[:match_type] == :content_cookies
+        match = _construct_match_response(check,data) if _cookies(data) =~ check[:match_content]
+      elsif check[:match_type] == :content_generator
+        match = _construct_match_response(check,data) if _generator(data) =~ check[:match_content]
+      elsif check[:match_type] == :content_title
+        match = _construct_match_response(check,data) if _title(data) =~ check[:match_content]
+      elsif check[:match_type] == :checksum_body
+        match = _construct_match_response(check,data) if _body_raw_checksum(data) == check[:match_content]
+      end
+    elsif check[:type] == "content"
+      match = _construct_match_response(check,data)
+    end
+  match
+  end
+
+end
+end
+end
+end

data/lib/ident.rb

@@ -0,0 +1,263 @@
+#!/usr/bin/env ruby
+
+require 'net/http'
+require 'openssl'
+require 'zlib'
+
+# load in generic utils
+require_relative 'utils'
+require_relative 'version'
+
+# integrate recog
+require_relative 'recog_wrapper'
+
+###
+### Start protocol requires 
+###
+
+##################################
+# Load in http matchers and checks
+###################################
+require_relative 'http/matchers'
+include Intrigue::Ident::Http::Matchers
+
+require_relative 'http/check_factory'
+require_relative '../checks/http/base'
+
+# http fingerprints
+check_folder = File.expand_path('../checks/http', File.dirname(__FILE__)) # get absolute directory
+Dir["#{check_folder}/*.rb"].each { |file| require_relative file }
+
+# http content checks (always run)
+content_check_folder = File.expand_path('../checks/http/content', File.dirname(__FILE__)) # get absolute directory
+Dir["#{content_check_folder}/*.rb"].each { |file| require_relative file }
+
+# http content, wordpress specific checks
+content_check_folder = File.expand_path('../checks/http/wordpress', File.dirname(__FILE__)) # get absolute directory
+Dir["#{content_check_folder}/*.rb"].each { |file| require_relative file }
+
+# General helpers (apply widely across protocols)
+
+require_relative 'simple_socket'
+require_relative 'banner_helpers'
+
+##################################
+# Load in ftp matchers and checks
+#################################
+require_relative 'ftp/matchers'
+include Intrigue::Ident::Ftp::Matchers
+
+require_relative 'ftp/check_factory'
+require_relative '../checks/ftp/base'
+
+# ftp fingerprints
+check_folder = File.expand_path('../checks/ftp', File.dirname(__FILE__)) # get absolute directory
+Dir["#{check_folder}/*.rb"].each { |file| require_relative file }
+
+##################################
+# Load in smtp matchers and checks
+##################################
+require_relative 'smtp/matchers'
+include Intrigue::Ident::Smtp::Matchers
+
+require_relative 'smtp/check_factory'
+require_relative '../checks/smtp/base'
+
+# smtp fingerprints
+check_folder = File.expand_path('../checks/smtp', File.dirname(__FILE__)) # get absolute directory
+Dir["#{check_folder}/*.rb"].each { |file| require_relative file }
+
+##################################
+# Load in snmp matchers and checks
+##################################
+require_relative 'snmp/matchers'
+include Intrigue::Ident::Snmp::Matchers
+
+require_relative 'snmp/check_factory'
+require_relative '../checks/snmp/base'
+
+# snmp fingerprints
+check_folder = File.expand_path('../checks/snmp', File.dirname(__FILE__)) # get absolute directory
+Dir["#{check_folder}/*.rb"].each { |file| require_relative file }
+
+
+##################################
+# Load in ssh matchers and checks
+##################################
+require_relative 'ssh/matchers'
+include Intrigue::Ident::Ssh::Matchers
+
+require_relative 'ssh/check_factory'
+require_relative '../checks/ssh/base'
+
+# ssh fingerprints
+check_folder = File.expand_path('../checks/ssh', File.dirname(__FILE__)) # get absolute directory
+Dir["#{check_folder}/*.rb"].each { |file| require_relative file }
+
+##################################
+# Load in telnet matchers and checks
+##################################
+require_relative 'telnet/matchers'
+include Intrigue::Ident::Telnet::Matchers
+
+require_relative 'telnet/check_factory'
+require_relative '../checks/telnet/base'
+
+# telnet fingerprints
+check_folder = File.expand_path('../checks/telnet', File.dirname(__FILE__)) # get absolute directory
+Dir["#{check_folder}/*.rb"].each { |file| require_relative file }
+
+###
+### End protocol requires 
+###
+
+# Load vulndb client 
+require_relative "vulndb_client"
+
+# set default encoding
+Encoding.default_external = Encoding::UTF_8
+Encoding.default_internal = Encoding::UTF_8
+
+
+# set a base directory so we can use in checks 
+$ident_dir = File.expand_path('../', File.dirname(__FILE__))
+
+module Intrigue
+  module Ident
+
+    private
+
+    def _construct_match_response(check, data)
+
+      if check[:type] == "fingerprint"
+        calculated_version = (check[:dynamic_version].call(data) if check[:dynamic_version]) || check[:version] || ""
+        calculated_update = (check[:dynamic_update].call(data) if check[:dynamic_update]) || check[:update] || ""
+
+        calculated_type = "a" if check[:category] == "application"
+        calculated_type = "h" if check[:category] == "hardware"
+        calculated_type = "o" if check[:category] == "operating_system"
+        calculated_type = "s" if check[:category] == "service" # literally made up
+
+        vendor_string = check[:vendor].gsub(" ","_") if check[:vendor]
+        product_string = check[:product].gsub(" ","_") if check[:product]
+
+        version = "#{calculated_version}".gsub(" ","_")
+        update = "#{calculated_update}".gsub(" ","_")
+
+        cpe_string = "cpe:2.3:#{calculated_type}:#{vendor_string}:#{product_string}:#{version}:#{update}".downcase
+
+        ##
+        ## Support for Dynamic 
+        ##
+        if check[:dynamic_issue]
+          issue = check[:dynamic_issue].call(data)
+        elsif check[:issue]
+          issue = check[:issue]
+        else
+          issue = nil
+        end
+        
+        ##
+        ## Support for Dynamic Hide
+        ##
+        if check[:dynamic_hide]
+          hide = check[:dynamic_hide].call(data)
+        elsif check[:hide]
+          hide = check[:hide]
+        else
+          hide = false
+        end
+
+        ##
+        ## Support for Dynamic Task
+        ##
+        if check[:dynamic_task]
+          task = check[:dynamic_task].call(data)
+        elsif check[:task]
+          task = check[:task]
+        else
+          task = nil
+        end
+
+        to_return = {
+          "method" => "ident",
+          "type" => check[:type],
+          "vendor" => check[:vendor],
+          "product" => check[:product],
+          "version" => calculated_version,
+          "update" => calculated_update,
+          "tags" => check[:tags],
+          "match_type" => check[:match_type],
+          "match_details" => check[:match_details],
+          "hide" => hide,
+          "cpe" => cpe_string,
+          "issue" => issue, 
+          "task" => task, # [{ :task_name => "example", :task_options => {}}]
+          "inference" => check[:inference]
+        }
+
+      elsif check[:type] == "content"
+
+        # Mandatory lambda
+        result = check[:dynamic_result].call(data)
+
+        ##
+        ## Support for Dynamic Issue (must be dynamic, these checks always run)
+        ##
+        if result
+        
+          ##
+          ## Support for Dynamic 
+          ##
+          if check[:dynamic_issue]
+            issue = check[:dynamic_issue].call(data)
+          elsif check[:issue]
+            issue = check[:issue]
+          else
+            issue = nil
+          end
+          
+          ##
+          ## Support for Dynamic Hide
+          ##
+          if check[:dynamic_hide]
+            hide = check[:dynamic_hide].call(data)
+          elsif check[:hide]
+            hide = check[:hide]
+          else
+            hide = false
+          end
+
+          ##
+          ## Support for Dynamic Task
+          ##
+          if check[:dynamic_task]
+            task = check[:dynamic_task].call(data)
+          elsif check[:task]
+            task = check[:task]
+          else
+            task = nil
+          end
+
+        end
+
+        to_return = {
+          "type" => check[:type],
+          "name" => check[:name],
+          "hide" => hide,
+          "issue" => issue,
+          "task" => task,
+          "result" => result
+        }
+      end
+
+    to_return
+    end
+
+end
+end
+
+
+
+# always include 
+include Intrigue::Ident

data/lib/recog_wrapper.rb

@@ -0,0 +1,70 @@
+require 'recog'
+
+module Intrigue
+module Ident
+module RecogWrapper
+
+module Helpers
+
+  def recog_to_ident_hash(recog_hash)
+
+    # do the field conversation 
+    out = {}
+    out["method"] = "recog"
+    out["vendor"] = recog_hash["service.vendor"]
+    out["product"] = recog_hash["service.product"]
+    out["version"] = recog_hash["service.version"]
+    out["cpe"] = recog_hash["service.cpe23"]
+    out["match_details"] = "#{recog_hash["matched"]} (Recog: #{recog_hash["fingerprint_db"]})"
+    out["inference"] = false
+    out["hide"] = false
+    out["issue"] = nil
+
+  out
+  end
+
+end
+
+module Http
+  include Intrigue::Ident::RecogWrapper::Helpers
+
+  def recog_match_http_server_banner(banner)
+    options = OpenStruct.new(color: false, detail: true, fail_fast: false, multi_match: true)
+    ndb = ::Recog::DB.new("http_servers.xml");nil
+    options.fingerprints = ndb.fingerprints;nil
+    matcher = ::Recog::MatcherFactory.build(options);nil
+    matches = matcher.match_banner(banner.gsub("server:","").strip)
+
+    # now convert a match to ident match format
+    matches.compact.map {|m| recog_to_ident_hash(m)}
+  end
+
+  def recog_match_http_cookies(string)
+    options = OpenStruct.new(color: false, detail: true, fail_fast: false, multi_match: true)
+    ndb = ::Recog::DB.new("http_cookies.xml");nil
+    options.fingerprints = ndb.fingerprints;nil
+    matcher = ::Recog::MatcherFactory.build(options);nil
+    matches = matcher.match_banner(string.gsub("set-cookie:","").strip)
+
+    # now convert it 
+    matches.map {|m| recog_to_ident_hash(m)}
+  end
+end
+
+module Smtp
+  include Intrigue::Ident::RecogWrapper::Helpers
+  def recog_match_smtp_banner(string)
+    options = OpenStruct.new(color: false, detail: true, fail_fast: false, multi_match: true)
+    ndb = ::Recog::DB.new("smtp_banners.xml");nil
+    options.fingerprints = ndb.fingerprints;nil
+    matcher = ::Recog::MatcherFactory.build(options);nil
+    matches = matcher.match_banner(string)
+
+    # now convert it 
+    matches.map {|m| recog_to_ident_hash(m)}
+  end
+end
+
+end
+end
+end