intrigue-ident 0.9.9 → 0.31

data/lib/ftp/matchers.rb

@@ -1,26 +0,0 @@
-module Intrigue
-module Ident
-module Ftp
-module Matchers
-
-  require_relative 'ftp'
-  include Intrigue::Ident::Ftp
-  
-  require_relative 'content'
-  include Intrigue::Ident::Ftp::Content
-
-  def match_ftp_response_hash(check,response_hash)
-    
-    if check[:type] == "fingerprint"
-      if check[:match_type] == :content_banner
-        puts "Banner: #{_banner(response_hash)}"
-        match = _construct_match_response(check,response_hash) if (_banner(response_hash) =~ check[:match_content])
-      end
-    end
-
-  end
-
-end
-end
-end
-end

data/lib/http/browser.rb

@@ -1,260 +0,0 @@
-###
-### Please note - these methods may be used inside task modules, or inside libraries within
-### Intrigue. An attempt has been made to make them abstract enough to use anywhere inside the
-### application, but they are primarily designed as helpers for tasks. This is why you'll see
-### references to @task_result in these methods. We do need to check to make sure it's available before
-### writing to it.
-###
-
-# This module exists for common web functionality - inside a web browser
-module Intrigue
-module Ident
-  module HttpBrowser
-
-    def ident_create_browser_session
-
-      # first check if we're allowed to create a session by the global config
-      return nil unless Intrigue::Config::GlobalConfig.config["browser_enabled"]
-
-      # start a new session
-      args = ['headless', 'disable-gpu', 'disable-dev-shm-usage', 
-       'ignore-certificate-errors', 'disable-popup-blocking', 'disable-translate']
-
-      # configure the driver to run in headless mode
-      options = Selenium::WebDriver::Chrome::Options.new(args: args)
-      
-      # create a driver
-      driver = Selenium::WebDriver.for :chrome, { options: options }
-
-      # set size 
-      driver.manage.window.size = Selenium::WebDriver::Dimension.new(1280, 1024)
-
-      # set a default timeout
-      driver.manage.timeouts.implicit_wait = 20 # seconds
-    
-    driver 
-    end
-
-    def ident_destroy_browser_session(session)
-
-      return false unless session
-
-      # get the full group id (driver + browser)
-      begin
-
-        # HACK HACK HACK- get the chromedriver process before we quit
-        #driver_pid = session.driver.browser.instance_variable_get(:@service).instance_variable_get(:@process).pid
-
-        # attempt to quit gracefully...
-        session.close
-
-        #pgid = Process.getpgid(driver_pid)
-
-        # violent delights have violent ends
-        #Process.kill('KILL', -pgid )
-        #Process.kill('KILL', driver_pid )
-      rescue Selenium::WebDriver::Error::UnknownError => e
-        _log_error "Error trying to kill our browser session #{e}"
-      rescue Errno::ESRCH => e
-          # already dead
-        _log_error "Error trying to kill our browser session #{e}"
-      rescue Net::ReadTimeout => e
-        _log_error "Timed out trying to close our session.. #{e}"
-      end
-
-    true 
-    end
-
-    def ident_safe_browser_action
-      begin
-
-        results = yield
-
-      rescue Errno::EMFILE => e
-        _log_error "Too many open files: #{e}" if @task_result
-      rescue Addressable::URI::InvalidURIError => e
-        _log_error "Invalid URI: #{e}" if @task_result
-      rescue Net::ReadTimeout => e
-        _log_error "Timed out, moving on" if @task_result
-      rescue Selenium::WebDriver::Error::WebDriverError => e
-        # skip simple errors where we're testing JS libs
-        unless ("#{e}" =~ /is not defined/ || "#{e}" =~ /Cannot read property/)
-          _log_error "Webdriver issue #{e}" if @task_result
-        end
-      rescue Selenium::WebDriver::Error::NoSuchWindowError => e
-        _log_error "Lost our window #{e}" if @task_result
-      rescue Selenium::WebDriver::Error::UnknownError => e
-        # skip simple errors where we're testing JS libs
-        unless ("#{e}" =~ /is not defined/ || "#{e}" =~ /Cannot read property/)
-          _log_error "#{e}" if @task_result
-        end
-      rescue Selenium::WebDriver::Error::UnhandledAlertError => e
-        _log_error "Unhandled alert open: #{e}" if @task_result
-      rescue Selenium::WebDriver::Error::NoSuchElementError
-        _log_error "No such element #{e}, moving on" if @task_result
-      rescue Selenium::WebDriver::Error::StaleElementReferenceError
-        _log_error "No such element ref #{e}, moving on" if @task_result
-      end
-    results
-    end
-
-    def ident_capture_document(session, uri)
-      return nil unless session # always make sure the session is real
-
-      # browse to our target
-      safe_browser_action do
-        # visit the page
-        session.navigate.to(uri)
-        # Capture Title
-        page_title = session.title
-        # Capture Body Text
-        page_contents = session.page_source
-        # Capture DOM
-        rendered_page = session.execute_script("return document.documentElement.innerHTML")
-
-        # return our hash
-        return { :title => page_title, :contents => page_contents, :rendered => rendered_page }
-      end
-    nil 
-    end
-
-    def ident_capture_screenshot(session, uri)
-      return nil unless session # always make sure the session is real
-
-      # browse to our target
-      safe_browser_action do
-        session.navigate.to(uri)
-      end
-
-      #
-      # Capture a screenshot
-      #
-      base64_image_contents = nil
-      safe_browser_action do
-        tempfile = Tempfile.new(['screenshot', '.png'])
-        session.save_screenshot(tempfile.path)
-        _log "Saved Screenshot to #{tempfile.path}"
-        # open and read the file's contents, and base64 encode them
-        base64_image_contents = Base64.encode64(File.read(tempfile.path))
-        # cleanup
-        tempfile.close
-        tempfile.unlink
-      end
-
-    base64_image_contents
-    end
-
-    def ident_gather_javascript_libraries(session, uri)
-      return nil unless session # always make sure the session is real
-
-      # Test site: https://www.jetblue.com/plan-a-trip/#/
-      # Examples: https://builtwith.angularjs.org/
-      # Examples: https://www.madewithangular.com/
-
-      safe_browser_action do
-        session.navigate.to(uri)
-      end
-
-      libraries = []
-
-      checks = [
-        { library: "Angular", script: 'angular.version.full' },
-        # Backbone
-        # Test site: https://app.casefriend.com/
-        # Examples: https://github.com/jashkenas/backbone/wiki/projects-and-companies-using-backbone
-        { library: "Backbone", script: 'Backbone.VERSION' },
-        # D3
-        # Test site: https://d3js.org/
-        # Examples: https://kartoweb.itc.nl/kobben/D3tests/index.html
-        { library: "D3", script: 'd3.version' },
-        # Dojo
-        # Test site: http://demos.dojotoolkit.org/demos/mobileCharting/demo.html
-        # Examples: http://demos.dojotoolkit.org/demos/
-        { library: "Dojo", script: 'dojo.version' },
-        # Ember
-        # Test site: https://secure.ally.com/
-        # Examples: http://builtwithember.io/
-        { library: "Ember", script: 'Ember.VERSION' },
-
-        # Honeybadger
-        { library: "Honeybadger", script: 'Honeybadger.getVersion()' },
-
-        # Intercom
-        # Examples: https://bugcrowd.com
-        { library: "Intercom", script: 'Intercom("version")' },
-
-        # Jquery
-        # Test site: http://www.eddiebauer.com/
-        # Test site: https://www.underarmour.com
-        { library: "jQuery", script: 'jQuery.fn.jquery' },
-        # Jquery tools
-        # Test site: http://www.eddiebauer.com/
-        { library: "jQuery Tools", script: 'jQuery.tools.version' },
-        # Jquery UI
-        # Test site: http://www.eddiebauer.com/
-        # Test site: https://www.underarmour.com
-        { library: "jQuery UI", script: 'jQuery.ui.version' },
-
-        # Test site:
-        # Examples: http://knockoutjs.com/examples/
-        #version = session.evaluate_script('knockout.version')
-        # { :product => "Knockout", check: 'knockout.version' }
-
-        # Modernizr
-        { library: "Modernizr", script: 'Modernizr._version' },
-
-        # Paper.js
-        # Test site: http://paperjs.org/examples/boolean-operations
-        # Examples: http://paperjs.org/examples
-
-        # Prototype
-        # Test site:
-        # Examples:
-        # version = session.evaluate_script('Prototype.version')
-        # { product: "Prototype", check: 'Prototype.version' },
-
-        { library: "Paper", script: 'paper.version' },
-
-        # React
-        # Test site: https://weather.com/
-        # Examples: https://react.rocks/
-        { library: "React", script: 'React.version' },
-
-        # RequireJS
-        # Test site: https://www.homedepot.com
-        { library: "RequireJS", script: 'requirejs.version' },
-
-        # Underscore
-        # Test site: https://app.casefriend.com/#sessions/login
-        # Test site: https://store.dji.com/
-        { library: "Underscore", script: '_.VERSION' },
-
-        # YUI
-        # Test site: https://yuilibrary.com/yui/docs/event/basic-example.html
-        # Examples: https://yuilibrary.com/yui/docs/examples/
-        { library: "YUI", script: 'YUI().version' }
-      ]
-
-      checks.each do |check|
-
-        hacky_javascript = "return #{check[:script]};"
-
-        # run our script in a browser
-        version = safe_browser_action do
-          session.execute_script(hacky_javascript)
-        end
-
-        if version
-          _log_good "Detected #{check[:library]} #{version}" if @task_result
-          libraries << {"library" => "#{check[:library]}", "version" => "#{version}" }
-        end
-
-      end
-
-    libraries
-    end
-
-
-  end
-end
-end

data/lib/http/check_factory.rb

@@ -1,47 +0,0 @@
-module Intrigue
-module Ident
-module Http
-class CheckFactory
-
-    #
-    # Register a new handler
-    #
-    def self.register(klass)
-      @checks = [] unless @checks
-      @checks << klass if klass
-    end
-
-    #
-    # Provide the full list of checks
-    #
-    def self.checks
-      @checks
-    end
-
-    #
-    # Provide the full list of checks
-    #
-    def self.generate_initial_checks(url)
-      @checks.map{ |x| x.new.generate_checks("#{url}") }.flatten.compact.select{|x| x[:require_product] == nil }
-    end
-
-    #
-    # Provide the full list of checks
-    #
-    def self.generate_checks_for_product(url,product)
-      @checks.map{ |x| x.new.generate_checks("#{url}") }.flatten.compact.select{|x| x[:require_product] == "#{product}" }
-    end
-
-
-   #
-    # Provide the full list of checks
-    #
-    def self.configuration_checks
-      @checks.map{ |x| x.new.generate_checks("") }.flatten.compact.select{|x| x[:type] == "content" }
-    end
-
-
-end
-end
-end
-end

data/lib/http/content.rb

@@ -1,45 +0,0 @@
-module Intrigue
-  module Ident
-    module Content
-      module Http
-
-      def _body(content)
-        _body_rendered(content) || _body_raw(content)
-      end
-
-      def _body_raw(content)  
-        content["details"]["hidden_response_data"]
-      end
-
-      def _body_rendered(content)
-        content["details"]["hidden_response_data_rendered"] 
-      end
-
-      def _body_raw_checksum(content)
-        Digest::MD5.hexdigest("#{_body_raw(content)}")
-      end
-
-      def _body_rendered_checksum(content)
-        Digest::MD5.hexdigest("#{_body_rendered(content)}")
-      end
-
-      def _generator(content)
-        content["details"]["generator"]
-      end
-
-      def _title(content)
-        content["details"]["title"]
-      end
-
-      def _headers(content)
-        (content["details"]["headers"] || []).join("\n")
-      end
-
-      def _cookies(content)
-        content["details"]["cookies"]
-      end
-
-    end
-  end
-end
-end

data/lib/http/http.rb

@@ -1,463 +0,0 @@
-# encoding: utf-8
-
-module Intrigue
-module Ident
-module Http
-
-  require_relative 'browser'
-  include Intrigue::Ident::HttpBrowser
-
-  # gives us the recog http matchers 
-  include Intrigue::Ident::RecogWrapper::Http 
-
-  # Used by intrigue-core... note that this will currently fail unless
-  def generate_http_requests_and_check(url, opts={})
-
-    dom_checks = opts[:enable_browser] || false
-    debug = opts[:debug] || false
-    only_base = opts[:'only-check-base-url']
-
-    # gather all fingeprints for each product
-    # this will look like an array of checks, each with a uri and a set of checks
-    initial_checks = Intrigue::Ident::Http::CheckFactory.generate_initial_checks("#{url}")
-
-    ##### 
-    ##### Sanity check!
-    #####
-    failing_checks = initial_checks.select{|x| x if !x[:paths] }
-    if failing_checks.compact.count > 0
-      puts "FATAL! Unable to continue, the following checks are invalid, missing a path!"
-      puts failing_checks.inspect
-      return
-    end
-
-    ###
-    ### Initial Checks
-    ###
-
-    # In order to ensure we check all urls associated with a check, we need to
-    # group them up by each path, which is annoying because they're stored in
-    # an array on each check. This line handles that. (take all the checks []
-    # with each of their paths [], flatten and group by them
-    initial_checks_by_path = initial_checks.map{|c| c[:paths].map{ |p|
-      c.merge({:unique_path => p})} }.flatten
-
-    # now we have them organized by a single path, group them up so we only
-    # have to make a single request per unique path 
-    grouped_initial_checks = initial_checks_by_path.group_by{|x| x[:unique_path] }
-
-    # allow us to only select the base path (speeds things up)
-    if only_base
-      grouped_initial_checks = grouped_initial_checks.select{|x,y| x == url}
-    end
-
-    # Run'm!!!
-    initial_results = run_grouped_http_checks(url, grouped_initial_checks, dom_checks, debug)
-
-    ###
-    ### APPLY THE RECOG (ONLY FIRST PAGE)!
-    ###
-    # now run recog against the current grab
-    recog_results = []
-    first_response = initial_results["responses"].first
-    if first_response 
-      server_headers = first_response[:response_headers].select{|x| x =~ /^server:.*$/i }
-      if server_headers.count > 0 
-        recog_results << recog_match_http_server_banner(server_headers.first)
-      end
-
-      cookies_headers = first_response[:response_headers].select{|x| x =~ /^set-cookie:.*$/i }
-      if cookies_headers.count > 0 
-        recog_results << recog_match_http_cookies(cookies_headers.first)
-      end
-    end
-
-    ###
-    ### Follow-on Checks
-    ### 
-
-    ### Okay so, now we have a set of detected products, let's figure out our follown checks
-    followon_checks = []
-    detected_products = initial_results["fingerprint"].map{|x| x["product"] }.uniq
-    detected_products.each do |prod|
-      followon_checks.concat(Intrigue::Ident::Http::CheckFactory.generate_checks_for_product("#{url}", prod))
-      #puts "Getting checks for product: #{prod} ... #{followon_checks.count}" if debug
-    end
-    
-    # group them up by path (there can be multiple paths)
-    followon_checks_by_path = followon_checks.map{|c| c[:paths].map{ |p|
-      c.merge({:unique_path => p})} }.flatten
-
-    # group'm as needed to run the checks
-    grouped_followon_checks = followon_checks_by_path.group_by{|x| x[:unique_path] }
-    
-    # allow us to only select the base path (speeds things up)
-    if only_base
-      grouped_followon_checks = grouped_followon_checks.select{|x,y| x == url}
-    end
-
-    ### OKAY NOW WE HAVE a set of output that we can run product-specific checks on, run'm
-    if grouped_followon_checks
-      followon_results = run_grouped_http_checks(url, grouped_followon_checks, dom_checks, debug)
-    else
-      followon_results = {
-        "fingerprint" => [], 
-        "content" => [],
-        "responses" => [],
-        "check_count" => []
-      }
-    end
-  
-    ###
-    ### Generate output
-    ###
-    out = {
-      "url" => initial_results["url"], # same
-      "fingerprint" => (initial_results["fingerprint"] + followon_results["fingerprint"] + recog_results.flatten).uniq,
-      "content" => initial_results["content"].concat(followon_results["content"]),
-      "responses" => initial_results["responses"].concat(followon_results["responses"]),
-      "initial_checks" => initial_results["check_count"],
-      "followon_checks" => followon_results["check_count"]
-    }
-
-  out 
-  end
-
-  private
-
-  def run_grouped_http_checks(url, grouped_generated_checks, dom_checks, debug)
-
-    # shove results into an array
-    results = []
-
-    # keep an array of the request / response details
-    responses = []
-
-    # keep track of timeouts
-    timeout_count = 0
-
-    # call the check on each uri
-    grouped_generated_checks.each do |ggc|
-
-      target_url = ggc.first
-
-      if timeout_count > 2
-        puts "Skipping #{target_url}, too many timeouts" if debug
-        next 
-      end
-
-      # get the response using a normal http request
-      # TODO - collect redirects here
-      puts "Getting #{target_url}" if debug
-      response_hash = ident_http_request :get, "#{target_url}"
-    
-      if response_hash[:timeout]
-        puts "ERROR timed out on #{target_url}" if debug
-        timeout_count += 1
-      end 
-
-      responses << response_hash
-
-      # Only if we are running browser checks
-      if dom_checks
-        # get the dom via a browser
-        if ggc.last.map{|c| c[:match_type] }.include?(:content_dom)
-          #puts "We have a check for #{target_url} that requires the DOM, firing a browser"
-          session = ident_create_browser_session
-          browser_response = ident_capture_document(session,"#{target_url}")
-
-          # save the response to our list of responses
-          # TODO - collect redirects here
-          # https://michaeltroutt.com/using-headless-chrome-to-find-link-redirects/
-          responses << browser_response
-
-          ident_destroy_browser_session session
-        end
-      end
-
-      puts "matching" if debug
-
-      # Go ahead and match it up if we got a response!
-      if response_hash || browser_response
-        
-        # call each check, collecting the product if it's a match
-        ###
-        ### APPLY THE IDENT!
-        ###
-        ggc.last.each do |check|
-
-          # if we have a check that should match the dom, run it
-          if (check[:match_type] == :content_dom)
-            results << match_browser_response_hash(check,browser_response) if dom_checks
-          else #otherwise use the normal flow
-            results << match_http_response_hash(check,response_hash)
-          end
-        end
-
-      end
-    end
-
-    return nil unless results
-
-    # Return all matches, minus the nils (non-matches), and grouped by check type
-    out = results.compact.group_by{|x| x["type"] }
-
-    # make sure we have an empty fingerprints array if we didnt' have any Matches
-    out["check_count"] = grouped_generated_checks.map{|x| {"url" => x.first, "count" => x.last.count } }
-    out["fingerprint"] = [] unless out["fingerprint"]
-    out["content"] = [] unless out["content"]
-
-    # only return unique results
-    out["fingerprint"] = out["fingerprint"].uniq
-    out["content"] = out["content"].uniq
-    out["url"] = url
-
-    # attach the responses
-    out["responses"] = responses
-    
-  out
-  end
-
-
-
-
-  #require_relative 'content_helpers'
-  #include Intrigue::Ident::Content::HttpHelpers
-
-  def ident_encode(string)
-    string.force_encoding('ISO-8859-1').encode('UTF-8')
-  end
-
-  ###
-  ### XXX - significant updates made to zlib, determine whether to
-  ### move this over to RestClient: https://github.com/ruby/ruby/commit/3cf7d1b57e3622430065f6a6ce8cbd5548d3d894
-  ###
-  def ident_http_request(method, uri_string, credentials=nil, headers={}, data=nil, limit = 3, open_timeout=15, read_timeout=15)
-
-    response = nil
-    begin
-
-      # set user agent
-      user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.73 Safari/537.36"
-      headers["User-Agent"] = user_agent
-
-      attempts=0
-      max_attempts=limit
-      found = false
-      timeout = false
-
-      uri = URI.parse uri_string
-
-      # keep track of redirects
-      response_urls = ["#{uri}"]
-
-      unless uri
-        _log error "Unable to parse URI from: #{uri_string}"
-        return
-      end
-
-      until( found || attempts >= max_attempts)
-
-        attempts+=1
-
-        if $global_config
-          if $global_config.config["http_proxy"]
-            proxy_config = $global_config.config["http_proxy"]
-            proxy_addr = $global_config.config["http_proxy"]["host"]
-            proxy_port = $global_config.config["http_proxy"]["port"]
-            proxy_user = $global_config.config["http_proxy"]["user"]
-            proxy_pass = $global_config.config["http_proxy"]["pass"]
-          end
-        end
-
-        # set options
-        opts = {}
-        if uri.instance_of? URI::HTTPS
-          opts[:use_ssl] = true
-          opts[:verify_mode] = OpenSSL::SSL::VERIFY_NONE
-        end
-
-        http = Net::HTTP.start(uri.host, uri.port, proxy_addr, proxy_port, opts)
-        http.open_timeout = open_timeout
-        http.read_timeout = read_timeout
-
-        path = "#{uri.path}"
-        path = "/" if path==""
-
-        # add in the query parameters
-        if uri.query
-          path += "?#{uri.query}"
-        end
-
-        ### ALLOW DIFFERENT VERBS HERE
-        if method == :get
-          request = Net::HTTP::Get.new(uri)
-        elsif method == :post
-          # see: https://coderwall.com/p/c-mu-a/http-posts-in-ruby
-          request = Net::HTTP::Post.new(uri)
-          request.body = data
-        elsif method == :head
-          request = Net::HTTP::Head.new(uri)
-        elsif method == :propfind
-          request = Net::HTTP::Propfind.new(uri.request_uri)
-          request.body = "Here's the body." # Set your body (data)
-          request["Depth"] = "1" # Set your headers: one header per line.
-        elsif method == :options
-          request = Net::HTTP::Options.new(uri.request_uri)
-        elsif method == :trace
-          request = Net::HTTP::Trace.new(uri.request_uri)
-          request.body = "blah blah"
-        end
-        ### END VERBS
-
-        # set the headers
-        headers.each do |k,v|
-          request[k] = v
-        end
-
-        # handle credentials
-        if credentials
-          request.basic_auth(credentials[:username],credentials[:password])
-        end
-
-        # USE THIS TO PRINT HTTP REQUEST
-        #request.each_header{|h| puts "#{h}: #{request[h]}" }
-        # END USE THIS TO PRINT HTTP REQUEST
-
-        # get the response
-        response = http.request(request)
-
-        ###
-        ### Handle redirects 
-        ### 
-        location_header = response.header['location'] || response.header['Location'] 
-        if location_header != nil
-          # location header redirect
-          #puts "Following redirect: #{location_header}"
-
-          newuri=URI.parse(location_header)
-
-          # handle relative uri 
-          if(newuri.relative?)
-            newuri=URI.parse("#{uri}#{location_header}")
-          end
-          
-          response_urls << ident_encode(newuri.to_s)
-          uri=newuri
-
-        elsif response.body =~ /META HTTP-EQUIV=\"?Refresh/i # meta refresh
-          # meta refresh redirect
-
-          # get the URL 
-          metaurl = URI.parse(response.body.scan(/META HTTP-EQUIV=Refresh CONTENT=.*; URL=(.*)"/i).first)
-          
-          if metaurl
-            newuri = metaurl.first  
-          else # unable to parse 
-            puts "ERROR Unable to parse redirection!!"
-            found = true 
-            break 
-          end
-          
-          # handle relative uri 
-          if(newuri.relative?)
-            newuri=URI.parse("#{uri}/#{newuri}")
-          end
-          
-          response_urls << ident_encode(newuri.to_s)
-          uri=newuri
-
-        else
-
-          found = true
-          break
-
-        end #end redirect handling
-
-      end #until
-
-      ###
-      ### Done Handling Redirects, proactively set final_url
-      ###
-      final_url = uri.to_s
-
-    ### TODO - create a global $debug config option
-    
-    #rescue ArgumentError => e
-      #puts "Unable to connect #{uri}: #{e}"
-    rescue Net::OpenTimeout => e
-      #puts "Unable to connect #{uri}: #{e}"
-      timeout = true
-    rescue Net::ReadTimeout => e
-      #puts "Unable to connect #{uri}: #{e}"
-      timeout = true
-    rescue Errno::ENETDOWN => e
-      #puts "Unable to connect #{uri}: #{e}" 
-    rescue Errno::ETIMEDOUT => e
-      #puts "Unable to connect #{uri}: #{e}" 
-      timeout = true
-    rescue Errno::EINVAL => e
-      #puts "Unable to connect #{uri}: #{e}"
-    rescue Errno::ENETUNREACH => e
-      #puts "Unable to connect #{uri}: #{e}"
-    rescue Errno::EHOSTUNREACH => e
-      #puts "Unable to connect #{uri}: #{e}"
-    rescue URI::InvalidURIError => e
-      #
-      # XXX - This is an issue. We should catch this and ensure it's not
-      # due to an underscore / other acceptable character in the URI
-      # http://stackoverflow.com/questions/5208851/is-there-a-workaround-to-open-urls-containing-underscores-in-ruby
-      #
-      #puts "Unable to connect #{uri}: #{e}"
-    rescue OpenSSL::SSL::SSLError => e
-      #puts "Unable to connect #{uri}: #{e}" 
-    rescue Errno::ECONNREFUSED => e
-      #puts "Unable to connect #{uri}: #{e}" 
-    rescue Errno::ECONNRESET => e
-      #puts "Unable to connect #{uri}: #{e}" 
-    rescue Net::HTTPBadResponse => e
-      #puts "Unable to connect #{uri}: #{e}" 
-    rescue Zlib::BufError => e
-      #puts "Unable to connect #{uri}: #{e}" 
-    rescue Zlib::DataError => e # "incorrect header check - may be specific to ruby 2.0"
-      #puts "Unable to connect #{uri}: #{e}" 
-    rescue EOFError => e
-      #puts "Unable to connect #{uri}: #{e}" 
-    rescue SocketError => e
-      #puts "Unable to connect #{uri}: #{e}" 
-    rescue Encoding::InvalidByteSequenceError => e
-      #puts "Encoding issue #{uri}: #{e}" 
-    rescue Encoding::UndefinedConversionError => e
-      #puts "Encoding issue #{uri}: #{e}" 
-    end
-
-    # generate our output
-    out = {
-      :timeout => timeout,
-      :start_url => uri_string,
-      :final_url => final_url,
-      :request_type => :ruby,
-      :request_method => method,
-      :request_credentials => credentials,
-      :request_headers => headers,
-      :request_data => data,
-      :request_attempts_limit => limit,
-      :request_attempts_used => attempts,
-      :request_user_agent => user_agent,
-      :request_proxy => proxy_config,
-      :response_urls => response_urls,
-      :response_object => response
-    }
-
-    # verify we have a response before adding these
-    if response
-      out[:response_headers] = response.each_header.map{|x| ident_encode "#{x}: #{response[x]}" }
-      out[:response_body] = ident_encode(response.body)
-    end
-
-    out
-  end
-
-end
-end
-end