http_utilities 1.0.1

data/lib/http_utilities/http/adapters/curb.rb

@@ -0,0 +1,107 @@
+require 'uri'
+
+module HttpUtilities
+  module Http
+    module Adapters
+      module Curb
+
+        def post_and_retrieve_content_using_curl(url, data, options = {})
+          request   =   self.set_curl_options(url, options)
+          response  =  nil
+
+          if (request.interface && data)
+            if (data.is_a?(Hash))
+              data = data.map { |key, value| Curl::PostField.content(key.to_s, value.to_s) }
+            end
+
+            request.interface.http_post(data)
+            response = request.interface.body_str
+            response = HttpUtilities::Http::Response.new(response, request, options)
+          end
+
+          return response
+        end
+
+        def retrieve_curl_content(url, options = {})
+          request     =   self.set_curl_options(url, options)
+          response    =   nil
+
+          begin
+            success   =   request.interface.perform
+            response  =   request.interface.body_str
+            response  =   HttpUtilities::Http::Response.new(response, request, options)
+
+          rescue Exception => e
+            puts "\n\n#{Time.now}: IMPORTANT! Error occurred while trying to retrieve content from url #{url} and parse it. Error: #{e.message}. Error Class: #{e.class}"
+          end
+
+          return response
+        end
+
+        def go_to_url(url, options = {})
+          success = false
+
+          request = self.set_curl_options(url, options)
+
+          success = request.interface.perform
+          success = (success && curl.response_code.eql?(200))
+
+          return success
+        end
+
+        def set_curl_options(url, options = {})
+          options               =   options.clone()
+          
+          request               =   HttpUtilities::Http::Request.new
+          request.set_proxy_options(options)
+
+          accept_cookies        =   options.delete(:accept_cookies) { |e| false }
+          timeout               =   options.delete(:timeout) { |e| 120 }
+          maximum_redirects     =   options.delete(:maximum_redirects) { |e| 10 }
+          disable_auth          =   options.delete(:disable_auth) { |e| false }
+          accept_content_type   =   options.delete(:accept_content_type) { |e| false }
+          content_type          =   options.delete(:content_type) { |e| false }
+          cookie_file           =   nil
+
+          curl = Curl::Easy.new(url) do |c|
+            c.headers ||= {}
+            c.headers["User-Agent"]     =   c.useragent = request.user_agent
+            c.headers["Accept"]         =   accept_content_type if (accept_content_type)
+            c.headers["Content-Type"]   =   content_type if (content_type)
+            c.timeout                   =   timeout
+            c.ssl_verify_host           =   false
+            c.follow_location           =   true
+            c.max_redirects             =   maximum_redirects
+
+            if (disable_auth)
+              c.http_auth_types   = nil
+              c.proxy_auth_types  = nil
+              c.unrestricted_auth = false
+            end
+          end
+
+          if (accept_cookies)
+            FileUtils.mkdir_p File.join(Rails.root, "tmp/cookies")
+            identifier = Time.now.to_date.to_s(:db).gsub("-", "_").gsub("\s", "_").gsub(":", "_")
+            cookie_file = File.join(Rails.root, "tmp/cookies", "cookies_#{identifier}.txt")
+
+            curl.enable_cookies   =   true
+            curl.cookiejar        =   cookie_file
+            curl.cookiefile       =   cookie_file
+          end
+
+          if (request.proxy[:host] && request.proxy[:port])
+            curl.proxy_url    =   ::Proxy.format_proxy_address(request.proxy[:host], request.proxy[:port], false)
+            curl.proxy_type   =   5 if (request.proxy[:protocol] && request.proxy[:protocol].downcase.eql?('socks5'))
+            curl.proxypwd     =   ::Proxy.format_proxy_credentials(request.proxy[:username], request.proxy[:password]) if (request.proxy[:username] && request.proxy[:password])
+          end
+          
+          request.interface = curl
+
+          return request
+        end
+
+      end
+    end
+  end
+end

data/lib/http_utilities/http/adapters/net_http.rb

@@ -0,0 +1,130 @@
+require 'open-uri'
+require 'net/http'
+require 'uri'
+
+module HttpUtilities
+  module Http
+    module Adapters
+      module NetHttp
+
+        def post_and_retrieve_content_using_net_http(url, data, options = {})
+          uri             =   URI.parse(url)
+          request         =   set_net_http_options(uri, options)
+          response        =   nil
+
+          opts            =   options.clone()
+          content_type    =   opts.delete(:content_type) { |e| nil }
+
+          if (request.interface && data)
+            data = (data.is_a?(Hash)) ? generate_request_params(data) : data
+
+            request.interface.start do |http|
+              headers = {}
+              headers["User-Agent"]     =   request.user_agent
+              headers["Content-Type"]   =   content_type if (content_type)
+
+              http.post(uri.request_uri, data, headers) do |response_data|
+                response  =   response_data
+              end
+              
+              response    =   HttpUtilities::Http::Response.new(response, request, options)
+            end
+          end
+
+          return response
+        end
+
+        def set_net_http_options(uri, options = {})
+          request             =   HttpUtilities::Http::Request.new
+          request.set_proxy_options(options)
+          request.interface   =   Net::HTTP.new(uri.host, uri.port, request.proxy[:host], request.proxy[:port])
+
+          return request
+        end
+
+        def retrieve_net_http_content(url, options = {})
+          uri         =   URI.parse(url)
+          request     =   set_net_http_options(uri, options)
+          return perform_net_http_request(request, uri, options)
+        end
+
+        def perform_net_http_request(request_or_url, uri = nil, options = {}, redirect_count = 0, max_redirects = 5)
+          request   =   nil
+          response  =   nil
+
+          if (request_or_url)
+            opts              =   (options.is_a?(Hash)) ? options.clone() : {}
+            retries           =   opts.delete(:retries) { |e| 3 }
+            force_encoding    =   opts.delete(:force_encoding) { |e| false }
+            cookies           =   opts.delete(:cookies) { |e| nil }
+            timeout           =   opts.delete(:timeout) { |e| 30 }
+
+            if (request_or_url.is_a?(String))
+              uri       =   URI.parse(request_or_url)
+              request   =   self.set_net_http_options(uri, options)
+            else
+              request   =   request_or_url
+            end
+
+            if (uri && uri.respond_to?(:request_uri) && uri.request_uri)
+              headers           =   {"User-Agent" => request.user_agent}
+              headers           =   set_cookies(headers, cookies)
+              request_uri       =   uri.request_uri
+              http_request      =   Net::HTTP::Get.new(request_uri, headers)
+              
+              begin
+                request.interface.start do |http|
+                  http.read_timeout   =   timeout
+                  response            =   http.request(http_request)
+                end
+
+              rescue Errno::ETIMEDOUT, Errno::ECONNREFUSED, Errno::ENETUNREACH, Errno::ECONNRESET, Timeout::Error, Net::HTTPUnauthorized, Net::HTTPForbidden => error
+                log(:error, "[HttpUtilities::Http::Client] - Error occurred while trying to fetch url '#{uri.request_uri}'. Error Class: #{error.class.name}. Error Message: #{error.message}")
+                retries -= 1
+                retry if (retries > 0)
+              end
+            end
+          end
+
+          if (response)
+            location = response['location']
+
+            if (!(response.code.to_s =~ /^30\d{1}/i).nil? && location && location.present?)
+              location            =   location.strip.downcase
+              redirect_count     +=   1
+              
+              if (redirect_count < max_redirects)
+                request.cookies   =   handle_cookies(response)
+                location          =   "http://#{uri.host}/#{location.gsub(/^\//i, "")}" if (uri && (location =~ /^http(s)?/i).nil?)
+                
+                log(:info, "[HttpUtilities::Http::Client] - Redirecting to location: #{location}.")
+                
+                options           =   options.merge(:cookies => request.cookies) if request.cookies
+                response          =   perform_net_http_request(location, uri, options, redirect_count, max_redirects)
+              end
+            end
+
+            request.cookies   =   handle_cookies(response)
+            response          =   set_response(response)
+            response          =   HttpUtilities::Http::Response.new(response, request, options)
+          end
+
+          return response
+        end
+        
+        def set_response(response)
+          if (response.is_a?(String))
+            response = response
+          elsif (response.is_a?(Net::HTTPResponse))
+            response = response.body
+          elsif (response.is_a?(HttpUtilities::Http::Response))
+            response = response.body
+          end
+          
+          return response
+        end
+
+      end
+    end
+  end
+end

data/lib/http_utilities/http/adapters/open_uri.rb

@@ -0,0 +1,46 @@
+require 'open-uri'
+require 'uri'
+
+module HttpUtilities
+  module Http
+    module Adapters
+      module OpenUri
+
+        def retrieve_open_uri_content(url, options = {}, retries = 0, max_retries = 3)
+          response  =   nil
+
+          options   =   options.clone()
+          request   =   HttpUtilities::Http::Request.new
+          request.set_proxy_options(options)
+
+          open_uri_options = {"UserAgent" => request.user_agent}
+          open_uri_options[:read_timeout] = options.delete(:timeout) { |e| 120 }
+
+          if (request.proxy[:host] && request.proxy[:port])
+            proxy_address = Proxy.format_proxy_address(request.proxy[:host], request.proxy[:port], true)
+            open_uri_options[:proxy] = proxy_address
+
+            if (request.proxy[:username] && request.proxy[:password])
+              open_uri_options[:proxy_http_basic_authentication] = [proxy_address, request.proxy[:username], request.proxy[:password]]
+            end
+          end
+
+          connection = nil
+          while (connection.nil? && retries < max_retries)
+            connection = open(url, open_uri_options)
+            retries += 1
+          end
+
+          if (connection)
+            connection.rewind
+            response  =   connection.readlines.join("\n")
+            response  =   HttpUtilities::Http::Response.new(response, request)
+          end
+
+          return response
+        end
+
+      end
+    end
+  end
+end

data/lib/http_utilities/http/client.rb

@@ -0,0 +1,22 @@
+# -*- encoding : utf-8 -*-
+require 'open-uri'
+require 'net/http'
+require 'uri'
+require 'cgi'
+
+module HttpUtilities
+  module Http
+    class Client
+      include HttpUtilities::Http::Cookies
+      include HttpUtilities::Http::Url
+      include HttpUtilities::Http::Get
+      include HttpUtilities::Http::Post
+      include HttpUtilities::Http::Logger
+
+      include HttpUtilities::Http::Adapters::NetHttp
+      include HttpUtilities::Http::Adapters::OpenUri
+      include HttpUtilities::Http::Adapters::Curb
+    end
+  end
+end
+

data/lib/http_utilities/http/cookies.rb

@@ -0,0 +1,49 @@
+module HttpUtilities
+  module Http
+    module Cookies
+
+      def handle_cookies(response)
+        cookies = nil
+
+        if (response && response.is_a?(Net::HTTPResponse))
+          cookie_fields = response.get_fields('Set-Cookie')
+          
+          if (cookie_fields && cookie_fields.any?)
+            cookies = []
+            cookie_fields.each do |cookie|
+              filtered_cookie = cookie.split('; ').first
+              cookies << filtered_cookie
+            end
+          end
+        
+        elsif (response && response.is_a?(HttpUtilities::Http::Response))
+          cookies = (response.request && response.request.cookies) ? response.request.cookies : nil
+        end
+
+        return cookies
+      end
+
+      def format_cookies(cookies)
+        cookie_string = ""
+        cookies.each {|cookie| cookie_string += "#{cookie}; "}
+
+        return cookie_string
+      end
+
+      def set_cookies(headers, cookies)
+        if (cookies && cookies.any?)
+          cookie_string     =   (cookies && cookies.is_a?(Array)) ? format_cookies(cookies) : nil
+
+          if (cookie_string)
+            cookie_hash     =   {'cookie' => cookie_string}
+            headers         =   (headers && !headers.empty?) ? headers.merge(cookie_hash) : cookie_hash
+          end
+        end
+
+        return headers
+      end
+
+    end
+  end
+end
+

data/lib/http_utilities/http/format.rb

@@ -0,0 +1,26 @@
+# -*- encoding : utf-8 -*-
+require 'nokogiri'
+
+module HttpUtilities
+  module Http
+    module Format
+      
+      def as_html
+        self.parsed_body = (self.body && self.body != "") ? Nokogiri::HTML(self.body, nil, "utf-8") : nil
+      end
+
+      def as_xml
+        self.parsed_body = (self.body && self.body != "") ? Nokogiri::XML(self.body, nil, "utf-8") : nil
+      end
+      
+      def as_multi_xml
+        self.parsed_body = (self.body && self.body != "") ? MultiXml.parse(self.body) : nil
+      end
+
+      def as_json
+        self.parsed_body = (self.body && self.body != "") ? self.body.to_json : nil
+      end
+      
+    end
+  end
+end

data/lib/http_utilities/http/get.rb

@@ -0,0 +1,67 @@
+module HttpUtilities
+  module Http
+    module Get
+
+      def retrieve_raw_content(url, options = {})
+        return retrieve_content_from_url(url, options)
+      end
+
+      def retrieve_raw_xml(url, options = {})
+        return retrieve_content_from_url(url, options)
+      end
+
+      def retrieve_parsed_xml(url, options = {})
+        options.merge!({:force_encoding => true, :format => :xml})
+        return retrieve_content_from_url(url, options)
+      end
+
+      def retrieve_parsed_html(url, options = {})
+        options.merge!({:force_encoding => true, :format => :html})
+        return retrieve_content_from_url(url, options)
+      end
+
+      def retrieve_parsed_html_and_fallback_to_proxies(url, options = {})
+        options.merge!({:force_encoding => true, :format => :html})
+        return retrieve_raw_content_and_fallback_to_proxies(url, options)
+      end
+
+      def retrieve_parsed_xml_and_fallback_to_proxies(url, options = {})
+        options.merge!({:force_encoding => true, :format => :xml})
+        return retrieve_raw_content_and_fallback_to_proxies(url, options)
+      end
+
+      def retrieve_raw_content_and_fallback_to_proxies(url, options = {})
+        retries = 0
+        max_retries = options.delete(:maximum_retrieval_retries) { |e| 5 }
+        options.merge!({:force_encoding => true})
+
+        response  =   retrieve_content_from_url(url, options)
+
+        while (!response && retries < max_retries) do
+          options.merge!({:use_proxy => true})
+          response  =   retrieve_content_from_url(url, options)
+          retries += 1
+        end
+
+        return response
+      end
+
+      def retrieve_content_from_url(url, options = {})
+        response        =   nil
+        method          =   options[:method] || :net_http
+
+        if (method.eql?(:open_uri))
+          response = retrieve_open_uri_content(url, options)
+        elsif (method.eql?(:net_http))
+          response = retrieve_net_http_content(url, options)
+        elsif (method.eql?(:curl))
+          response = retrieve_curl_content(url, options)
+        end
+
+        return response
+      end
+
+    end
+  end
+end
+

data/lib/http_utilities/http/logger.rb

@@ -0,0 +1,11 @@
+module HttpUtilities
+  module Http
+    module Logger
+      
+      def log(level, message)
+        (defined?(Rails) && Rails.logger) ? Rails.logger.send(level, message) : puts(message)
+      end
+      
+    end
+  end
+end

data/lib/http_utilities/http/mechanize/client.rb

@@ -0,0 +1,197 @@
+# -*- encoding : utf-8 -*-
+require 'open-uri'
+require 'net/http'
+require 'uri'
+require 'cgi'
+require 'mechanize'
+
+module HttpUtilities
+  module Http
+    module Mechanize
+
+      class Client
+        attr_accessor :agent, :proxy, :user_agent
+        
+        include HttpUtilities::Http::Logger
+        include HttpUtilities::Http::Url
+        include HttpUtilities::Http::ProxySupport
+        include HttpUtilities::Http::UserAgent
+        
+        def initialize(options = {})
+          init_agent(options)
+        end
+        
+        def init_agent(options = {})
+          verbose                   =   options.fetch(:verbose, false)
+          logger                    =   options.fetch(:logger, STDOUT)
+          
+          self.agent                =   ::Mechanize.new
+          self.agent.log            =   ::Logger.new(logger) if (verbose)
+          
+          self.set_proxy_options(options)
+          self.agent.set_proxy(self.proxy[:host], self.proxy[:port], self.proxy[:username], self.proxy[:password]) if (self.proxy[:host] && self.proxy[:port])
+          
+          self.set_user_agent
+          (self.user_agent) ? self.agent.user_agent = self.user_agent : self.agent.user_agent_alias = 'Mac Safari'
+          
+          timeout                   =   options.fetch(:timeout, 300)
+          self.agent.open_timeout   =   self.agent.read_timeout = timeout if (timeout)
+        end
+        
+        def reset_agent(options = {})
+          self.agent, self.proxy, self.user_agent = nil
+          init_agent(options)
+        end
+        
+        def open_url(url, options = {}, retries = 3)
+          page = nil
+
+          begin
+            page = self.agent.get(url)
+
+          rescue Net::HTTPNotFound, ::Mechanize::ResponseCodeError => error
+            log(:error, "[HttpUtilities::Http::Mechanize::Client] - Response Code Error occurred for url #{url}. Error class: #{error.class.name}. Error message: #{error.message}")
+            
+            if (retries > 0)
+              reset_agent(options)
+              retries -= 1
+              retry
+            end
+            
+          rescue Errno::ECONNREFUSED, Errno::ETIMEDOUT, Errno::ECONNRESET, Timeout::Error, Net::HTTPUnauthorized, Net::HTTPForbidden, StandardError => connection_error
+            log(:error, "[HttpUtilities::Http::Mechanize::Client] - Error occurred. Error class: #{connection_error.class.name}. Message: #{connection_error.message}")
+
+            if (retries > 0)
+              reset_agent(options)
+              retries -= 1
+              retry
+            end
+          end
+
+          return page
+        end
+        
+        def get_page(url_or_page, options = {})
+          page        =   nil
+          
+          if (url_or_page.is_a?(String))
+            page      =   open_url(url_or_page, options)
+          else
+            page      =   url_or_page
+          end
+          
+          page        =   (page && page.is_a?(::Mechanize::Page)) ? page : nil #Occasionally proxies will yield Mechanize::File instead of a proper page
+          
+          return page
+        end
+        
+        def get_form(url_or_page, form_identifier = {}, options = {})
+          form        =   nil
+          index       =   form_identifier.delete(:index) { |el| 0 }
+          page        =   (url_or_page.is_a?(String)) ? get_page(url_or_page, options) : url_or_page
+          
+          if (page)
+            if (form_identifier.empty?)
+              form    =   page.forms[index]
+            else
+              forms   =   page.forms_with(form_identifier)
+              form    =   (forms && forms.any?) ? forms[index] : nil
+            end
+          end
+          
+          return form
+        end
+
+        def set_form_and_submit(url_or_page, form_identifier = {}, submit_identifier = :first, fields = {}, options = {}, retries = 3)
+          should_reset_radio_buttons  =   options.fetch(:should_reset_radio_buttons, false)
+          page                        =   get_page(url_or_page, options)
+          form                        =   page ? get_form(page, form_identifier) : nil
+          response_page               =   nil
+
+          if (form)
+            form.action               =     "#{url_or_page}#{form.action}"  if (url_or_page.is_a?(String) && form.action.starts_with?("#"))
+            form                      =     reset_radio_buttons(form)       if (should_reset_radio_buttons)
+            form                      =     set_form_fields(form, fields)
+            button                    =     (submit_identifier.nil? || submit_identifier.eql?(:first)) ? form.buttons.first : form.button_with(submit_identifier)
+
+            begin
+              response_page           =     form.submit(button)
+            rescue Exception => e
+              log(:error, "[HttpUtilities::Http::Mechanize::Client] - Failed to submit form. Error: #{e.class.name} - #{e.message}.")
+            end
+
+          elsif (!form && retries > 0)
+            log(:info, "[HttpUtilities::Http::Mechanize::Client] - Couldn't find page or form with identifier #{form_identifier.inspect}")
+            retries -= 1
+            reset_agent(options)
+            set_form_and_submit(url_or_page, form_identifier, submit_identifier, fields, options, retries)
+          end
+
+          return response_page
+        end
+
+        def reset_radio_buttons(form)
+          radio_buttons = form.radiobuttons
+          
+          radio_buttons.each do |radio_button|
+            radio_button.checked = false
+          end if (form && radio_buttons && radio_buttons.any?)
+
+          return form
+        end
+
+        def set_form_fields(form, fields)
+          if (form && fields && !fields.empty?)
+            fields.each do |key, value|
+              form = set_form_field(form, key, value)
+            end
+          end
+
+          return form
+        end
+
+        def set_form_field(form, key, value)
+          type          =   value.fetch(:type, :input)
+          identifier    =   value.fetch(:identifier, :name)
+          
+          if (type.eql?(:input))
+            log(:info, "[HttpUtilities::Http::Mechanize::Client] - Setting text field #{key} to value #{value[:value]}.")
+            form.has_field?(key.to_s) ? form.field_with(identifier => key.to_s).value = value[:value].to_s : set_form_fields(form, value[:fallbacks])
+          
+          elsif (type.eql?(:checkbox))
+            log(:info, "[HttpUtilities::Http::Mechanize::Client] - Setting checkbox #{key} to checked: #{value[:checked]}.")
+            form.checkbox_with(identifier => key.to_s).checked        =   value[:checked]
+          
+          elsif (type.eql?(:radiobutton))
+            log(:info, "[HttpUtilities::Http::Mechanize::Client] - Setting radio button #{key} to checked: #{value[:checked]}.")
+            form.radiobutton_with(identifier => key.to_s).checked     =   value[:checked]
+          
+          elsif (type.eql?(:select))
+            log(:info, "[HttpUtilities::Http::Mechanize::Client] - Setting select/dropdown #{key} to value: #{value[:value]}.")
+            form.field_with(identifier => key.to_s).value             =   value[:value].to_s
+          
+          elsif (type.eql?(:file_upload))
+            log(:info, "[HttpUtilities::Http::Mechanize::Client] - Setting file upload #{key} to value #{value[:value]}.")
+            form.file_upload_with(identifier => key.to_s).file_name   =   value[:value].to_s
+          end
+
+          return form
+        end
+
+        def get_parser(page)
+          parser = nil
+
+          if (page.is_a?(::Mechanize::Page))
+            parser = page.parser
+          elsif (page.is_a?(::Mechanize::File))
+            parser = Nokogiri::HTML(page.body, nil, "utf-8")
+          end
+
+          return parser
+        end
+
+      end
+
+    end
+  end
+end

data/lib/http_utilities/http/post.rb

@@ -0,0 +1,32 @@
+module HttpUtilities
+  module Http
+    module Post
+
+      def post_and_retrieve_parsed_html(url, data, options = {})
+        options.merge!({:force_encoding => true, :format => :html})
+        return post_and_retrieve_content(url, data, options)
+      end
+
+      def post_and_retrieve_parsed_xml(url, data, options = {})
+        options.merge!({:force_encoding => true, :format => :xml})
+        return post_and_retrieve_content(url, data, options)
+      end
+
+      def post_and_retrieve_content(url, data, options = {})
+        response        =   nil
+        method          =   options[:method] || :net_http
+        response_only   =   options.delete(:response_only) { |e| true }
+
+        if (method.eql?(:net_http))
+          response = post_and_retrieve_content_using_net_http(url, data, options)
+        elsif (method.eql?(:curl))
+          response = post_and_retrieve_content_using_curl(url, data, options)
+        end
+
+        return response
+      end
+
+    end
+  end
+end
+