http_utilities 1.0.1

data/lib/http_utilities/http/proxy_support.rb

@@ -0,0 +1,88 @@
+require 'open-uri'
+require 'uri'
+
+module HttpUtilities
+  module Http
+    module ProxySupport
+
+      def set_proxy_options(options = {})
+        use_proxy                 =   options.fetch(:use_proxy, false)
+        specific_proxy            =   options.fetch(:proxy, nil)
+        proxy_username            =   options.fetch(:proxy_username, nil)
+        proxy_password            =   options.fetch(:proxy_password, nil)
+        proxy_credentials         =   options.fetch(:proxy_credentials, nil)
+        reset_proxy               =   options.fetch(:reset_proxy, true)
+
+        if (reset_proxy)
+          self.proxy              =   {}
+          self.proxy[:host]       =   options.fetch(:proxy_host, nil)
+          self.proxy[:port]       =   options.fetch(:proxy_port, nil)
+          self.proxy[:protocol]   =   options.fetch(:proxy_protocol, :http)
+          self.proxy[:type]       =   options.fetch(:proxy_type, :all)
+        end
+        
+        if (use_proxy || (specific_proxy && !self.using_proxy?))
+          if (specific_proxy && specific_proxy.is_a?(String))
+            specific_proxy        =   specific_proxy.gsub(/^http(s)?:\/\//i, "")
+            parts                 =   specific_proxy.split(":")
+
+            if (parts.size.eql?(2))
+              self.proxy[:host]   =   parts.first
+              self.proxy[:port]   =   parts.second.to_i
+            end
+
+          elsif (specific_proxy && specific_proxy.is_a?(Hash) && !specific_proxy.empty? && specific_proxy[:host] && specific_proxy[:port])
+            self.proxy            =   specific_proxy
+
+          elsif (proxy_model_defined?)
+            proxy_object = Proxy.get_random_proxy(self.proxy[:protocol], self.proxy[:type])
+            
+            #log(:info, "[HttpUtilities::Http::ProxySupport] - Randomized Proxy object: #{proxy_object.inspect}")
+
+            if (proxy_object)
+              self.proxy[:host]   =   proxy_object.host
+              self.proxy[:port]   =   proxy_object.port
+            end
+          end
+        end
+
+        set_proxy_credentials(proxy_username, proxy_password, proxy_credentials)
+      end
+
+      def set_proxy_credentials(proxy_username, proxy_password, proxy_credentials)
+        if (self.using_proxy? && (!self.proxy[:username] || !self.proxy[:password]))
+          if (proxy_username && proxy_password)
+            self.proxy[:username] = proxy_username
+            self.proxy[:password] = proxy_password
+
+          elsif (proxy_credentials)
+            if (proxy_credentials.is_a?(Hash))
+              self.proxy[:username] = proxy_credentials[:username]
+              self.proxy[:password] = proxy_credentials[:password]
+
+            elsif (proxy_credentials.is_a?(String))
+              parts = proxy_credentials.split(":")
+
+              if (parts && parts.any? && parts.size >= 2)
+                self.proxy[:username] = parts.first
+                self.proxy[:password] = parts.second
+              end
+            end
+          end
+        end
+      end
+      
+      def proxy_model_defined?
+        defined = Module.const_get("Proxy").is_a?(Class) rescue false
+        defined = (defined && Proxy.respond_to?(:get_random_proxy))
+        
+        return defined
+      end
+      
+      def using_proxy?
+        return (self.proxy[:host] && self.proxy[:port] && self.proxy[:port] > 0)
+      end
+
+    end
+  end
+end

data/lib/http_utilities/http/request.rb

@@ -0,0 +1,20 @@
+module HttpUtilities
+  module Http
+    class Request
+      include HttpUtilities::Http::Logger
+      include HttpUtilities::Http::ProxySupport
+      include HttpUtilities::Http::UserAgent
+      
+      attr_accessor :interface, :proxy, :cookies, :user_agent
+      
+      def initialize(interface = nil, proxy = {}, cookies = [])
+        self.interface  =   interface
+        self.proxy      =   proxy
+        self.cookies    =   cookies
+        
+        self.set_user_agent
+      end
+            
+    end
+  end
+end

data/lib/http_utilities/http/response.rb

@@ -0,0 +1,50 @@
+module HttpUtilities
+  module Http
+    class Response
+      include HttpUtilities::Http::Format
+      include HttpUtilities::Http::Logger
+
+      attr_accessor :body, :parsed_body, :page, :format, :request, :force_encoding
+
+      def initialize(body = nil, request = nil, options = {})
+        options               =   options.clone()
+
+        self.body             =   body
+        self.request          =   request
+
+        self.parsed_body      =   nil
+
+        self.format           =   options.delete(:format) { |e| nil }
+        self.force_encoding   =   options.delete(:force_encoding) { |e| true }
+
+        encode if (self.force_encoding)
+        parse_response
+      end
+
+      def encode
+        if (self.body)
+          begin
+            self.body = self.body.force_encoding('UTF-8').encode("UTF-8", :invalid => :replace, :undef => :replace, :replace => "")
+          rescue Exception => e
+            log(:error, "[HttpUtilities::Http::Format] - Failed to convert response with String#encode. Error: #{e.class.name}. Message: #{e.message}.")
+          end
+        end
+      end
+
+      def parse_response
+        self.send("as_#{self.format}".to_sym) if (self.body && self.format)
+      end
+
+      def set_page(page)
+        self.page = page
+
+        if (page && page.parser)
+          self.body         =   page.parser.content
+          self.parsed_body  =   page.parser
+        end
+      end
+
+    end
+  end
+end
+

data/lib/http_utilities/http/url.rb

@@ -0,0 +1,48 @@
+require 'open-uri'
+require 'uri'
+require 'cgi'
+
+module HttpUtilities
+  module Http
+    module Url
+      
+      def generate_request_url(params = {})
+        params.symbolize_keys!
+        url = params.delete(:url) { |e| "" }
+
+        sorted_params = params.sort
+        query_parts = []
+
+        sorted_params.each do |param_row|
+          param = encode_param(param_row.first)
+          value = encode_param(param_row.last)
+          query_parts << "#{param}=#{value}"
+        end
+
+        query = query_parts.join("&")
+        request = "#{url}?#{query}"
+        return request
+      end
+      
+      def generate_request_params(params)
+        sorted_params = params.sort
+        query_parts = []
+
+        sorted_params.each do |param_row|
+          param = param_row.first
+          value = param_row.last
+          query_parts << "#{param}=#{value}"
+        end
+
+        query = query_parts.join("&")
+        
+        return query
+      end
+
+      def encode_param(param)
+        return CGI.escape(param.to_s).to_s.gsub("+", "%20").gsub("%7E", "~") if (param)
+      end
+      
+    end
+  end
+end