http_utilities 1.1.2 → 1.1.5

data/lib/http_utilities/http/cookies.rb

@@ -1,49 +0,0 @@
-module HttpUtilities
-  module Http
-    module Cookies
-
-      def handle_cookies(response)
-        cookies = nil
-
-        if (response && response.is_a?(Net::HTTPResponse))
-          cookie_fields = response.get_fields('Set-Cookie')
-          
-          if (cookie_fields && cookie_fields.any?)
-            cookies = []
-            cookie_fields.each do |cookie|
-              filtered_cookie = cookie.split('; ').first
-              cookies << filtered_cookie
-            end
-          end
-        
-        elsif (response && response.is_a?(HttpUtilities::Http::Response))
-          cookies = (response.request && response.request.cookies) ? response.request.cookies : nil
-        end
-
-        return cookies
-      end
-
-      def format_cookies(cookies)
-        cookie_string = ""
-        cookies.each {|cookie| cookie_string += "#{cookie}; "}
-
-        return cookie_string
-      end
-
-      def set_cookies(headers, cookies)
-        if (cookies && cookies.any?)
-          cookie_string     =   (cookies && cookies.is_a?(Array)) ? format_cookies(cookies) : nil
-
-          if (cookie_string)
-            cookie_hash     =   {'cookie' => cookie_string}
-            headers         =   (headers && !headers.empty?) ? headers.merge(cookie_hash) : cookie_hash
-          end
-        end
-
-        return headers
-      end
-
-    end
-  end
-end
-

data/lib/http_utilities/http/format.rb

@@ -1,26 +0,0 @@
-# -*- encoding : utf-8 -*-
-require 'nokogiri'
-
-module HttpUtilities
-  module Http
-    module Format
-      
-      def as_html
-        self.parsed_body = (self.body && self.body != "") ? Nokogiri::HTML(self.body, nil, "utf-8") : nil
-      end
-
-      def as_xml
-        self.parsed_body = (self.body && self.body != "") ? Nokogiri::XML(self.body, nil, "utf-8") : nil
-      end
-      
-      def as_multi_xml
-        self.parsed_body = (self.body && self.body != "") ? MultiXml.parse(self.body) : nil
-      end
-
-      def as_json
-        self.parsed_body = (self.body && self.body != "") ? self.body.to_json : nil
-      end
-      
-    end
-  end
-end

data/lib/http_utilities/http/get.rb

@@ -1,67 +0,0 @@
-module HttpUtilities
-  module Http
-    module Get
-
-      def retrieve_raw_content(url, options = {})
-        return retrieve_content_from_url(url, options)
-      end
-
-      def retrieve_raw_xml(url, options = {})
-        return retrieve_content_from_url(url, options)
-      end
-
-      def retrieve_parsed_xml(url, options = {})
-        options.merge!({:force_encoding => true, :format => :xml})
-        return retrieve_content_from_url(url, options)
-      end
-
-      def retrieve_parsed_html(url, options = {})
-        options.merge!({:force_encoding => true, :format => :html})
-        return retrieve_content_from_url(url, options)
-      end
-
-      def retrieve_parsed_html_and_fallback_to_proxies(url, options = {})
-        options.merge!({:force_encoding => true, :format => :html})
-        return retrieve_raw_content_and_fallback_to_proxies(url, options)
-      end
-
-      def retrieve_parsed_xml_and_fallback_to_proxies(url, options = {})
-        options.merge!({:force_encoding => true, :format => :xml})
-        return retrieve_raw_content_and_fallback_to_proxies(url, options)
-      end
-
-      def retrieve_raw_content_and_fallback_to_proxies(url, options = {})
-        retries = 0
-        max_retries = options.delete(:maximum_retrieval_retries) { |e| 5 }
-        options.merge!({:force_encoding => true})
-
-        response  =   retrieve_content_from_url(url, options)
-
-        while (!response && retries < max_retries) do
-          options.merge!({:use_proxy => true})
-          response  =   retrieve_content_from_url(url, options)
-          retries += 1
-        end
-
-        return response
-      end
-
-      def retrieve_content_from_url(url, options = {})
-        response        =   nil
-        method          =   options[:method] || :net_http
-
-        if (method.eql?(:open_uri))
-          response = retrieve_open_uri_content(url, options)
-        elsif (method.eql?(:net_http))
-          response = retrieve_net_http_content(url, options)
-        elsif (method.eql?(:curl))
-          response = retrieve_curl_content(url, options)
-        end
-
-        return response
-      end
-
-    end
-  end
-end
-

data/lib/http_utilities/http/post.rb

@@ -1,32 +0,0 @@
-module HttpUtilities
-  module Http
-    module Post
-
-      def post_and_retrieve_parsed_html(url, data, options = {})
-        options.merge!({:force_encoding => true, :format => :html})
-        return post_and_retrieve_content(url, data, options)
-      end
-
-      def post_and_retrieve_parsed_xml(url, data, options = {})
-        options.merge!({:force_encoding => true, :format => :xml})
-        return post_and_retrieve_content(url, data, options)
-      end
-
-      def post_and_retrieve_content(url, data, options = {})
-        response        =   nil
-        method          =   options[:method] || :net_http
-        response_only   =   options.delete(:response_only) { |e| true }
-
-        if (method.eql?(:net_http))
-          response = post_and_retrieve_content_using_net_http(url, data, options)
-        elsif (method.eql?(:curl))
-          response = post_and_retrieve_content_using_curl(url, data, options)
-        end
-
-        return response
-      end
-
-    end
-  end
-end
-

data/lib/http_utilities/proxies/proxy_module.rb

@@ -1,89 +0,0 @@
-module HttpUtilities
-  module Proxies
-    module ProxyModule
-      
-      def self.included(base)
-        base.send :extend, ClassMethods
-        base.send :include, InstanceMethods
-      end
-
-      module ClassMethods
-        def should_be_checked(protocol: :all, proxy_type: :all, date: Time.now, limit: 10, maximum_failed_attempts: 10)
-          proxies     =   get_proxies_for_protocol_and_proxy_type(protocol, proxy_type)
-          proxies     =   proxies.where(["(last_checked_at IS NULL OR last_checked_at < ?)", date])
-          proxies     =   proxies.where(["failed_attempts <= ?", maximum_failed_attempts])
-          proxies     =   proxies.order("valid_proxy ASC, failed_attempts ASC, last_checked_at ASC")
-          proxies     =   proxies.limit(limit)
-          
-          return proxies
-        end
-        
-        def get_random_proxy(protocol: :all, proxy_type: :all, maximum_failed_attempts: nil)
-          proxies     =   get_proxies_for_protocol_and_proxy_type(protocol, proxy_type)
-          proxies     =   proxies.where(["valid_proxy = ? AND last_checked_at IS NOT NULL", true])
-          proxies     =   proxies.where(["failed_attempts <= ?", maximum_failed_attempts]) if maximum_failed_attempts
-          
-          order_clause = case ActiveRecord::Base.connection.class.name
-            when "ActiveRecord::ConnectionAdapters::MysqlAdapter", "ActiveRecord::ConnectionAdapters::Mysql2Adapter"
-              "RAND() DESC"
-            when "ActiveRecord::ConnectionAdapters::SQLite3Adapter"
-              "RANDOM() DESC"
-            else
-              "RAND() DESC"
-          end
-          
-          proxies     =   proxies.order(order_clause)
-          
-          proxy       =   nil
-          
-          uncached do
-            proxy     =   proxies.limit(1).first
-          end
-
-          return proxy
-        end
-        
-        def get_proxies_for_protocol_and_proxy_type(protocol, proxy_type)
-          proxies     =   ::Proxy.where(nil)
-          proxies     =   proxies.where(protocol: protocol)     if (protocol && !protocol.downcase.to_sym.eql?(:all))
-          proxies     =   proxies.where(proxy_type: proxy_type) if (proxy_type && !proxy_type.downcase.to_sym.eql?(:all))
-          
-          return proxies
-        end
-
-        def format_proxy_address(proxy_host, proxy_port = 80, include_http = false)
-          proxy_address = "#{proxy_host}:#{proxy_port}"
-          proxy_address.insert(0, "http://") if (include_http && !proxy_address.start_with?("http://"))
-          return proxy_address
-        end
-
-        def format_proxy_credentials(username, password)
-          return "#{username}:#{password}"
-        end
-      end
-
-      module InstanceMethods
-        def proxy_address(include_http = false)
-          return ::Proxy.format_proxy_address(self.host, self.port, include_http)
-        end
-        
-        def proxy_credentials
-          return ::Proxy.format_proxy_credentials(self.username, self.password)
-        end
-        
-        def socks_proxy_credentials
-          credentials     =   {}
-          
-          if (!self.username.empty? && !self.password.empty?)
-            credentials   =   {user: self.username, password: self.password}
-          elsif (!self.username.empty? && self.password.empty?)
-            credentials   =   {user: self.username}
-          end
-          
-          return credentials
-        end
-      end
-      
-    end
-  end
-end