http_utilities 1.1.2 → 1.1.5

data/lib/http_utilities.rb

@@ -1,35 +1,32 @@
 # -*- encoding : utf-8 -*-
 module HttpUtilities
-  VERSION = "1.1.2"
+  VERSION = "1.1.5"
   
   require File.join(File.dirname(__FILE__), 'http_utilities/railtie') if defined?(Rails)
 
   require File.join(File.dirname(__FILE__), 'http_utilities/http/proxy_support')
-  require File.join(File.dirname(__FILE__), 'http_utilities/http/cookies')
   require File.join(File.dirname(__FILE__), 'http_utilities/http/user_agent')
   require File.join(File.dirname(__FILE__), 'http_utilities/http/url')
-  require File.join(File.dirname(__FILE__), 'http_utilities/http/format')
-  require File.join(File.dirname(__FILE__), 'http_utilities/http/get')
-  require File.join(File.dirname(__FILE__), 'http_utilities/http/post')
   require File.join(File.dirname(__FILE__), 'http_utilities/http/logger')
 
   require File.join(File.dirname(__FILE__), 'http_utilities/http/request')
   require File.join(File.dirname(__FILE__), 'http_utilities/http/response')
 
-  require File.join(File.dirname(__FILE__), 'http_utilities/http/adapters/net_http')
-  require File.join(File.dirname(__FILE__), 'http_utilities/http/adapters/open_uri')
-  require File.join(File.dirname(__FILE__), 'http_utilities/http/adapters/curb')
-
   require File.join(File.dirname(__FILE__), 'http_utilities/http/client')
 
   require File.join(File.dirname(__FILE__), 'http_utilities/http/mechanize/client')
 
   if defined?(ActiveRecord)
-    require File.join(File.dirname(__FILE__), 'http_utilities/proxies/proxy_module')
-    require File.join(File.dirname(__FILE__), 'http_utilities/proxies/proxy_checker')
+    require File.join(File.dirname(__FILE__), 'http_utilities/proxies/mysql/proxy_module')
     require File.join(File.dirname(__FILE__), 'http_utilities/proxies/proxy_seeder')
   end
   
+  if defined?(Mongoid)
+    require File.join(File.dirname(__FILE__), 'http_utilities/proxies/mongo/proxy_module')
+  end
+  
+  require File.join(File.dirname(__FILE__), 'http_utilities/proxies/proxy_checker')
+  
   if defined?(Resque)
     require File.join(File.dirname(__FILE__), 'http_utilities/jobs/resque/proxies/check_proxies_job')
     require File.join(File.dirname(__FILE__), 'http_utilities/jobs/resque/proxies/check_proxy_job')
@@ -39,9 +36,4 @@ module HttpUtilities
     require File.join(File.dirname(__FILE__), 'http_utilities/jobs/sidekiq/proxies/check_proxies_job')
     require File.join(File.dirname(__FILE__), 'http_utilities/jobs/sidekiq/proxies/check_proxy_job')
   end
-
-  if defined?(MultiXml)
-    MultiXml.parser = :nokogiri
-  end
-end
-
+end

data/spec/http_utilities/client_spec.rb

@@ -9,42 +9,22 @@ describe HttpUtilities::Http::Client do
       @response   =   HttpUtilities::Http::Response.new
     end
 
-    it "should respond to a net http module method" do
-      @client.should respond_to(:post_and_retrieve_content_using_net_http)
+    it "should respond to #get" do
+      @client.should respond_to(:get)
     end
 
-    it "should respond to a open uri module method" do
-      @client.should respond_to(:retrieve_open_uri_content)
-    end
-
-    it "should respond to a curb module method" do
-      @client.should respond_to(:post_and_retrieve_content_using_curl)
+    it "should respond to #post" do
+      @client.should respond_to(:post)
     end
 
     it "should respond to a proxy module method" do
       @request.should respond_to(:set_proxy_options)
     end
 
-    it "should respond to a cookies module method" do
-      @client.should respond_to(:handle_cookies)
-    end
-
     it "should respond to a user agent module method" do
       @request.should respond_to(:user_agent)
     end
 
-    it "should respond to a request module method" do
-      @client.should respond_to(:generate_request_url)
-    end
-
-    it "should respond to a get module method" do
-      @client.should respond_to(:retrieve_raw_content)
-    end
-
-    it "should respond to a post module method" do
-      @client.should respond_to(:post_and_retrieve_content)
-    end
-
     it "should respond to a format module method" do
       @response.should respond_to(:as_html)
     end
@@ -59,87 +39,5 @@ describe HttpUtilities::Http::Client do
     it "should have assigned user agent" do
       @request.user_agent.should_not == nil
     end
-
-    it "should return a properly formatted request url using supplied parameters" do
-      params = {:url => "http://www.google.com", :q => "ruby on rails", :start => 0}
-      @client.generate_request_url(params).should == "http://www.google.com?q=ruby%20on%20rails&start=0"
-    end
-
-    describe "when retrieving content using Net::Http" do
-      it "should fetch Google results as unparsed HTML" do
-        params = {:url => "http://www.google.com", :q => "ruby on rails", :start => 0}
-        response = @client.retrieve_raw_content(@client.generate_request_url(params), {:method => :net_http})
-        response.body.should be_a(String)
-      end
-
-      it "should fetch Google results as a Nokogiri::HTML::Document" do
-        params = {:url => "http://www.google.com", :q => "ruby on rails", :start => 0}
-        response = @client.retrieve_parsed_html(@client.generate_request_url(params), {:method => :net_http})
-        response.parsed_body.should be_a(Nokogiri::HTML::Document)
-      end
-
-      it "should fetch Google Weather data a Nokogiri::XML::Document" do
-        params = {:url => "http://www.google.com/ig/api", :weather => 90120}
-        response = @client.retrieve_parsed_xml(@client.generate_request_url(params), {:method => :net_http})
-        response.parsed_body.should be_a(Nokogiri::XML::Document)
-      end
-    end
-
-    describe "when retrieving content using a proxy" do
-      it "should have the proxy instance variable properly set" do
-        options = {:method => :net_http, :proxy => "127.0.0.1:80", :response_only => false}
-        params = {:url => "http://www.google.com", :q => "ruby on rails", :start => 0}
-
-        response = @client.retrieve_parsed_html(@client.generate_request_url(params), options)
-        proxy = response.request.proxy
-
-        proxy.should_not be_nil
-        proxy[:host].should == '127.0.0.1'
-        proxy[:port].should == 80
-      end
-    end
-
-    describe "when persisting cookies" do
-      it "should have the cookie instance variable properly set" do
-        options = {:method => :net_http, :use_cookies => true, :save_cookies => true, :response_only => false}
-        params  = {:url => "http://www.google.com", :q => "ruby on rails", :start => 0}
-
-        response  =   @client.retrieve_parsed_html(@client.generate_request_url(params), options)
-        cookies   =   response.request.cookies
-        
-        cookies.should_not be_nil
-      end
-    end
-
-    describe "when posting content" do
-      before(:each) do
-        @trackback_url    =   "http://techcrunch.com/wp-trackback.php?p=314942"
-        @post_data        =   {
-          :url        =>  "http://www.google.com",
-          :blog_name  =>  "Testing",
-          :title      =>  "Title",
-          :excerpt    =>  "Testing..."
-        }
-      end
-
-      if (!defined?(JRUBY_VERSION))
-        it "should send a trackback to a TechCrunch post using Curb and return the response as a Nokogiri::XML::Document" do
-          options = {:method => :curl}
-
-          response = @client.post_and_retrieve_parsed_xml(@trackback_url, @post_data, options)
-          response.parsed_body.should be_a(Nokogiri::XML::Document)
-        end
-      end
-
-      it "should send a trackback to a TechCrunch post using Net::Http and return the response as a Nokogiri::XML::Document" do
-        options = {:method => :net_http}
-
-        response = @client.post_and_retrieve_parsed_xml(@trackback_url, @post_data, options)
-        response.parsed_body.should be_a(Nokogiri::XML::Document)
-      end
-    end
-
   end
-
-end
-
+end

data/spec/spec_helper.rb

@@ -31,11 +31,9 @@ if File.exists?(database_yml)
   ActiveRecord::Base.establish_connection(active_record_configuration)
   ActiveRecord::Base.logger = Logger.new(File.join(File.dirname(__FILE__), "debug.log"))
   
-  ActiveRecord::Base.silence do
-    ActiveRecord::Migration.verbose = false
-    load(File.dirname(__FILE__) + '/schema.rb')
-    load(File.dirname(__FILE__) + '/models.rb')
-  end
+  ActiveRecord::Migration.verbose = false
+  load(File.dirname(__FILE__) + '/schema.rb')
+  load(File.dirname(__FILE__) + '/models.rb')
 else
   raise "Please create #{database_yml} first to configure your database. Take a look at: #{database_yml}.example"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: http_utilities
 version: !ruby/object:Gem::Version
-  version: 1.1.2
+  version: 1.1.5
 platform: ruby
 authors:
 - Sebastian Johnsson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-07 00:00:00.000000000 Z
+date: 2015-08-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -16,64 +16,50 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.5.5
+        version: '1.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.5.5
+        version: '1.6'
 - !ruby/object:Gem::Dependency
   name: mechanize
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '2.5'
+        version: '2.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '2.5'
-- !ruby/object:Gem::Dependency
-  name: multi_xml
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0.5'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '2.7'
 - !ruby/object:Gem::Dependency
   name: net-ssh
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '2.8'
+        version: '2.9'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '2.8'
+        version: '2.9'
 - !ruby/object:Gem::Dependency
-  name: activerecord-import
+  name: rails
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
@@ -81,7 +67,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rails
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -95,7 +81,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: sqlite3
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -109,7 +95,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: sqlite3
+  name: mysql2
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -123,20 +109,20 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: mysql2
+  name: activerecord-import
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.3.11
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.3.11
-description: Wrapper for common Http Libraries (Net:HTTP/Open URI/Curl)
+        version: '0'
+description: Wrapper for Faraday with additional functionality
 email: 
 executables: []
 extensions: []
@@ -156,16 +142,9 @@ files:
 - lib/generators/templates/http_utilities.rb
 - lib/generators/templates/user_agents.yml
 - lib/http_utilities.rb
-- lib/http_utilities/http/adapters/curb.rb
-- lib/http_utilities/http/adapters/net_http.rb
-- lib/http_utilities/http/adapters/open_uri.rb
 - lib/http_utilities/http/client.rb
-- lib/http_utilities/http/cookies.rb
-- lib/http_utilities/http/format.rb
-- lib/http_utilities/http/get.rb
 - lib/http_utilities/http/logger.rb
 - lib/http_utilities/http/mechanize/client.rb
-- lib/http_utilities/http/post.rb
 - lib/http_utilities/http/proxy_support.rb
 - lib/http_utilities/http/request.rb
 - lib/http_utilities/http/response.rb
@@ -175,8 +154,9 @@ files:
 - lib/http_utilities/jobs/resque/proxies/check_proxy_job.rb
 - lib/http_utilities/jobs/sidekiq/proxies/check_proxies_job.rb
 - lib/http_utilities/jobs/sidekiq/proxies/check_proxy_job.rb
+- lib/http_utilities/proxies/mongo/proxy_module.rb
+- lib/http_utilities/proxies/mysql/proxy_module.rb
 - lib/http_utilities/proxies/proxy_checker.rb
-- lib/http_utilities/proxies/proxy_module.rb
 - lib/http_utilities/proxies/proxy_seeder.rb
 - lib/http_utilities/railtie.rb
 - lib/tasks/http_utilities_tasks.rake
@@ -208,10 +188,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.5
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 2
-summary: Wrapper for common Http Libraries (Net:HTTP/Open URI/Curl)
+summary: Wrapper for Faraday with additional functionality
 test_files:
 - spec/http_utilities/client_spec.rb
 - spec/http_utilities/mechanize_client_spec.rb

data/lib/http_utilities/http/adapters/curb.rb

@@ -1,107 +0,0 @@
-require 'uri'
-
-module HttpUtilities
-  module Http
-    module Adapters
-      module Curb
-
-        def post_and_retrieve_content_using_curl(url, data, options = {})
-          request   =   self.set_curl_options(url, options)
-          response  =  nil
-
-          if (request.interface && data)
-            if (data.is_a?(Hash))
-              data = data.map { |key, value| Curl::PostField.content(key.to_s, value.to_s) }
-            end
-
-            request.interface.http_post(data)
-            response = request.interface.body_str
-            response = HttpUtilities::Http::Response.new(response, request, options)
-          end
-
-          return response
-        end
-
-        def retrieve_curl_content(url, options = {})
-          request     =   self.set_curl_options(url, options)
-          response    =   nil
-
-          begin
-            success   =   request.interface.perform
-            response  =   request.interface.body_str
-            response  =   HttpUtilities::Http::Response.new(response, request, options)
-
-          rescue Exception => e
-            puts "\n\n#{Time.now}: IMPORTANT! Error occurred while trying to retrieve content from url #{url} and parse it. Error: #{e.message}. Error Class: #{e.class}"
-          end
-
-          return response
-        end
-
-        def go_to_url(url, options = {})
-          success = false
-
-          request = self.set_curl_options(url, options)
-
-          success = request.interface.perform
-          success = (success && curl.response_code.eql?(200))
-
-          return success
-        end
-
-        def set_curl_options(url, options = {})
-          options               =   options.clone()
-          
-          request               =   HttpUtilities::Http::Request.new
-          request.set_proxy_options(options)
-
-          accept_cookies        =   options.delete(:accept_cookies) { |e| false }
-          timeout               =   options.delete(:timeout) { |e| 120 }
-          maximum_redirects     =   options.delete(:maximum_redirects) { |e| 10 }
-          disable_auth          =   options.delete(:disable_auth) { |e| false }
-          accept_content_type   =   options.delete(:accept_content_type) { |e| false }
-          content_type          =   options.delete(:content_type) { |e| false }
-          cookie_file           =   nil
-
-          curl = Curl::Easy.new(url) do |c|
-            c.headers ||= {}
-            c.headers["User-Agent"]     =   c.useragent = request.user_agent
-            c.headers["Accept"]         =   accept_content_type if (accept_content_type)
-            c.headers["Content-Type"]   =   content_type if (content_type)
-            c.timeout                   =   timeout
-            c.ssl_verify_host           =   false
-            c.follow_location           =   true
-            c.max_redirects             =   maximum_redirects
-
-            if (disable_auth)
-              c.http_auth_types   = nil
-              c.proxy_auth_types  = nil
-              c.unrestricted_auth = false
-            end
-          end
-
-          if (accept_cookies)
-            FileUtils.mkdir_p File.join(Rails.root, "tmp/cookies")
-            identifier = Time.now.to_date.to_s(:db).gsub("-", "_").gsub("\s", "_").gsub(":", "_")
-            cookie_file = File.join(Rails.root, "tmp/cookies", "cookies_#{identifier}.txt")
-
-            curl.enable_cookies   =   true
-            curl.cookiejar        =   cookie_file
-            curl.cookiefile       =   cookie_file
-          end
-
-          if (request.proxy[:host] && request.proxy[:port])
-            curl.proxy_url    =   ::Proxy.format_proxy_address(request.proxy[:host], request.proxy[:port], false)
-            curl.proxy_type   =   5 if (request.proxy[:protocol] && request.proxy[:protocol].downcase.eql?('socks5'))
-            curl.proxypwd     =   ::Proxy.format_proxy_credentials(request.proxy[:username], request.proxy[:password]) if (request.proxy[:username] && request.proxy[:password])
-          end
-          
-          request.interface = curl
-
-          return request
-        end
-
-      end
-    end
-  end
-end

data/lib/http_utilities/http/adapters/net_http.rb

@@ -1,135 +0,0 @@
-require 'open-uri'
-require 'net/http'
-require 'uri'
-
-module HttpUtilities
-  module Http
-    module Adapters
-      module NetHttp
-
-        def post_and_retrieve_content_using_net_http(url, data, options = {})
-          uri             =   URI.parse(url)
-          request         =   set_net_http_options(uri, options)
-          response        =   nil
-
-          opts            =   options.clone()
-          content_type    =   opts.delete(:content_type) { |e| nil }
-
-          if (request.interface && data)
-            data = (data.is_a?(Hash)) ? generate_request_params(data) : data
-
-            request.interface.start do |http|
-              headers = {}
-              headers["User-Agent"]     =   request.user_agent
-              headers["Content-Type"]   =   content_type if (content_type)
-
-              http.post(uri.request_uri, data, headers) do |response_data|
-                response  =   response_data
-              end
-              
-              response    =   HttpUtilities::Http::Response.new(response, request, options)
-            end
-          end
-
-          return response
-        end
-
-        def set_net_http_options(uri, options = {})
-          request                         =   HttpUtilities::Http::Request.new
-          request.set_proxy_options(options)
-          
-          request.interface               =   Net::HTTP.new(uri.host, uri.port, request.proxy[:host], request.proxy[:port])
-          
-          use_ssl                         =   options.fetch(:use_ssl, false)
-          request.interface.use_ssl       =   use_ssl
-          request.interface.verify_mode   =   OpenSSL::SSL::VERIFY_NONE if use_ssl
-
-          return request
-        end
-
-        def retrieve_net_http_content(url, options = {})
-          uri         =   URI.parse(url)
-          request     =   set_net_http_options(uri, options)
-          return perform_net_http_request(request, uri, options)
-        end
-
-        def perform_net_http_request(request_or_url, uri = nil, options = {}, redirect_count = 0, max_redirects = 5)
-          request   =   nil
-          response  =   nil
-
-          if (request_or_url)
-            opts              =   (options.is_a?(Hash)) ? options.clone() : {}
-            retries           =   opts.delete(:retries) { |e| 3 }
-            force_encoding    =   opts.delete(:force_encoding) { |e| false }
-            cookies           =   opts.delete(:cookies) { |e| nil }
-            timeout           =   opts.delete(:timeout) { |e| 30 }
-
-            if (request_or_url.is_a?(String))
-              uri       =   URI.parse(request_or_url)
-              request   =   self.set_net_http_options(uri, options)
-            else
-              request   =   request_or_url
-            end
-
-            if (uri && uri.respond_to?(:request_uri) && uri.request_uri)
-              headers           =   {"User-Agent" => request.user_agent}
-              headers           =   set_cookies(headers, cookies)
-              request_uri       =   uri.request_uri
-              http_request      =   Net::HTTP::Get.new(request_uri, headers)
-              
-              begin
-                request.interface.start do |http|
-                  http.read_timeout   =   timeout
-                  response            =   http.request(http_request)
-                end
-
-              rescue Errno::ETIMEDOUT, Errno::ECONNREFUSED, Errno::ENETUNREACH, Errno::ECONNRESET, Timeout::Error, Net::HTTPUnauthorized, Net::HTTPForbidden => error
-                log(:error, "[HttpUtilities::Http::Client] - Error occurred while trying to fetch url '#{uri.request_uri}'. Error Class: #{error.class.name}. Error Message: #{error.message}")
-                retries -= 1
-                retry if (retries > 0)
-              end
-            end
-          end
-
-          if (response)
-            location = response['location']
-
-            if (!(response.code.to_s =~ /^30\d{1}/i).nil? && location && location.present?)
-              location            =   location.strip.downcase
-              redirect_count     +=   1
-              
-              if (redirect_count < max_redirects)
-                request.cookies   =   handle_cookies(response)
-                location          =   "http://#{uri.host}/#{location.gsub(/^\//i, "")}" if (uri && (location =~ /^http(s)?/i).nil?)
-                
-                log(:info, "[HttpUtilities::Http::Client] - Redirecting to location: #{location}.")
-                
-                options           =   options.merge(:cookies => request.cookies) if request.cookies
-                response          =   perform_net_http_request(location, uri, options, redirect_count, max_redirects)
-              end
-            end
-
-            request.cookies   =   handle_cookies(response)
-            response          =   set_response(response)
-            response          =   HttpUtilities::Http::Response.new(response, request, options)
-          end
-
-          return response
-        end
-        
-        def set_response(response)
-          if (response.is_a?(String))
-            response = response
-          elsif (response.is_a?(Net::HTTPResponse))
-            response = response.body
-          elsif (response.is_a?(HttpUtilities::Http::Response))
-            response = response.body
-          end
-          
-          return response
-        end
-
-      end
-    end
-  end
-end

data/lib/http_utilities/http/adapters/open_uri.rb

@@ -1,46 +0,0 @@
-require 'open-uri'
-require 'uri'
-
-module HttpUtilities
-  module Http
-    module Adapters
-      module OpenUri
-
-        def retrieve_open_uri_content(url, options = {}, retries = 0, max_retries = 3)
-          response  =   nil
-
-          options   =   options.clone()
-          request   =   HttpUtilities::Http::Request.new
-          request.set_proxy_options(options)
-
-          open_uri_options = {"UserAgent" => request.user_agent}
-          open_uri_options[:read_timeout] = options.delete(:timeout) { |e| 120 }
-
-          if (request.proxy[:host] && request.proxy[:port])
-            proxy_address = Proxy.format_proxy_address(request.proxy[:host], request.proxy[:port], true)
-            open_uri_options[:proxy] = proxy_address
-
-            if (request.proxy[:username] && request.proxy[:password])
-              open_uri_options[:proxy_http_basic_authentication] = [proxy_address, request.proxy[:username], request.proxy[:password]]
-            end
-          end
-
-          connection = nil
-          while (connection.nil? && retries < max_retries)
-            connection = open(url, open_uri_options)
-            retries += 1
-          end
-
-          if (connection)
-            connection.rewind
-            response  =   connection.readlines.join("\n")
-            response  =   HttpUtilities::Http::Response.new(response, request)
-          end
-
-          return response
-        end
-
-      end
-    end
-  end
-end