http_utilities 1.0.1

data/lib/http_utilities/jobs/resque/proxies/check_proxies_job.rb

@@ -0,0 +1,15 @@
+module HttpUtilities
+  module Jobs
+    module Resque
+      module Proxies
+        class CheckProxiesJob
+          @queue = :proxies
+
+          def perform(protocol = :all, proxy_type = :all, mode = :synchronous)
+            HttpUtilities::Proxies::ProxyChecker.new.check_proxies(protocol.to_sym, proxy_type.to_sym, mode.to_sym)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/http_utilities/jobs/resque/proxies/check_proxy_job.rb

@@ -0,0 +1,21 @@
+module HttpUtilities
+  module Jobs
+    module Resque
+      module Proxies
+        class CheckProxyJob
+          @queue = :proxies
+
+          def self.perform(proxy_id)
+            proxy_object = ::Proxy.where(:id => proxy_id).first
+
+            if (proxy_object)
+              checker = HttpUtilities::Proxies::ProxyChecker.new
+              checker.check_proxy(proxy_object)
+              checker.update_proxies
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/http_utilities/jobs/sidekiq/proxies/check_proxies_job.rb

@@ -0,0 +1,17 @@
+module HttpUtilities
+  module Jobs
+    module Sidekiq
+      module Proxies
+        class CheckProxiesJob
+          include ::Sidekiq::Worker
+          sidekiq_options :queue    =>  :proxies,
+                          :unique   =>  false
+
+          def perform(protocol = :all, proxy_type = :all, mode = :synchronous)
+            HttpUtilities::Proxies::ProxyChecker.new.check_proxies(protocol.to_sym, proxy_type.to_sym, mode.to_sym)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/http_utilities/jobs/sidekiq/proxies/check_proxy_job.rb

@@ -0,0 +1,22 @@
+module HttpUtilities
+  module Jobs
+    module Sidekiq
+      module Proxies
+        class CheckProxyJob
+          include ::Sidekiq::Worker
+          sidekiq_options :queue    =>  :proxies
+
+          def perform(proxy_id)
+            proxy_object = ::Proxy.where(:id => proxy_id).first
+
+            if (proxy_object)
+              checker = HttpUtilities::Proxies::ProxyChecker.new
+              checker.check_proxy(proxy_object)
+              checker.update_proxies
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/http_utilities/proxies/proxy_checker.rb

@@ -0,0 +1,122 @@
+# -*- encoding : utf-8 -*-
+
+module HttpUtilities
+  module Proxies
+    class ProxyChecker
+      require 'activerecord-import'
+      attr_accessor :client, :processed_proxies
+      attr_accessor :limit, :minimum_successful_attempts, :maximum_failed_attempts
+
+      def initialize
+        self.client                       =   HttpUtilities::Http::Client.new
+        self.processed_proxies            =   []
+        
+        self.limit                        =   1000
+        self.minimum_successful_attempts  =   1
+        self.maximum_failed_attempts      =   10
+      end
+
+      def check_and_update_proxies(protocol = :all, proxy_type = :all, mode = :synchronous)
+        check_proxies(protocol, proxy_type, mode)
+        update_proxies
+      end
+
+      def check_proxies(protocol = :all, proxy_type = :all, mode = :synchronous)
+        proxies = Proxy.should_be_checked(protocol, proxy_type, Time.now, self.limit)
+
+        if (proxies && proxies.any?)
+          Rails.logger.info "Found #{proxies.size} #{proxy_type} proxies to check."
+
+          proxies.each do |proxy|
+            case mode
+              when :synchronous
+                check_proxy(proxy)
+              when :resque
+                Resque.enqueue(HttpUtilities::Jobs::Resque::Proxies::CheckProxyJob, proxy.id)
+              when :sidekiq
+                HttpUtilities::Jobs::Sidekiq::Proxies::CheckProxyJob.perform_async(proxy.id)
+            end
+          end
+
+        else
+          Rails.logger.info "Couldn't find any proxies to check!"
+        end
+      end
+      
+      def check_proxy(proxy, timeout = 60)
+        document      =   nil
+        valid_proxy   =   false
+
+        options = {:method              =>  :net_http,
+                   :use_proxy           =>  true, 
+                   :proxy               =>  proxy.proxy_address, 
+                   :proxy_protocol      =>  proxy.protocol,
+                   :timeout             =>  timeout,
+                   :maximum_redirects   =>  1,
+                   :disable_auth        =>  true
+                  }
+
+        Rails.logger.info "#{Time.now}: Fetching Proxy #{proxy.proxy_address}."
+
+        response            =   self.client.retrieve_parsed_html("http://www.google.com/webhp?hl=en", options)
+
+        if (response && response.parsed_body)
+          title             =   response.parsed_body.css("title").first
+
+          if (title && title.content)
+            begin
+              title         =   title.content.encode("UTF-8").strip.downcase
+              body_content  =   response.parsed_body.content.to_s.encode("UTF-8").strip.downcase
+              
+              valid_proxy   =   (title.eql?("google") || !(body_content =~ /google home/i).nil?)
+              
+              Rails.logger.info "Title is: #{title}. Proxy #{proxy.proxy_address}"
+              
+            rescue Exception => e
+              Rails.logger.error "Exception occured while trying to validate proxy. Error Class: #{e.class}. Error Message: #{e.message}"
+              valid_proxy   =   false
+            end
+          end
+        end
+
+        if (valid_proxy)
+          Rails.logger.info "#{Time.now}: Proxy #{proxy.proxy_address} is working!"
+        else
+          Rails.logger.info "#{Time.now}: Proxy #{proxy.proxy_address} is not working!"
+        end
+
+        self.processed_proxies << {:proxy => proxy, :valid => valid_proxy}
+      end
+
+      def update_proxies()
+        columns   =   [:host, :port, :last_checked_at, :valid_proxy, :successful_attempts, :failed_attempts]
+        values    =   []
+
+        Rails.logger.info "Updating/Importing #{self.processed_proxies.size} proxies"
+
+        if (self.processed_proxies && self.processed_proxies.any?)
+          self.processed_proxies.each do |value|
+            proxy                 =   value[:proxy]
+            valid                 =   value[:valid]
+            successful_attempts   =   proxy.successful_attempts
+            failed_attempts       =   proxy.failed_attempts
+
+            if (valid)
+              successful_attempts +=  1
+            else
+              failed_attempts     +=  1
+            end
+
+            is_valid              =   (successful_attempts >= self.minimum_successful_attempts && failed_attempts < self.maximum_failed_attempts)
+            value_arr             =   [proxy.host, proxy.port, Time.now, is_valid, successful_attempts, failed_attempts]
+            values                <<  value_arr
+          end
+
+          ::Proxy.import(columns, values, :on_duplicate_key_update => [:last_checked_at, :valid_proxy, :successful_attempts, :failed_attempts], :validate => false)
+        end
+
+      end
+
+    end
+  end
+end

data/lib/http_utilities/proxies/proxy_module.rb

@@ -0,0 +1,70 @@
+module HttpUtilities
+  module Proxies
+    module ProxyModule
+      
+      def self.included(base)
+        base.send :extend, ClassMethods
+        base.send :include, InstanceMethods
+      end
+
+      module ClassMethods
+        def should_be_checked(protocol = :all, proxy_type = :all, date = Time.now, limit = 10)
+          conditions = set_protocol_and_proxy_type_conditions(protocol, proxy_type)
+          conditions << ActiveRecord::Base.send(:sanitize_sql_array, ["(last_checked_at IS NULL OR last_checked_at < ?)", date])
+          conditions << "failed_attempts <= 10"
+          query = conditions.join(" AND ")
+          
+          where(query).order("valid_proxy ASC, failed_attempts ASC, last_checked_at ASC").limit(limit) 
+        end
+        
+        def get_random_proxy(protocol = :all, proxy_type = :all)
+          conditions = set_protocol_and_proxy_type_conditions(protocol, proxy_type)
+          conditions << ActiveRecord::Base.send(:sanitize_sql_array, ["valid_proxy = ?", true])
+          conditions << "last_checked_at IS NOT NULL"
+          query = conditions.join(" AND ")
+          
+          order_clause = case ActiveRecord::Base.connection.class.name
+            when "ActiveRecord::ConnectionAdapters::MysqlAdapter", "ActiveRecord::ConnectionAdapters::Mysql2Adapter" then "RAND() DESC"
+            when "ActiveRecord::ConnectionAdapters::SQLite3Adapter" then "RANDOM() DESC"
+          end
+          
+          proxy = nil
+          
+          uncached do
+            proxy = where(query).order(order_clause).limit(1).first
+          end
+
+          return proxy
+        end
+        
+        def set_protocol_and_proxy_type_conditions(protocol, proxy_type)
+          conditions = []
+          conditions << ActiveRecord::Base.send(:sanitize_sql_array, ["protocol = ?", protocol]) if (protocol && !protocol.downcase.to_sym.eql?(:all))
+          conditions << ActiveRecord::Base.send(:sanitize_sql_array, ["proxy_type = ?", proxy_type]) if (proxy_type && !proxy_type.downcase.to_sym.eql?(:all))
+          return conditions
+        end
+
+        def format_proxy_address(proxy_host, proxy_port = 80, include_http = false)
+          proxy_address = "#{proxy_host}:#{proxy_port}"
+          proxy_address.insert(0, "http://") if (include_http && !proxy_address.start_with?("http://"))
+          return proxy_address
+        end
+
+        def format_proxy_credentials(username, password)
+          return "#{username}:#{password}"
+        end
+      end
+
+      module InstanceMethods
+        def proxy_address(include_http = false)
+          return ::Proxy.format_proxy_address(self.host, self.port, include_http)
+        end
+        
+        def proxy_credentials
+          return ::Proxy.format_proxy_credentials(self.username, self.password)
+        end
+      end
+      
+    end
+  end
+end

data/lib/http_utilities/proxies/proxy_seeder.rb

@@ -0,0 +1,104 @@
+module HttpUtilities
+  module Proxies
+    class ProxySeeder
+      require 'activerecord-import'
+      attr_accessor :protocols, :proxy_types, :categories
+
+      def initialize
+        self.protocols      =   ['http', 'socks5']
+        self.proxy_types    =   ['public', 'shared', 'private']
+        self.categories     =   ['L1', 'L2', 'L3', 'unspecified']
+      end
+
+      def seed
+        import_proxies
+      end
+
+      def import_proxies
+        proxy_data = parse_proxies
+
+        proxy_data.each do |protocol, types|
+          types.each do |type, categories|
+            categories.each do |category, proxies|
+              bulk_import_proxies(proxies, protocol, type, category)
+            end
+          end        
+        end if (proxy_data && !proxy_data.empty?)
+      end
+
+      def bulk_import_proxies(proxy_list, protocol, proxy_type, category)        
+        columns = [:host, :port, :protocol, :proxy_type, :category]
+        category = (category && !category.eql?('unspecified')) ? category : nil
+
+        begin
+          values = []
+
+          proxy_list.slice!(0..1000).each do |proxy|
+            host = proxy[:host]
+            port = proxy[:port]
+            value_arr = [host, port, protocol, proxy_type, category]
+            values << value_arr
+          end
+
+          ::Proxy.import(columns, values, :on_duplicate_key_update => [:proxy_type], :validate => false) if (values && values.any?)
+        end while (proxy_list && proxy_list.any?)
+      end
+
+      def parse_proxies
+        proxies = {}
+
+        self.protocols.each do |protocol|
+          proxies[protocol] = {}
+
+          self.proxy_types.each do |proxy_type|
+            proxies[protocol][proxy_type] = {}
+            proxies[protocol][proxy_type]['unspecified'] = []
+
+            if (protocol.eql?("http"))
+              self.categories.each do |category|
+                proxies[protocol][proxy_type][category] = get_proxies_from_files("#{get_seed_root}#{protocol}/#{proxy_type}/#{category}/*.txt")
+              end
+            end
+            
+            proxies[protocol][proxy_type]['unspecified'] = proxies[protocol][proxy_type]['unspecified'] + get_proxies_from_files("#{get_seed_root}#{protocol}/#{proxy_type}/*.txt")
+          end
+        end
+
+        return proxies
+      end
+
+      def get_proxies_from_files(pattern)
+        proxies = []
+        file_paths = Dir.glob(pattern)
+
+        file_paths.each do |file_path|
+          proxy_rows = []
+          File.open(file_path, 'r') {|f| proxy_rows = f.readlines("\n") }
+
+          proxy_rows.each do |row|
+            host, port  =   nil
+            
+            parts       =   row.include?(":") ? row.split(":") : nil
+            
+            if (parts && parts.any? && parts.size >= 2)
+              host        =   parts.first
+              port        =   parts.second.to_i
+            end
+            
+            proxies << {:host => host, :port => port} if (host && port)
+          end
+        end
+
+        return proxies
+      end
+      
+      def get_seed_root
+        rails_seed_root   =   defined?(Rails) ? "#{Rails.root}/db/seed_data/proxies/" : nil
+        gem_seed_root     =   File.join(File.dirname(__FILE__), "../../generators/templates/seed_data/proxies/")
+        
+        return (rails_seed_root && File.exists?(rails_seed_root)) ? rails_seed_root : gem_seed_root
+      end
+
+    end
+  end
+end

data/lib/http_utilities/railtie.rb

@@ -0,0 +1,11 @@
+require 'http_utilities'
+require 'rails'
+module HttpUtilities
+  class Railtie < Rails::Railtie
+    
+    rake_tasks do
+      Dir[File.join(File.dirname(__FILE__), '../tasks/*.rake')].each { |ext| load ext }
+    end
+    
+  end
+end

data/lib/http_utilities.rb

@@ -0,0 +1,47 @@
+# -*- encoding : utf-8 -*-
+module HttpUtilities
+  VERSION = "1.0.1"
+  
+  require File.join(File.dirname(__FILE__), 'http_utilities/railtie') if defined?(Rails)
+
+  require File.join(File.dirname(__FILE__), 'http_utilities/http/proxy_support')
+  require File.join(File.dirname(__FILE__), 'http_utilities/http/cookies')
+  require File.join(File.dirname(__FILE__), 'http_utilities/http/user_agent')
+  require File.join(File.dirname(__FILE__), 'http_utilities/http/url')
+  require File.join(File.dirname(__FILE__), 'http_utilities/http/format')
+  require File.join(File.dirname(__FILE__), 'http_utilities/http/get')
+  require File.join(File.dirname(__FILE__), 'http_utilities/http/post')
+  require File.join(File.dirname(__FILE__), 'http_utilities/http/logger')
+
+  require File.join(File.dirname(__FILE__), 'http_utilities/http/request')
+  require File.join(File.dirname(__FILE__), 'http_utilities/http/response')
+
+  require File.join(File.dirname(__FILE__), 'http_utilities/http/adapters/net_http')
+  require File.join(File.dirname(__FILE__), 'http_utilities/http/adapters/open_uri')
+  require File.join(File.dirname(__FILE__), 'http_utilities/http/adapters/curb')
+
+  require File.join(File.dirname(__FILE__), 'http_utilities/http/client')
+
+  require File.join(File.dirname(__FILE__), 'http_utilities/http/mechanize/client')
+
+  if defined?(ActiveRecord)
+    require File.join(File.dirname(__FILE__), 'http_utilities/proxies/proxy_module')
+    require File.join(File.dirname(__FILE__), 'http_utilities/proxies/proxy_checker')
+    require File.join(File.dirname(__FILE__), 'http_utilities/proxies/proxy_seeder')
+
+    if defined?(Resque)
+      require File.join(File.dirname(__FILE__), 'http_utilities/jobs/resque/proxies/check_proxies_job')
+      require File.join(File.dirname(__FILE__), 'http_utilities/jobs/resque/proxies/check_proxy_job')
+    end
+    
+    if defined?(Sidekiq)
+      require File.join(File.dirname(__FILE__), 'http_utilities/jobs/sidekiq/proxies/check_proxies_job')
+      require File.join(File.dirname(__FILE__), 'http_utilities/jobs/sidekiq/proxies/check_proxy_job')
+    end
+  end
+
+  if defined?(MultiXml)
+    MultiXml.parser = :nokogiri
+  end
+end
+

data/lib/tasks/http_utilities_tasks.rake

@@ -0,0 +1,19 @@
+namespace :http_utilities do
+  namespace :proxies do
+    desc "Proxy tasks"
+
+    task :seed_proxies => :environment do |task, args|
+      seeder = HttpUtilities::Proxies::ProxySeeder.new
+      seeder.seed
+    end
+
+    task :check_proxies, [:protocol, :proxy_type, :processing_method] => [:environment] do |task, args|
+      protocol                =   (args.protocol)             ?   args.protocol.to_sym            : :http
+      proxy_type              =   (args.proxy_type)           ?   args.proxy_type.to_sym          : :public
+      processing_method       =   (args.processing_method)    ?   args.processing_method.to_sym   : :synchronous
+      
+      proxy_checker = HttpUtilities::Proxies::ProxyChecker.new
+      proxy_checker.check_and_update_proxies(protocol, proxy_type, processing_method)
+    end
+  end
+end

data/spec/database.yml.example

@@ -0,0 +1,10 @@
+sqlite3:
+  adapter: sqlite3
+  database: http_utilities.sqlite3
+
+mysql:
+  adapter: mysql2
+  encoding: utf8
+  database: http_utilities_test
+  username: 
+  password:

data/spec/http_utilities/client_spec.rb

@@ -0,0 +1,145 @@
+require File.expand_path('../../spec_helper', __FILE__)
+
+describe HttpUtilities::Http::Client do
+
+  describe "when modules have been included" do
+    before(:each) do
+      @client     =   HttpUtilities::Http::Client.new
+      @request    =   HttpUtilities::Http::Request.new
+      @response   =   HttpUtilities::Http::Response.new
+    end
+
+    it "should respond to a net http module method" do
+      @client.should respond_to(:post_and_retrieve_content_using_net_http)
+    end
+
+    it "should respond to a open uri module method" do
+      @client.should respond_to(:retrieve_open_uri_content)
+    end
+
+    it "should respond to a curb module method" do
+      @client.should respond_to(:post_and_retrieve_content_using_curl)
+    end
+
+    it "should respond to a proxy module method" do
+      @request.should respond_to(:set_proxy_options)
+    end
+
+    it "should respond to a cookies module method" do
+      @client.should respond_to(:handle_cookies)
+    end
+
+    it "should respond to a user agent module method" do
+      @request.should respond_to(:user_agent)
+    end
+
+    it "should respond to a request module method" do
+      @client.should respond_to(:generate_request_url)
+    end
+
+    it "should respond to a get module method" do
+      @client.should respond_to(:retrieve_raw_content)
+    end
+
+    it "should respond to a post module method" do
+      @client.should respond_to(:post_and_retrieve_content)
+    end
+
+    it "should respond to a format module method" do
+      @response.should respond_to(:as_html)
+    end
+  end
+
+  describe "when initialized" do
+    before(:each) do
+      @client     =   HttpUtilities::Http::Client.new
+      @request    =   HttpUtilities::Http::Request.new
+    end
+
+    it "should have assigned user agent" do
+      @request.user_agent.should_not == nil
+    end
+
+    it "should return a properly formatted request url using supplied parameters" do
+      params = {:url => "http://www.google.com", :q => "ruby on rails", :start => 0}
+      @client.generate_request_url(params).should == "http://www.google.com?q=ruby%20on%20rails&start=0"
+    end
+
+    describe "when retrieving content using Net::Http" do
+      it "should fetch Google results as unparsed HTML" do
+        params = {:url => "http://www.google.com", :q => "ruby on rails", :start => 0}
+        response = @client.retrieve_raw_content(@client.generate_request_url(params), {:method => :net_http})
+        response.body.should be_a(String)
+      end
+
+      it "should fetch Google results as a Nokogiri::HTML::Document" do
+        params = {:url => "http://www.google.com", :q => "ruby on rails", :start => 0}
+        response = @client.retrieve_parsed_html(@client.generate_request_url(params), {:method => :net_http})
+        response.parsed_body.should be_a(Nokogiri::HTML::Document)
+      end
+
+      it "should fetch Google Weather data a Nokogiri::XML::Document" do
+        params = {:url => "http://www.google.com/ig/api", :weather => 90120}
+        response = @client.retrieve_parsed_xml(@client.generate_request_url(params), {:method => :net_http})
+        response.parsed_body.should be_a(Nokogiri::XML::Document)
+      end
+    end
+
+    describe "when retrieving content using a proxy" do
+      it "should have the proxy instance variable properly set" do
+        options = {:method => :net_http, :proxy => "127.0.0.1:80", :response_only => false}
+        params = {:url => "http://www.google.com", :q => "ruby on rails", :start => 0}
+
+        response = @client.retrieve_parsed_html(@client.generate_request_url(params), options)
+        proxy = response.request.proxy
+
+        proxy.should_not be_nil
+        proxy[:host].should == '127.0.0.1'
+        proxy[:port].should == 80
+      end
+    end
+
+    describe "when persisting cookies" do
+      it "should have the cookie instance variable properly set" do
+        options = {:method => :net_http, :use_cookies => true, :save_cookies => true, :response_only => false}
+        params  = {:url => "http://www.google.com", :q => "ruby on rails", :start => 0}
+
+        response  =   @client.retrieve_parsed_html(@client.generate_request_url(params), options)
+        cookies   =   response.request.cookies
+        
+        cookies.should_not be_nil
+      end
+    end
+
+    describe "when posting content" do
+      before(:each) do
+        @trackback_url    =   "http://techcrunch.com/wp-trackback.php?p=314942"
+        @post_data        =   {
+          :url        =>  "http://www.google.com",
+          :blog_name  =>  "Testing",
+          :title      =>  "Title",
+          :excerpt    =>  "Testing..."
+        }
+      end
+
+      if (!defined?(JRUBY_VERSION))
+        it "should send a trackback to a TechCrunch post using Curb and return the response as a Nokogiri::XML::Document" do
+          options = {:method => :curl}
+
+          response = @client.post_and_retrieve_parsed_xml(@trackback_url, @post_data, options)
+          response.parsed_body.should be_a(Nokogiri::XML::Document)
+        end
+      end
+
+      it "should send a trackback to a TechCrunch post using Net::Http and return the response as a Nokogiri::XML::Document" do
+        options = {:method => :net_http}
+
+        response = @client.post_and_retrieve_parsed_xml(@trackback_url, @post_data, options)
+        response.parsed_body.should be_a(Nokogiri::XML::Document)
+      end
+    end
+
+  end
+
+end
+

data/spec/http_utilities/mechanize_client_spec.rb

@@ -0,0 +1,35 @@
+require File.expand_path('../../spec_helper', __FILE__)
+
+describe HttpUtilities::Http::Mechanize::Client do
+
+  describe "when modules have been included" do
+    before(:each) do
+      @client     =   HttpUtilities::Http::Mechanize::Client.new
+    end
+
+    it "should respond to a user agent module method" do
+      @client.should respond_to(:user_agent)
+    end
+
+    it "should respond to a request module method" do
+      @client.should respond_to(:generate_request_url)
+    end
+  end
+
+  describe "when initialized" do
+    before(:each) do
+      @client     =   HttpUtilities::Http::Mechanize::Client.new
+    end
+
+    it "should have assigned user agents" do
+      @client.user_agent.should_not be_nil
+    end
+
+    it "should submit a google search query successfully" do
+      #mock this later on...
+      page = @client.set_form_and_submit("http://www.google.com/webhp", {:name => "f"}, :first, {:q => {:type => :input, :value => "Ruby on Rails"}})
+      page.parser.should_not be_nil
+    end
+  end
+
+end

data/spec/http_utilities/proxy_checker_spec.rb

@@ -0,0 +1,11 @@
+require File.expand_path('../../spec_helper', __FILE__)
+
+describe HttpUtilities::Proxies::ProxyChecker do
+
+  describe "when initialized" do
+    before(:each) do
+      @checker = HttpUtilities::Proxies::ProxyChecker.new
+    end
+    
+  end
+end