http_utilities 1.0.1

data/spec/http_utilities/proxy_seeder_spec.rb

@@ -0,0 +1,24 @@
+require File.expand_path('../../spec_helper', __FILE__)
+
+describe HttpUtilities::Proxies::ProxySeeder do
+
+  describe "when initialized" do
+    before(:each) do
+      @seeder = HttpUtilities::Proxies::ProxySeeder.new
+    end
+        
+    it "should parse proxies from text files" do
+      proxy_data = @seeder.parse_proxies
+      proxy_data.should_not be_nil
+    end
+    
+    #Can only be tested on mysql, sqlite will fail with ConstraintExceptions (since it does not support on duplicate key update ...)
+    if (ENV['DB'] == 'mysql')
+      it "should import parsed proxies using activerecord-import" do
+        @seeder.seed
+        ::Proxy.count.should > 0
+      end
+    end
+    
+  end
+end

data/spec/http_utilities/proxy_spec.rb

@@ -0,0 +1,114 @@
+require File.expand_path('../../spec_helper', __FILE__)
+
+describe Proxy do
+
+  describe "when initialized" do
+    before(:each) do
+      clean_database!
+      @proxy = Proxy.new
+    end
+    
+    it "should respond to proxy address module instance method" do
+      @proxy.should respond_to(:proxy_address)
+    end
+    
+    it "should correctly return a formatted proxy address" do
+      @proxy.host = "127.0.0.1"
+      @proxy.port = 80
+      
+      @proxy.proxy_address.should == "127.0.0.1:80"
+      @proxy.proxy_address(include_http = true).should == "http://127.0.0.1:80"
+    end
+    
+    it "should correctly return formatted proxy credentials" do
+      @proxy.username = "usr"
+      @proxy.password = "passw"
+      
+      @proxy.proxy_credentials.should == "usr:passw"
+    end
+  end
+  
+  describe "in a class context" do
+    it "should respond to get random proxy module class method" do
+      Proxy.should respond_to(:get_random_proxy)
+    end
+    
+    it "should respond to format proxy address module class method" do
+      Proxy.should respond_to(:format_proxy_address)
+    end
+    
+    it "should respond to format proxy credentials module class method" do
+      Proxy.should respond_to(:format_proxy_credentials)
+    end
+    
+    it "should respond to should_be_checked-scope" do
+      Proxy.should respond_to(:should_be_checked)
+    end
+  end
+  
+  describe "when fetching proxies" do
+    before(:each) do
+      clean_database!
+      
+      data = {:host => "127.0.0.1", 
+              :port => 80, 
+              :protocol => 'http', 
+              :proxy_type => 'public', 
+              :category => 'L1', 
+              :valid_proxy => true,
+              :last_checked_at => 1.week.ago
+             }
+      
+      #Http Proxies
+      Proxy.create(data)
+      Proxy.create(data.merge!({:port => 81, :proxy_type => 'shared'}))
+      Proxy.create(data.merge!({:port => 82, :proxy_type => 'private'}))
+      
+      #Socks Proxies
+      Proxy.create(data.merge!({:port => 83, :protocol => 'socks', :proxy_type => 'public'}))
+      Proxy.create(data.merge!({:port => 84, :protocol => 'socks', :proxy_type => 'shared'}))
+      Proxy.create(data.merge!({:port => 85, :protocol => 'socks', :proxy_type => 'private'}))
+    end
+    
+    it "should return proxies that should be checked (using default parameters)" do
+      proxies = Proxy.should_be_checked
+      proxies.should_not be_nil
+      proxies.size.should == 6
+    end
+    
+    [:http, :socks].each do |protocol|
+      it "should return #{protocol.to_s} proxies that should be checked" do
+        proxies = Proxy.should_be_checked(protocol)
+        proxies.should_not be_nil
+        proxies.size.should == 3
+      end
+      
+      [:public, :shared, :private].each do |proxy_type|
+        it "should return #{proxy_type.to_s} #{protocol.to_s} proxies that should be checked" do
+          proxies = Proxy.should_be_checked(protocol, proxy_type)
+          proxies.should_not be_nil
+          proxies.size.should == 1
+        end
+      end
+    end
+    
+    [:http, :socks].each do |protocol|
+      it "should return a random #{protocol.to_s} proxy" do
+        random_proxy = Proxy.get_random_proxy(protocol)
+        random_proxy.should_not be_nil
+        random_proxy.protocol.should == protocol.to_s
+      end
+      
+      [:public, :shared, :private].each do |proxy_type|
+        it "should return a random #{proxy_type.to_s} #{protocol.to_s} proxy" do
+          random_proxy = Proxy.get_random_proxy(protocol, proxy_type)
+          random_proxy.should_not be_nil
+          random_proxy.protocol.should == protocol.to_s
+          random_proxy.proxy_type.should == proxy_type.to_s
+        end
+      end
+    end
+
+  end
+  
+end

data/spec/models.rb

@@ -0,0 +1,6 @@
+require "active_record"
+require File.expand_path('../../lib/http_utilities/proxies/proxy_module', __FILE__)
+
+class Proxy < ActiveRecord::Base
+  include HttpUtilities::Proxies::ProxyModule
+end

data/spec/schema.rb

@@ -0,0 +1,30 @@
+ActiveRecord::Schema.define :version => 0 do
+  execute "DROP TABLE IF EXISTS proxies"
+  
+  create_table :proxies do |t|
+    
+    t.string :host, :null => false
+    t.integer :port, :null => false
+    t.string :username
+    t.string :password
+    
+    t.string :protocol, :null => false, :default => 'http'
+    t.string :proxy_type, :null => false, :defaut => 'public'
+    t.string :category
+    
+    t.datetime :last_checked_at
+    t.boolean :valid_proxy, :null => false, :default => false
+    t.integer :successful_attempts, :null => false, :default => 0
+    t.integer :failed_attempts, :null => false, :default => 0
+
+    t.timestamps
+  end
+  
+  add_index :proxies, [:host, :port], :unique => true, :name => 'index_unique_proxy'
+  add_index :proxies, :protocol, :name => 'index_protocol'
+  add_index :proxies, :proxy_type, :name => 'index_proxy_type'
+  add_index :proxies, :category, :name => 'index_category'
+  add_index :proxies, :valid_proxy, :name => 'index_valid_proxy'
+  add_index :proxies, :successful_attempts, :name => 'index_successful_attempts'
+  add_index :proxies, :failed_attempts, :name => 'index_failed_attempts'
+end

data/spec/spec_helper.rb

@@ -0,0 +1,50 @@
+$LOAD_PATH << "." unless $LOAD_PATH.include?(".")
+
+begin
+  require "rubygems"
+  require "bundler"
+
+  if Gem::Version.new(Bundler::VERSION) <= Gem::Version.new("0.9.5")
+    raise RuntimeError, "Your bundler version is too old." +
+     "Run `gem install bundler` to upgrade."
+  end
+
+  # Set up load paths for all bundled gems
+  Bundler.setup
+rescue Bundler::GemNotFound
+  raise RuntimeError, "Bundler couldn't find some gems." +
+    "Did you run \`bundlee install\`?"
+end
+
+require "active_record"
+require "logger"
+Bundler.require
+
+require File.expand_path('../../lib/http_utilities', __FILE__)
+
+ENV['DB'] ||= 'sqlite3'
+
+database_yml = File.expand_path('../database.yml', __FILE__)
+if File.exists?(database_yml)
+  active_record_configuration = YAML.load_file(database_yml)[ENV['DB']]
+  
+  ActiveRecord::Base.establish_connection(active_record_configuration)
+  ActiveRecord::Base.logger = Logger.new(File.join(File.dirname(__FILE__), "debug.log"))
+  
+  ActiveRecord::Base.silence do
+    ActiveRecord::Migration.verbose = false
+    load(File.dirname(__FILE__) + '/schema.rb')
+    load(File.dirname(__FILE__) + '/models.rb')
+  end
+else
+  raise "Please create #{database_yml} first to configure your database. Take a look at: #{database_yml}.example"
+end
+
+def clean_database!
+  models = [Proxy]
+  models.each do |model|
+    ActiveRecord::Base.connection.execute "DELETE FROM #{model.table_name}"
+  end
+end
+
+clean_database!

metadata

@@ -0,0 +1,209 @@
+--- !ruby/object:Gem::Specification
+name: http_utilities
+version: !ruby/object:Gem::Version
+  version: 1.0.1
+platform: ruby
+authors:
+- Sebastian Johnsson
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-11-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.5.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.5.5
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '2.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '2.5'
+- !ruby/object:Gem::Dependency
+  name: multi_xml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0.5'
+- !ruby/object:Gem::Dependency
+  name: activerecord-import
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.3.11
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.3.11
+description: Wrapper for common Http Libraries (Net:HTTP/Open URI/Curl)
+email: 
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- README
+- Rakefile
+- VERSION
+- http_utilities.gemspec
+- lib/generators/active_record/http_utilities_generator.rb
+- lib/generators/active_record/templates/migration.rb
+- lib/generators/active_record/templates/proxy.rb
+- lib/generators/helpers/file_helper.rb
+- lib/generators/helpers/orm_helpers.rb
+- lib/generators/http_utilities/http_utilities_generator.rb
+- lib/generators/templates/http_utilities.rb
+- lib/generators/templates/user_agents.yml
+- lib/http_utilities.rb
+- lib/http_utilities/http/adapters/curb.rb
+- lib/http_utilities/http/adapters/net_http.rb
+- lib/http_utilities/http/adapters/open_uri.rb
+- lib/http_utilities/http/client.rb
+- lib/http_utilities/http/cookies.rb
+- lib/http_utilities/http/format.rb
+- lib/http_utilities/http/get.rb
+- lib/http_utilities/http/logger.rb
+- lib/http_utilities/http/mechanize/client.rb
+- lib/http_utilities/http/post.rb
+- lib/http_utilities/http/proxy_support.rb
+- lib/http_utilities/http/request.rb
+- lib/http_utilities/http/response.rb
+- lib/http_utilities/http/url.rb
+- lib/http_utilities/http/user_agent.rb
+- lib/http_utilities/jobs/resque/proxies/check_proxies_job.rb
+- lib/http_utilities/jobs/resque/proxies/check_proxy_job.rb
+- lib/http_utilities/jobs/sidekiq/proxies/check_proxies_job.rb
+- lib/http_utilities/jobs/sidekiq/proxies/check_proxy_job.rb
+- lib/http_utilities/proxies/proxy_checker.rb
+- lib/http_utilities/proxies/proxy_module.rb
+- lib/http_utilities/proxies/proxy_seeder.rb
+- lib/http_utilities/railtie.rb
+- lib/tasks/http_utilities_tasks.rake
+- spec/database.yml.example
+- spec/http_utilities/client_spec.rb
+- spec/http_utilities/mechanize_client_spec.rb
+- spec/http_utilities/proxy_checker_spec.rb
+- spec/http_utilities/proxy_seeder_spec.rb
+- spec/http_utilities/proxy_spec.rb
+- spec/models.rb
+- spec/schema.rb
+- spec/spec_helper.rb
+homepage: http://github.com/Agiley/http_utilities
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: 1.3.5
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 2
+summary: Wrapper for common Http Libraries (Net:HTTP/Open URI/Curl)
+test_files:
+- spec/http_utilities/client_spec.rb
+- spec/http_utilities/mechanize_client_spec.rb
+- spec/http_utilities/proxy_checker_spec.rb
+- spec/http_utilities/proxy_seeder_spec.rb
+- spec/http_utilities/proxy_spec.rb
+- spec/models.rb
+- spec/schema.rb
+- spec/spec_helper.rb