http_utilities 1.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b148d1430516c285cb9cf4fc58712acabd0d268c
+  data.tar.gz: 8a18be0594fca823f8abfe88ac54a194bf9b9602
+SHA512:
+  metadata.gz: 4ee2b56ece9840c4160969ea00f7f79611bbdae4cfe77702bb21c18f726c6d3b6fbb6fd8978ffdfa3bf0b82b28e4accd3c514126ef3ad16b4b3afd5722f7abcc
+  data.tar.gz: 3541bf3e7c7e79183c14636c5aec50c626494cc20c43c97c4d3451e4ab882881342cb2314bb717bc1be7b409cfb9fa319e4c063f91b9ef829678166c7fb77ad0

data/Gemfile

@@ -0,0 +1,22 @@
+source "http://rubygems.org"
+
+gem "nokogiri", ">= 1.5.5"
+gem "mechanize", ">= 2.5"
+gem "multi_xml", ">= 0.5"
+
+gem "activerecord-import", :require => false
+
+platforms :ruby do
+  gem 'curb'
+end
+
+group :development, :test do
+  gem 'rails'
+  gem 'jeweler'
+  gem 'rspec'
+  gem 'sqlite3'
+  
+  platforms :ruby do
+    gem "mysql2", ">= 0.3.11"
+  end
+end

data/README

@@ -0,0 +1,15 @@
+===== Http-Utilities =====
+
+Http-Utilities is a wrapper for common HTTP-libraries (Net::Http, Open Uri, Curb) using a unified API.
+
+I personally use this in quite a few projects and decided to extract it into a separate gem.
+
+Features:
+- Support for Net:Http, Open Uri, Curb
+- Proxy support (adds a proxy model). Enables you to randomly use different proxies on every request.
+- Proxy checker (comes with Resque jobs for scheduling)
+- Yaml-file containing a few thousand User Agents that are randomly used on every request.
+
+There are some specs but the gem needs more test/spec coverage. Working on it though.
+
+This is primarily intended for private use, but get back to me if you use it and run into issues.

data/Rakefile

@@ -0,0 +1,87 @@
+## helper functions
+
+def name
+  @name ||= Dir['*.gemspec'].first.split('.').first
+end
+
+def version
+  line = File.read("lib/#{name}.rb")[/^\s*VERSION\s*=\s*.*/]
+  line.match(/.*VERSION\s*=\s*['"](.*)['"]/)[1]
+end
+
+def gemspec_file
+  "#{name}.gemspec"
+end
+
+def gem_file
+  "#{name}-#{version}.gem"
+end
+
+def replace_header(head, header_name)
+  head.sub!(/(\.#{header_name}\s*= ').*'/) { "#{$1}#{send(header_name)}'"}
+end
+
+begin
+  # Rspec 2.0
+  require 'rspec/core/rake_task'
+
+  desc 'Default: run specs'
+  task :default => :spec
+  RSpec::Core::RakeTask.new do |t|
+    t.pattern = "spec/**/*_spec.rb"
+  end
+
+  RSpec::Core::RakeTask.new('rcov') do |t|
+    t.pattern = "spec/**/*_spec.rb"
+    t.rcov = true
+    t.rcov_opts = ['--exclude', 'spec']
+  end
+
+rescue LoadError
+  puts "Rspec not available. Install it with: gem install rspec"
+end
+
+## release management tasks
+
+desc "Commit, create tag v#{version} and build and push #{gem_file} to Rubygems"
+task :release => :build do
+  sh "git commit --allow-empty -a -m 'Release #{version}'"
+  sh "git tag v#{version}"
+  sh "git push"
+  sh "git push origin v#{version}"
+  sh "gem push pkg/#{gem_file}"
+end
+
+desc "Build #{gem_file} into the pkg directory"
+task :build => :gemspec do
+  sh "mkdir -p pkg"
+  sh "gem build #{gemspec_file}"
+  sh "mv #{gem_file} pkg"
+end
+
+desc "Generate #{gemspec_file}"
+task :gemspec do
+  # read spec file and split out manifest section
+  spec = File.read(gemspec_file)
+  head, manifest, tail = spec.split(" # = MANIFEST =\n")
+
+  # replace name version and date
+  replace_header(head, :name)
+  replace_header(head, :version)
+
+  # determine file list from git ls-files
+  files = `git ls-files`.
+    split("\n").
+    sort.
+    reject { |file| file =~ /^\./ }.
+    reject { |file| file =~ /^(rdoc|pkg)/ }.
+    map { |file| " #{file}" }.
+    join("\n")
+
+  # piece file back together and write
+  manifest = " s.files = %w[\n#{files}\n ]\n"
+  spec = [head, manifest, tail].join(" # = MANIFEST =\n")
+  File.open(gemspec_file, 'w') { |io| io.write(spec) }
+  puts "Updated #{gemspec_file}"
+end
+

data/VERSION

@@ -0,0 +1 @@
+0.0.5

data/http_utilities.gemspec

@@ -0,0 +1,78 @@
+Gem::Specification.new do |s|
+  s.specification_version     = 2 if s.respond_to? :specification_version=
+  s.required_rubygems_version = Gem::Requirement.new(">= 1.3.5") if s.respond_to? :required_rubygems_version=
+  
+  s.name = "http_utilities"
+  s.version = "1.0.1"
+
+  s.authors = ["Sebastian Johnsson"]
+  s.date = "2012-11-22"
+  s.description = "Wrapper for common Http Libraries (Net:HTTP/Open URI/Curl)"
+
+  s.homepage = "http://github.com/Agiley/http_utilities"
+  s.summary = "Wrapper for common Http Libraries (Net:HTTP/Open URI/Curl)"
+  
+  s.add_dependency(%q<nokogiri>, [">= 1.5.5"])
+  s.add_dependency(%q<mechanize>, [">= 2.5"])
+  s.add_dependency(%q<multi_xml>, [">= 0.5"])
+  s.add_dependency(%q<activerecord-import>, [">= 0"])
+  
+  s.add_development_dependency(%q<rails>, [">= 0"])
+  s.add_development_dependency(%q<rspec>, [">= 0"])
+  s.add_development_dependency(%q<sqlite3>, [">= 0"])
+  s.add_development_dependency(%q<mysql2>, [">= 0.3.11"])
+  
+  # = MANIFEST =
+ s.files = %w[
+ Gemfile
+ README
+ Rakefile
+ VERSION
+ http_utilities.gemspec
+ lib/generators/active_record/http_utilities_generator.rb
+ lib/generators/active_record/templates/migration.rb
+ lib/generators/active_record/templates/proxy.rb
+ lib/generators/helpers/file_helper.rb
+ lib/generators/helpers/orm_helpers.rb
+ lib/generators/http_utilities/http_utilities_generator.rb
+ lib/generators/templates/http_utilities.rb
+ lib/generators/templates/user_agents.yml
+ lib/http_utilities.rb
+ lib/http_utilities/http/adapters/curb.rb
+ lib/http_utilities/http/adapters/net_http.rb
+ lib/http_utilities/http/adapters/open_uri.rb
+ lib/http_utilities/http/client.rb
+ lib/http_utilities/http/cookies.rb
+ lib/http_utilities/http/format.rb
+ lib/http_utilities/http/get.rb
+ lib/http_utilities/http/logger.rb
+ lib/http_utilities/http/mechanize/client.rb
+ lib/http_utilities/http/post.rb
+ lib/http_utilities/http/proxy_support.rb
+ lib/http_utilities/http/request.rb
+ lib/http_utilities/http/response.rb
+ lib/http_utilities/http/url.rb
+ lib/http_utilities/http/user_agent.rb
+ lib/http_utilities/jobs/resque/proxies/check_proxies_job.rb
+ lib/http_utilities/jobs/resque/proxies/check_proxy_job.rb
+ lib/http_utilities/jobs/sidekiq/proxies/check_proxies_job.rb
+ lib/http_utilities/jobs/sidekiq/proxies/check_proxy_job.rb
+ lib/http_utilities/proxies/proxy_checker.rb
+ lib/http_utilities/proxies/proxy_module.rb
+ lib/http_utilities/proxies/proxy_seeder.rb
+ lib/http_utilities/railtie.rb
+ lib/tasks/http_utilities_tasks.rake
+ spec/database.yml.example
+ spec/http_utilities/client_spec.rb
+ spec/http_utilities/mechanize_client_spec.rb
+ spec/http_utilities/proxy_checker_spec.rb
+ spec/http_utilities/proxy_seeder_spec.rb
+ spec/http_utilities/proxy_spec.rb
+ spec/models.rb
+ spec/schema.rb
+ spec/spec_helper.rb
+ ]
+ # = MANIFEST =
+   
+   s.test_files = s.files.select { |path| path =~ %r{^spec/*/.+\.rb} }
+end 

data/lib/generators/active_record/http_utilities_generator.rb

@@ -0,0 +1,21 @@
+require 'rails/generators/active_record'
+require 'generators/helpers/orm_helpers'
+
+module ActiveRecord
+  module Generators
+    class HttpUtilitiesGenerator < ActiveRecord::Generators::Base
+      
+      include HttpUtilities::Generators::OrmHelpers
+      source_root File.expand_path("../templates", __FILE__)
+      
+      def copy_proxy_migration
+        migration_template "migration.rb", "db/migrate/create_proxies" unless model_exists?('proxy') && behavior == :invoke
+      end
+      
+      def copy_proxy_model
+        template "proxy.rb", "app/models/proxy.rb" unless model_exists?('proxy') && behavior == :invoke
+      end
+
+    end
+  end
+end

data/lib/generators/active_record/templates/migration.rb

@@ -0,0 +1,34 @@
+class CreateProxies < ActiveRecord::Migration
+  def self.up
+    create_table :proxies do |t|
+      
+      t.string :host, :null => false
+      t.integer :port, :null => false
+      t.string :username
+      t.string :password
+      
+      t.string :protocol, :null => false, :default => 'http'
+      t.string :proxy_type, :null => false, :defaut => 'public'
+      t.string :category
+      
+      t.datetime :last_checked_at
+      t.boolean :valid_proxy, :null => false, :default => false
+      t.integer :successful_attempts, :null => false, :default => 0
+      t.integer :failed_attempts, :null => false, :default => 0
+
+      t.timestamps
+    end
+    
+    add_index :proxies, [:host, :port], :unique => true, :name => 'index_unique_proxy'
+    add_index :proxies, :protocol, :name => 'index_protocol'
+    add_index :proxies, :proxy_type, :name => 'index_proxy_type'
+    add_index :proxies, :category, :name => 'index_category'
+    add_index :proxies, :valid_proxy, :name => 'index_valid_proxy'
+    add_index :proxies, :successful_attempts, :name => 'index_successful_attempts'
+    add_index :proxies, :failed_attempts, :name => 'index_failed_attempts'
+  end
+
+  def self.down
+    drop_table :proxies
+  end
+end

data/lib/generators/active_record/templates/proxy.rb

@@ -0,0 +1,3 @@
+class Proxy < ActiveRecord::Base
+  include HttpUtilities::Proxies::ProxyModule  
+end

data/lib/generators/helpers/file_helper.rb

@@ -0,0 +1,35 @@
+module HttpUtilities
+  module Generators
+    module FileHelper
+
+      private
+
+      def copy_dir(source, destination)
+        root_dir=File.join(self.class.source_root, source)
+        Dir[File.join(root_dir, "**/*")].each do |file|
+          relative = file.gsub(/^#{root_dir}\//, '')
+          copy_file file, File.join(Rails.root, destination, relative) if File.file?(file)
+        end
+      end
+      
+      def append_to_file(source, destination, prepend_with = "\n")
+        source_file = File.join(self.class.source_root, source)
+        destination_file = File.join(Rails.root, destination)
+        
+        if (File.exists?(source_file) && File.exists?(destination_file))
+          source_data = []
+          File.open(source_file, 'r') {|f| source_data = f.readlines("\n") }
+          open(destination_file, 'a') { |dest_file|
+            dest_file << prepend_with if (prepend_with)
+            source_data.each { |line| dest_file.puts line }
+          }
+        end
+      end
+
+      def file_exists? path
+        File.exists?(File.join(destination_root, path))
+      end
+      
+    end
+  end
+end

data/lib/generators/helpers/orm_helpers.rb

@@ -0,0 +1,15 @@
+module HttpUtilities
+  module Generators
+    module OrmHelpers
+      
+      def model_exists?(model)
+        File.exists?(File.join(destination_root, model_path(model)))
+      end
+
+      def model_path(model)
+        @model_path ||= File.join("app", "models", "#{model}.rb")
+      end
+      
+    end
+  end
+end

data/lib/generators/http_utilities/http_utilities_generator.rb

@@ -0,0 +1,25 @@
+require 'generators/helpers/file_helper'
+
+module HttpUtilities
+  module Generators
+    class HttpUtilitiesGenerator < Rails::Generators::Base
+      include HttpUtilities::Generators::FileHelper
+      namespace "http_utilities"
+      source_root File.expand_path("../../templates", __FILE__)
+      
+      class_option :orm
+      hook_for :orm
+            
+      desc "Copies an initializer, a .yml-file containing user-agents as well as a proxy model."
+      
+      #def copy_initializer
+      #  template "http_utilities.rb", "config/initializers/http_utilities.rb" unless file_exists?("config/initializers/http_utilities.rb")
+      #end
+      
+      #def copy_user_agents
+      #  template "user_agents.yml", "config/http_utilities/user_agents.yml" unless file_exists?("config/http_utilities/user_agents.yml")
+      #end
+      
+    end
+  end
+end

data/lib/generators/templates/http_utilities.rb

@@ -0,0 +1,2 @@
+HttpUtilities.setup do |config|
+end