proxy_list 0.0.2 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c51892cbf975069f356d92fb168b9d181b652daf
-  data.tar.gz: 1ad2d09cff9e82492fb64a69b938160cdcc2a96b
+  metadata.gz: 85bfdd544c66b2e0982a0b91b68b99485fbf8a7f
+  data.tar.gz: c9fb305fca79235196a29f52da1ff8afa6a02da5
 SHA512:
-  metadata.gz: 50f7ab472df018cfdc13c01ad8d56ab3cfdeed29b668cbfbc1aaca5d9a92f19a55c1dea6a6c1dd2c88b7fa08ef4e595f2a8f67370ab4be670b78bc10e8f9b30e
-  data.tar.gz: 378ab040f2fbfbc14ae7cd16d195b6303ebcc6bf46e08f4bc995324d45835ef3975d5069594da903f85aa29eed4483a8356b487b535f38ee10d0830e0735be98
+  metadata.gz: 30d4e20af049309ee2ae65c8c7f16c07b90885418cd0933cb945ec967d54aaae59bfa13efb134b7284ce07e41b70ef0e3eba02feaf9e4c27bba7968cda702ea6
+  data.tar.gz: 05dffee7013d08d43323143da3fdc21a153ca0ab868bd40c13f1afab52a6f8405d787cee028a6997f161fcb599d1fb0497cbd837256e0ad9c1b83b4537491931

data/.coveralls.yml

@@ -0,0 +1,2 @@
+service_name: travis-pro
+repo_token: Q4HvzSOz25mLF0WIL7wTqk9TudN7q1OyG

data/.travis.yml

@@ -1,3 +1,12 @@
 language: ruby
 rvm:
-  - 2.1.2
+  - 1.9.3
+  - 2.0.0
+  - 2.1.3
+  - 2.1.4
+  - 2.1.5
+bundler_args: --jobs=2
+script: bundle exec rspec
+branches:
+  only:
+    - master

data/Gemfile

@@ -5,3 +5,4 @@ gemspec
 #gem 'nokogiri', '~> 1.6.6.2'
 gem 'capybara', '~> 2.4.4'
 gem 'poltergeist', '~> 1.5.1'
+gem 'coveralls', require: false

data/README.md

@@ -1,6 +1,11 @@
-# ProxyList
+# ProxyList 
+[![Build Status](https://travis-ci.org/tatsu07/proxy_list.svg?branch=master)](https://travis-ci.org/tatsu07/proxy_list) [![Coverage Status](https://coveralls.io/repos/tatsu07/proxy_list/badge.svg)](https://coveralls.io/r/tatsu07/proxy_list)
 
-TODO: Write a gem description
+Get Free Proxy List
+
+## Environment  
+
+・[phantomjs](http://phantomjs.org/)
 
 ## Installation
 
@@ -20,7 +25,15 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+The country of proxy you can get by specifying the ISO country code  
+
+```ruby
+require 'proxy_list'  
+#default US free proxy  
+proxy_list = ProxyList.get_lists  
+#useage country code
+proxy_list = ProxyList.get_lists('GB')
+```
 
 ## Contributing
 

data/lib/proxy_list.rb

@@ -9,7 +9,9 @@ require 'proxy_list/site_freeproxylists'
 require 'proxy_list/site_cybersyndrome'
 require 'proxy_list/site_spys'
 
+Capybara.current_driver = :poltergeist
 Capybara.javascript_driver = :poltergeist
+Capybara.run_server = false
 Capybara.register_driver :poltergeist do |app|
   Capybara::Poltergeist::Driver.new(app,{:js_errors => false,:timeout => 1000,
     :debug => false,:inspector => false,
@@ -23,5 +25,8 @@ module ProxyList
   # @params [String] country_code ISO two byte code
   def self.get_lists(country_code='US')
      proxy_lists = ProxyList::SiteFreeproxylists.new(country_code).proxy_lists
+     proxy_lists.concat(ProxyList::SiteSpys.new(country_code).proxy_lists)
+     proxy_lists.concat(ProxyList::SiteCybersyndrome.new(country_code).proxy_lists)
+     proxy_lists.uniq
   end
 end

data/lib/proxy_list/site_cybersyndrome.rb

@@ -1,25 +1,29 @@
 # encoding : utf-8
 module ProxyList
   class SiteCybersyndrome
-    @url = "http://www.cybersyndrome.net/search.cgi?q=COUNTRY_CODE"
+    @@url = "http://www.cybersyndrome.net/search.cgi?q=COUNTRY_CODE"
     ##
     # initialize
     # @params [String] country_code
     def initialize(country_code)
-      @url.gsub!(/COUNTRY_CODE/,country_code)
+      @@url.gsub!(/COUNTRY_CODE/,country_code)
     end
 
     ##
     # get proxy lists
     # @return [Array] proxy_lists
     def proxy_lists
-      # TODO for javascript site
-      html = open(url,{'User-Agent' => USER_AGENTS.sample}).read
-      doc = Nokogiri.HTML(html)
+      session = Capybara::Session.new(:poltergeist)
+      session.driver.headers = {'User-Agent' => USER_AGENTS.sample}
+      session.visit(@@url)
       proxy_lists = []
-      doc.xpath("id('div_result')/table//tr/td[2]").each do |node|
-        proxy_lists.push(node.text)
+      session.all(:xpath,"id('div_result')/table//tr/td[2]").each do |node|
+        if !node.text.nil?
+          proxy_lists.push("http://#{node.text}")
+        end
       end
+      Capybara.reset_sessions!
+      session.driver.quit
       proxy_lists
     end
   end

data/lib/proxy_list/site_freeproxylists.rb

@@ -22,6 +22,8 @@ module ProxyList
           proxy_lists.push("#{node.find(:xpath,'td[3]').text.downcase}://#{node.find(:xpath,'td[1]').text}:#{node.find(:xpath,'td[2]').text}")
         end
       end
+      Capybara.reset_sessions!
+      session.driver.quit
       proxy_lists
     end
 

data/lib/proxy_list/site_spys.rb

@@ -1,13 +1,13 @@
 # encoding : utf-8
 module ProxyList
   class SiteSpys
-    @url = "http://spys.ru/free-proxy-list/COUNTRY_CODE/"
+    @@url = "http://spys.ru/free-proxy-list/COUNTRY_CODE/"
 
     ##
     # initialize
     # @params [String] country_code
     def initialize(country_code)
-      @url.gsub!(/COUNTRY_CODE/,country_code)
+      @@url.gsub!(/COUNTRY_CODE/,country_code)
     end
 
 
@@ -15,18 +15,22 @@ module ProxyList
     # get proxy lists
     # @return [Array] proxy_lists
     def proxy_lists
-      # TODO for javascript site
-      html = open(url,{'User-Agent' => USER_AGENTS.sample}).read
-      doc = Nokogiri.HTML(html)
+      session = Capybara::Session.new(:poltergeist)
+      session.driver.headers = {'User-Agent' => USER_AGENTS.sample}
+      session.visit(@@url)
       proxy_lists = []
-      doc.xpath("/html/body/table[2]//tr[4]/td/table//tr[position() > 3]").each do |node|
-        if node.xpath('td[2]/font').text.include?('HTTP')
-          http = 'http'
-        else
-          http = 'https'
+      session.all(:xpath,"//table[2]//tr[4]/td/table//tr[position()>3]").each do |node|
+        if !node.text.nil? && node.has_xpath?('td[2]/font')
+          if node.find(:xpath,'td[2]/font').text.include?('HTTP')
+            http = 'http'
+          else
+            http = 'https'
+          end
+          proxy_lists.push("#{http}://#{node.find(:xpath,'td[1]/font[2]').text}")
         end
-        proxy_lists.push("#{http}://#{node.xpath('td[1]/font[2]').text}")
       end
+      Capybara.reset_sessions!
+      session.driver.quit
       proxy_lists
     end
   end

data/lib/proxy_list/version.rb

@@ -1,3 +1,3 @@
 module ProxyList
-  VERSION = "0.0.2"
+  VERSION = "1.0.0"
 end

data/proxy_list.gemspec

@@ -21,4 +21,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.7"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec"
+  spec.add_development_dependency "capybara",'~> 2.4.4'
+  spec.add_development_dependency 'poltergeist', '~> 1.5.1'
 end

data/spec/proxy_list_spec.rb

@@ -8,5 +8,21 @@ describe ProxyList do
   it 'get proxy list' do
     puts ProxyList.get_lists
     expect(ProxyList.get_lists.count).to be > 0
+    puts ProxyList.get_lists('GB')
+    expect(ProxyList.get_lists.count).to be > 0
+    puts ProxyList.get_lists('JP')
+    expect(ProxyList.get_lists.count).to be > 0
+    puts ProxyList.get_lists('KR')
+    expect(ProxyList.get_lists.count).to be > 0
+#    puts ProxyList.get_lists('TW')
+#    expect(ProxyList.get_lists.count).to be > 0
+#    puts ProxyList.get_lists('HK')
+#    expect(ProxyList.get_lists.count).to be > 0
+#    puts ProxyList.get_lists('CN')
+#    expect(ProxyList.get_lists.count).to be > 0
+#    puts ProxyList.get_lists('CA')
+#    expect(ProxyList.get_lists.count).to be > 0
+#    puts ProxyList.get_lists('FR')
+#    expect(ProxyList.get_lists.count).to be > 0
   end
 end

data/spec/spec_helper.rb

@@ -1,5 +1,4 @@
 $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
 require 'proxy_list'
-#RSpec.configure do |config|
-#  config.output_stream = $stdout
-#end
+require 'coveralls'
+Coveralls.wear!

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: proxy_list
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 1.0.0
 platform: ruby
 authors:
 - tatsunori_nishikori
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-05 00:00:00.000000000 Z
+date: 2015-02-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -52,6 +52,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: capybara
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.4.4
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.4.4
+- !ruby/object:Gem::Dependency
+  name: poltergeist
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.5.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.5.1
 description: get free proxy list
 email:
 - tora.1986.tatsu@gmail.com
@@ -59,6 +87,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".coveralls.yml"
 - ".gitignore"
 - ".rspec"
 - ".ruby-version"