proxy_list 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 15582f80fed77a826d66efa6ef064c2b01c9aec6
-  data.tar.gz: f38d254e6a6e160390f658eb08f0865ef8914db2
+  metadata.gz: c51892cbf975069f356d92fb168b9d181b652daf
+  data.tar.gz: 1ad2d09cff9e82492fb64a69b938160cdcc2a96b
 SHA512:
-  metadata.gz: adf2b30b816991a17df8b7067d943eb7437c8a754aa422fbc4b734a05e0e5ab34b4686c00e240d2ff2093a4a373377e8a0382a68e1bd855ee44beed513f259c6
-  data.tar.gz: b94398f5ab833f9d3fd0d89e6aa899b4d442dacc3c94c182547c2f911eb6d04c6a45bb1386dc227cbab2808d706a3553bf545b76e8532836fead640788e5f8b2
+  metadata.gz: 50f7ab472df018cfdc13c01ad8d56ab3cfdeed29b668cbfbc1aaca5d9a92f19a55c1dea6a6c1dd2c88b7fa08ef4e595f2a8f67370ab4be670b78bc10e8f9b30e
+  data.tar.gz: 378ab040f2fbfbc14ae7cd16d195b6303ebcc6bf46e08f4bc995324d45835ef3975d5069594da903f85aa29eed4483a8356b487b535f38ee10d0830e0735be98

data/.gitignore

@@ -14,3 +14,4 @@
 mkmf.log
 .rbenv-gemsets
 *.gem
+log/

data/Gemfile

@@ -2,3 +2,6 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in proxy_list.gemspec
 gemspec
+#gem 'nokogiri', '~> 1.6.6.2'
+gem 'capybara', '~> 2.4.4'
+gem 'poltergeist', '~> 1.5.1'

data/lib/proxy_list.rb

@@ -1,5 +1,27 @@
+# encoding : utf-8
+#require 'open-uri'
+#require 'nokogiri'
+require 'capybara'
+require 'capybara/poltergeist'
 require "proxy_list/version"
+require "proxy_list/user_agents"
+require 'proxy_list/site_freeproxylists'
+require 'proxy_list/site_cybersyndrome'
+require 'proxy_list/site_spys'
 
+Capybara.javascript_driver = :poltergeist
+Capybara.register_driver :poltergeist do |app|
+  Capybara::Poltergeist::Driver.new(app,{:js_errors => false,:timeout => 1000,
+    :debug => false,:inspector => false,
+    :phantomjs_logger => File.open(File.expand_path('../../log/phantomjs.log',__FILE__),'a')
+    })
+end
 module ProxyList
-  # Your code goes here...
+
+  ##
+  # get proxy lists
+  # @params [String] country_code ISO two byte code
+  def self.get_lists(country_code='US')
+     proxy_lists = ProxyList::SiteFreeproxylists.new(country_code).proxy_lists
+  end
 end

data/lib/proxy_list/site_cybersyndrome.rb

@@ -0,0 +1,26 @@
+# encoding : utf-8
+module ProxyList
+  class SiteCybersyndrome
+    @url = "http://www.cybersyndrome.net/search.cgi?q=COUNTRY_CODE"
+    ##
+    # initialize
+    # @params [String] country_code
+    def initialize(country_code)
+      @url.gsub!(/COUNTRY_CODE/,country_code)
+    end
+
+    ##
+    # get proxy lists
+    # @return [Array] proxy_lists
+    def proxy_lists
+      # TODO for javascript site
+      html = open(url,{'User-Agent' => USER_AGENTS.sample}).read
+      doc = Nokogiri.HTML(html)
+      proxy_lists = []
+      doc.xpath("id('div_result')/table//tr/td[2]").each do |node|
+        proxy_lists.push(node.text)
+      end
+      proxy_lists
+    end
+  end
+end

data/lib/proxy_list/site_freeproxylists.rb

@@ -0,0 +1,29 @@
+# encoding : utf-8
+module ProxyList
+  class SiteFreeproxylists
+    @@url = "http://www.freeproxylists.net/ja/?c=COUNTRY_CODE&pt=&pr=&a[]=0&a[]=1&a[]=2&u=0"
+    ##
+    # initialize
+    # @params [String] country_code
+    def initialize(country_code)
+      @@url.gsub!(/COUNTRY_CODE/,country_code)
+    end
+
+    ##
+    # get proxy lists
+    # @return [Array] proxy_lists
+    def proxy_lists
+      session = Capybara::Session.new(:poltergeist)
+      session.driver.headers = {'User-Agent' => USER_AGENTS.sample}
+      session.visit(@@url)
+      proxy_lists = []
+      session.all(:xpath,"/html/body/div[1]/div[2]/table//tr[not(@class='Caption')]").each do |node|
+        if !node.text.nil? && node.has_xpath?('td[3]')
+          proxy_lists.push("#{node.find(:xpath,'td[3]').text.downcase}://#{node.find(:xpath,'td[1]').text}:#{node.find(:xpath,'td[2]').text}")
+        end
+      end
+      proxy_lists
+    end
+
+  end
+end

data/lib/proxy_list/site_spys.rb

@@ -0,0 +1,33 @@
+# encoding : utf-8
+module ProxyList
+  class SiteSpys
+    @url = "http://spys.ru/free-proxy-list/COUNTRY_CODE/"
+
+    ##
+    # initialize
+    # @params [String] country_code
+    def initialize(country_code)
+      @url.gsub!(/COUNTRY_CODE/,country_code)
+    end
+
+
+    ##
+    # get proxy lists
+    # @return [Array] proxy_lists
+    def proxy_lists
+      # TODO for javascript site
+      html = open(url,{'User-Agent' => USER_AGENTS.sample}).read
+      doc = Nokogiri.HTML(html)
+      proxy_lists = []
+      doc.xpath("/html/body/table[2]//tr[4]/td/table//tr[position() > 3]").each do |node|
+        if node.xpath('td[2]/font').text.include?('HTTP')
+          http = 'http'
+        else
+          http = 'https'
+        end
+        proxy_lists.push("#{http}://#{node.xpath('td[1]/font[2]').text}")
+      end
+      proxy_lists
+    end
+  end
+end

data/lib/proxy_list/user_agents.rb

@@ -0,0 +1,8 @@
+# encoding : utf-8
+module ProxyList
+  USER_AGENTS = [
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:34.0) Gecko/20100101 Firefox/34.0",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/7.1.2 Safari/537.85.11"
+  ]
+end

data/lib/proxy_list/version.rb

@@ -1,3 +1,3 @@
 module ProxyList
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/spec/proxy_list_spec.rb

@@ -5,7 +5,8 @@ describe ProxyList do
     expect(ProxyList::VERSION).not_to be nil
   end
 
-  it 'does something useful' do
-    expect(false).to eq(true)
+  it 'get proxy list' do
+    puts ProxyList.get_lists
+    expect(ProxyList.get_lists.count).to be > 0
   end
 end

data/spec/spec_helper.rb

@@ -1,2 +1,5 @@
 $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
 require 'proxy_list'
+#RSpec.configure do |config|
+#  config.output_stream = $stdout
+#end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: proxy_list
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - tatsunori_nishikori
@@ -68,7 +68,12 @@ files:
 - README.md
 - Rakefile
 - lib/proxy_list.rb
+- lib/proxy_list/site_cybersyndrome.rb
+- lib/proxy_list/site_freeproxylists.rb
+- lib/proxy_list/site_spys.rb
+- lib/proxy_list/user_agents.rb
 - lib/proxy_list/version.rb
+- log/.gitkeep
 - proxy_list.gemspec
 - spec/proxy_list_spec.rb
 - spec/spec_helper.rb