proxy_fetcher 0.1.4 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9b5976adc9dfa09ec920347577b06e900c094a6d
-  data.tar.gz: 1ffb7978312183ffffe89ec596426e2c64a4d51f
+  metadata.gz: 04cf23c5f1bb6abfd29e5d6180a5a9e167a2d147
+  data.tar.gz: 5590d391eee582e511027a8282ef38917aa653a9
 SHA512:
-  metadata.gz: 0b65e73a438263f073e344550a87f438c0045f5b521b3e063c292a72838cbbf8a316b48dc0d47aacba797fb3a7244db03cbe8288e78f104a1681ad402fcd3a59
-  data.tar.gz: 2983502c4ba43fe5db8b9496f4ee973c78c13eb0fb4d8cd8dfafc21bd96568a19724423d8f517ff2f239c5742d764ac1c256463455624c96d43067e18f964716
+  metadata.gz: ccd0ef56339916919c8bf1b70dd3332d9972d6db1cc5b676b28b16691161955cffafbfa25bb310ba5b1d78a7b02dc7e8e440bb0a019736442b8bcbcfa3f8a004
+  data.tar.gz: e72892832780fddb424adc29dcb3b500ede2998d452c333aeffcb2a4766483a079288ec3e52e1eeca4ff61f7801ddae3ba80cddac2d527eaeabed09cbd8f6736

data/README.md

@@ -15,6 +15,12 @@ If using bundler, first add 'proxy_fetcher' to your Gemfile:
 gem 'proxy_fetcher', '~> 0.1'
 ```
 
+or if you want to use the latest version (from `master` branch), then:
+
+```ruby
+gem 'proxy_fetcher', git: 'https://github.com/nbulaj/proxy_fetcher.git'
+```
+
 And run:
 
 ```sh
@@ -88,8 +94,8 @@ Also you can call next instance method for every Proxy object:
 
 You can use two methods to get the first proxy from the list:
 
-* `get` (will return first proxy and move it to the end of the list)
-* `get!` (will return first **connectable** proxy and move it to the end of the list; all the proxies till the working one will be removed)
+* `get` or aliased `pop` (will return first proxy and move it to the end of the list)
+* `get!` or aliased `pop!` (will return first **connectable** proxy and move it to the end of the list; all the proxies till the working one will be removed)
 
 If you wanna clear current proxy manager list from dead servers, you can just call `cleanup!` method:
 
@@ -97,6 +103,12 @@ If you wanna clear current proxy manager list from dead servers, you can just ca
 manager.cleanup! # or manager.validate!
 ```
 
+You can sort or find any proxy by speed using next 3 instance methods:
+
+* `fast?`
+* `medium?`
+* `slow?`'
+
 To change open/read timeout for `cleanup!` and `connectable?` methods yu need to change ProxyFetcher::Manager config:
 
 ```ruby
@@ -123,7 +135,7 @@ To contribute:
 4. Add documentation for your feature or bug fix.
 5. Run <tt>rake doc:yard</tt>. If your changes are not 100% documented, go back to step 4.
 6. Add tests for your feature or bug fix.
-7. Run `rake` to make sure all tests pass.
+7. Run `rake spec` to make sure all tests pass.
 8. Commit your changes (`git commit -am 'Add new feature'`).
 9. Push to the branch (`git push origin my-new-feature`).
 10. Create new pull request.
@@ -135,3 +147,5 @@ Thanks.
 proxy_fetcher gem is released under the [MIT License](http://www.opensource.org/licenses/MIT).
 
 Copyright (c) 2017 Nikita Bulai (bulajnikita@gmail.com).
+
+Some parser code (c) [pifleo](https://gist.github.com/pifleo/3889803)

data/lib/proxy_fetcher.rb

@@ -4,92 +4,7 @@ require 'nokogiri'
 
 require 'proxy_fetcher/configuration'
 require 'proxy_fetcher/proxy'
+require 'proxy_fetcher/manager'
 
 module ProxyFetcher
-  class Manager
-    PROXY_PROVIDER_URL = 'http://proxylist.hidemyass.com/'.freeze
-
-    class << self
-      def config
-        @config ||= ProxyFetcher::Configuration.new
-      end
-    end
-
-    attr_reader :proxies
-
-    # refresh: true - load proxy list from the remote server on initialization
-    # refresh: false - just initialize the class, proxy list will be empty ([])
-    def initialize(refresh: true)
-      if refresh
-        refresh_list!
-      else
-        @proxies = []
-      end
-    end
-
-    # Update current proxy list from the provider
-    def refresh_list!
-      doc = Nokogiri::HTML(load_html(PROXY_PROVIDER_URL))
-      rows = doc.xpath('//table[@id="listable"]/tbody/tr')
-
-      @proxies = rows.map { |row| Proxy.new(row) }
-    end
-
-    alias_method :fetch!, :refresh_list!
-
-    # Pop just first proxy (and back it to the end of the proxy list)
-    def get
-      return if @proxies.empty?
-
-      first_proxy = @proxies.shift
-      @proxies << first_proxy
-
-      first_proxy
-    end
-
-    alias_method :pop, :get
-
-    # Pop first valid proxy (and back it to the end of the proxy list)
-    # Invalid proxies will be removed from the list
-    def get!
-      index = @proxies.find_index(&:connectable?)
-      return if index.nil?
-
-      proxy = @proxies.delete_at(index)
-      tail = @proxies[index..-1]
-
-      @proxies = tail << proxy
-
-      proxy
-    end
-
-    alias_method :pop!, :get!
-
-    # Clean current proxy list from dead proxies (doesn't respond by timeout)
-    def cleanup!
-      proxies.keep_if(&:connectable?)
-    end
-
-    alias_method :validate!, :cleanup!
-
-    # Just schema + host + port
-    def raw_proxies
-      proxies.map(&:url)
-    end
-
-    # No need to put all the attr_readers
-    def inspect
-      to_s
-    end
-
-    private
-
-    # Get HTML from the requested URL
-    def load_html(url)
-      uri = URI.parse(url)
-      http = Net::HTTP.new(uri.host, uri.port)
-      response = http.get(uri.request_uri)
-      response.body
-    end
-  end
 end

data/lib/proxy_fetcher/manager.rb

@@ -0,0 +1,88 @@
+module ProxyFetcher
+  class Manager
+    PROXY_PROVIDER_URL = 'http://proxylist.hidemyass.com/'.freeze
+
+    class << self
+      def config
+        @config ||= ProxyFetcher::Configuration.new
+      end
+    end
+
+    attr_reader :proxies
+
+    # refresh: true - load proxy list from the remote server on initialization
+    # refresh: false - just initialize the class, proxy list will be empty ([])
+    def initialize(refresh: true)
+      if refresh
+        refresh_list!
+      else
+        @proxies = []
+      end
+    end
+
+    # Update current proxy list from the provider
+    def refresh_list!
+      doc = Nokogiri::HTML(load_html(PROXY_PROVIDER_URL))
+      rows = doc.xpath('//table[@id="listable"]/tbody/tr')
+
+      @proxies = rows.map { |row| Proxy.new(row) }
+    end
+
+    alias fetch! refresh_list!
+
+    # Pop just first proxy (and back it to the end of the proxy list)
+    def get
+      return if @proxies.empty?
+
+      first_proxy = @proxies.shift
+      @proxies << first_proxy
+
+      first_proxy
+    end
+
+    alias pop get
+
+    # Pop first valid proxy (and back it to the end of the proxy list)
+    # Invalid proxies will be removed from the list
+    def get!
+      index = @proxies.find_index(&:connectable?)
+      return if index.nil?
+
+      proxy = @proxies.delete_at(index)
+      tail = @proxies[index..-1]
+
+      @proxies = tail << proxy
+
+      proxy
+    end
+
+    alias pop! get!
+
+    # Clean current proxy list from dead proxies (doesn't respond by timeout)
+    def cleanup!
+      proxies.keep_if(&:connectable?)
+    end
+
+    alias validate! cleanup!
+
+    # Just schema + host + port
+    def raw_proxies
+      proxies.map(&:url)
+    end
+
+    # No need to put all the attr_readers
+    def inspect
+      to_s
+    end
+
+    private
+
+    # Get HTML from the requested URL
+    def load_html(url)
+      uri = URI.parse(url)
+      http = Net::HTTP.new(uri.host, uri.port)
+      response = http.get(uri.request_uri)
+      response.body
+    end
+  end
+end

data/lib/proxy_fetcher/proxy.rb

@@ -22,7 +22,7 @@ module ProxyFetcher
       false
     end
 
-    alias_method :valid?, :connectable?
+    alias valid? connectable?
 
     def http?
       type.casecmp('http').zero?
@@ -32,6 +32,18 @@ module ProxyFetcher
       type.casecmp('https').zero?
     end
 
+    def fast?
+      speed >= 66
+    end
+
+    def medium?
+      speed >= 33 && speed < 66
+    end
+
+    def slow?
+      speed < 33
+    end
+
     def uri
       URI::Generic.build(host: addr, port: port, scheme: type)
     end

data/lib/proxy_fetcher/version.rb

@@ -9,7 +9,7 @@ module ProxyFetcher
     # Minor version number
     MINOR = 1
     # Smallest version number
-    TINY  = 4
+    TINY  = 5
 
     # Full version number
     STRING = [MAJOR, MINOR, TINY].compact.join('.')

data/proxy_fetcher.gemspec

@@ -5,7 +5,7 @@ require 'proxy_fetcher/version'
 Gem::Specification.new do |gem|
   gem.name = 'proxy_fetcher'
   gem.version = ProxyFetcher.gem_version
-  gem.date = '2017-05-25'
+  gem.date = '2017-05-31'
   gem.summary = 'Ruby gem for dealing with proxy lists '
   gem.description = 'This gem can help your Ruby application to make HTTP(S) requests ' \
                     'from proxy server, fetching and validating current proxy lists from the HideMyAss service.'

data/spec/proxy_fetcher/proxy_spec.rb

@@ -8,8 +8,13 @@ describe ProxyFetcher::Proxy do
   let(:proxy) { @manager.proxies.first }
 
   it 'checks schema' do
-    expect(proxy.http?).to be_falsey.or(be_truthy)
-    expect(proxy.https?).to be_falsey.or(be_truthy)
+    proxy.instance_variable_set(:@type, 'HTTP')
+    expect(proxy.http?).to be_truthy
+    expect(proxy.https?).to be_falsey
+
+    proxy.instance_variable_set(:@type, 'HTTPS')
+    expect(proxy.https?).to be_truthy
+    expect(proxy.http?).to be_falsey
   end
 
   it 'not connectable if IP addr is wrong' do
@@ -20,6 +25,7 @@ describe ProxyFetcher::Proxy do
   it "not connectable if server doesn't respond to head" do
     allow_any_instance_of(Net::HTTP).to receive(:start).and_return(false)
     expect(proxy.connectable?).to be_falsey
+    expect(proxy.valid?).to be_falsey
   end
 
   it 'returns URI::Generic' do
@@ -29,4 +35,31 @@ describe ProxyFetcher::Proxy do
   it 'returns URL' do
     expect(proxy.url).to be_a(String)
   end
+
+  it 'must be slow if speed < 33' do
+    proxy.instance_variable_set(:@speed, 32)
+    expect(proxy.slow?).to be_truthy
+
+    proxy.instance_variable_set(:@speed, 33)
+    expect(proxy.slow?).to be_falsey
+  end
+
+  it 'must be medium if speed >= 33 and < 66' do
+    proxy.instance_variable_set(:@speed, 32)
+    expect(proxy.medium?).to be_falsey
+
+    proxy.instance_variable_set(:@speed, 33)
+    expect(proxy.medium?).to be_truthy
+
+    proxy.instance_variable_set(:@speed, 65)
+    expect(proxy.medium?).to be_truthy
+  end
+
+  it 'must be fast if speed >= 66' do
+    proxy.instance_variable_set(:@speed, 65)
+    expect(proxy.fast?).to be_falsey
+
+    proxy.instance_variable_set(:@speed, 66)
+    expect(proxy.fast?).to be_truthy
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: proxy_fetcher
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
 platform: ruby
 authors:
 - Nikita Bulai
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-25 00:00:00.000000000 Z
+date: 2017-05-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -59,6 +59,7 @@ files:
 - Rakefile
 - lib/proxy_fetcher.rb
 - lib/proxy_fetcher/configuration.rb
+- lib/proxy_fetcher/manager.rb
 - lib/proxy_fetcher/proxy.rb
 - lib/proxy_fetcher/version.rb
 - proxy_fetcher.gemspec