proxy_fetcher 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 929248449265daefef4079decba21be77d36c610
-  data.tar.gz: 1efc0967583b3a1cc3164b5a524d1b9cfa6e4a1e
+  metadata.gz: dac8f3b98762e29d5067e32689cef73be6905d4b
+  data.tar.gz: 5b488bb4180524c2cf64640aa031c9b2604c81cc
 SHA512:
-  metadata.gz: 9c12cd3940d6f81a7a9ee462d8b9e4a8e364013a735172c1ca9444a2d69741f96642de532384cd1de8d6a4f1e77daa038048b6a1140c9e835e2a9f70d28d3a8c
-  data.tar.gz: 3e01791ebbdff7608f206a85a86cc7a7701b830aad425475f47fb1d14b4bd11c32225a279a72a131b40b1dbd441973da0b2f5fb4e9381da0c09caa9cfeeb129e
+  metadata.gz: aa4ff379c85314d81e660c84e65f9772a72b5a0d36110fcda2cff37d4ced8745cc82378bc98d290ab78c4c51868b201504a1067e2741ca3385d285a36bc525a8
+  data.tar.gz: 2fbc3811ca622aadda97e235b650a8603367fd4851d5a5319e71f994cc4b7520e2e88cbc45d4299c3fd74b58a569d0bf762e35c16eba1aaa9da01b1553759c3d

data/README.md

@@ -1,4 +1,5 @@
 # Ruby lib for managing proxies
+[![Gem Version](https://badge.fury.io/rb/proxy_fetcher.svg)](http://badge.fury.io/rb/proxy_fetcher)
 [![Build Status](https://travis-ci.org/nbulaj/proxy_fetcher.svg?branch=master)](https://travis-ci.org/nbulaj/proxy_fetcher)
 [![Coverage Status](https://coveralls.io/repos/github/nbulaj/proxy_fetcher/badge.svg)](https://coveralls.io/github/nbulaj/proxy_fetcher)
 [![License](http://img.shields.io/badge/license-MIT-brightgreen.svg)](#license)
@@ -38,6 +39,15 @@ manager.proxies
  #     @response_time=5217, @speed=48, @connection_time=100, @type="HTTP", @anonymity="High">, ... ]
 ```
 
+You can initialize proxy manager without loading proxy list from the remote server by passing `refresh: false` on initialization:
+
+```ruby
+manager = ProxyFetcher::Manager.new(refresh: false) # just initialize class instance
+manager.proxies
+
+ #=> []
+```
+
 Get raw proxy URLs:
 
 ```ruby
@@ -51,7 +61,7 @@ manager.raw_proxies
 If `ProxyFetcher::Manager` was already initialized somewhere, you can refresh the proxy list by calling `#refresh_list!` method:
 
 ```ruby
-manager.refresh_list!
+manager.refresh_list! # or manager.fetch!
 
  #=> [#<ProxyFetcher::Proxy:0x00000002879680 @addr="97.77.104.22", @port=3128, @country="USA", 
  #     @response_time=5217, @speed=48, @connection_time=100, @type="HTTP", @anonymity="High">, ... ]
@@ -79,7 +89,7 @@ Also you can call next instance method for every Proxy object:
 If you wanna clear current proxy manager list from dead servers, you can just call `cleanup!` method:
 
 ```ruby
-manager.cleanup!
+manager.cleanup! # or manager.validate!
 ```
 
 To change open/read timeout for `cleanup!` and `connectable?` methods yu need to change ProxyFetcher::Manager config:
@@ -92,6 +102,10 @@ manager = ProxyFetcher::Manager.new
 manager.cleanup!
 ```
 
+## TODO
+
+# Proxy filters
+
 ## Contributing
 
 You are very welcome to help improve ProxyFetcher if you have suggestions for features that other people can use.

data/lib/proxy_fetcher.rb

@@ -37,7 +37,7 @@ module ProxyFetcher
 
     alias_method :fetch!, :refresh_list!
 
-    # Clean current proxies list from dead proxies
+    # Clean current proxy list from dead proxies (doesn't respond by timeout)
     def cleanup!
       proxies.keep_if(&:connectable?)
     end
@@ -56,6 +56,7 @@ module ProxyFetcher
 
     private
 
+    # Get HTML from the requested URL
     def load_html(url)
       uri = URI.parse(url)
       http = Net::HTTP.new(uri.host, uri.port)

data/lib/proxy_fetcher/proxy.rb

@@ -11,6 +11,7 @@ module ProxyFetcher
 
     def connectable?
       connection = Net::HTTP.new(addr, port)
+      connection.use_ssl = true if https?
       connection.open_timeout = ProxyFetcher::Manager.config.open_timeout
       connection.read_timeout = ProxyFetcher::Manager.config.read_timeout
 
@@ -21,12 +22,14 @@ module ProxyFetcher
       false
     end
 
+    alias_method :valid?, :connectable?
+
     def http?
       type.casecmp('http').zero?
     end
 
     def https?
-      !http?
+      type.casecmp('https').zero?
     end
 
     def uri
@@ -39,6 +42,7 @@ module ProxyFetcher
 
     private
 
+    # HideMyAss proxy list rows parsing by columns
     def parse_row!(html)
       html.xpath('td').each_with_index do |td, index|
         case index
@@ -67,11 +71,11 @@ module ProxyFetcher
       good = []
       bytes = []
       css = html.at_xpath('span/style/text()').to_s
-      css.split.each { |l| good << $1 if l.match(/\.(.+?)\{.*inline/) }
+      css.split.each { |l| good << Regexp.last_match(1) if l =~ /\.(.+?)\{.*inline/ }
 
       html.xpath('span/span | span | span/text()').each do |span|
         if span.is_a?(Nokogiri::XML::Text)
-          bytes << $1 if span.content.strip.match(/\.{0,1}(.+)\.{0,1}/)
+          bytes << Regexp.last_match(1) if span.content.strip =~ /\.{0,1}(.+)\.{0,1}/
         elsif (span['style'] && span['style'] =~ /inline/) ||
               (span['class'] && good.include?(span['class'])) ||
               (span['class'] =~ /^[0-9]/)

data/lib/proxy_fetcher/version.rb

@@ -9,7 +9,7 @@ module ProxyFetcher
     # Minor version number
     MINOR = 1
     # Smallest version number
-    TINY  = 1
+    TINY  = 2
 
     # Full version number
     STRING = [MAJOR, MINOR, TINY].compact.join('.')

data/spec/proxy_fetcher/manager_spec.rb

@@ -28,4 +28,9 @@ describe ProxyFetcher::Manager do
 
     expect { manager.cleanup! }.to change { manager.proxies }.to([])
   end
+
+  it "doesn't pollute the output with array of proxies" do
+    manager = described_class.new(refresh: false)
+    expect(manager.inspect).to eq(manager.to_s)
+  end
 end

data/spec/proxy_fetcher/proxy_spec.rb

@@ -12,11 +12,16 @@ describe ProxyFetcher::Proxy do
     expect(proxy.https?).to be_falsey.or(be_truthy)
   end
 
-  it 'checks connection status' do
+  it 'not connectable if IP addr is wrong' do
     allow_any_instance_of(ProxyFetcher::Proxy).to receive(:addr).and_return('192.168.1.1')
     expect(proxy.connectable?).to be_falsey
   end
 
+  it "not connectable if server doesn't respond to head" do
+    allow_any_instance_of(Net::HTTP).to receive(:request_head).and_return(false)
+    expect(proxy.connectable?).to be_falsey
+  end
+
   it 'returns URI::Generic' do
     expect(proxy.uri).to be_a(URI::Generic)
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: proxy_fetcher
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Nikita Bulai