proxy_fetcher 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 405be695c325a43b7669a1c8511b9003e3640e18
+  data.tar.gz: e5b4caa06fb9e5b2fa5f04f91c1ed7c9f11fdfa6
+SHA512:
+  metadata.gz: 6516ad0696069ce2ec2e9bf52ed83f47dadb4fd8dc202277105f61bc57ee3a366d99f8023fb3e5fd3c0de8242a586919e9d1172b86e62f21f11216fbc0b4ba70
+  data.tar.gz: a331a1db2a45667c1ef132e7ec4f9fa1c789435e1ab3ff7bcad9a31da5340416b221260923a40b0009909a448c08157da7d32b5834acdade3e2124aa73dff6f7

data/.gitignore

@@ -0,0 +1,46 @@
+*.rbc
+capybara-*.html
+.rspec
+/log
+/tmp
+/db/*.sqlite3
+/db/*.sqlite3-journal
+/public/system
+/coverage/
+/spec/tmp
+*.orig
+rerun.txt
+pickle-email-*.html
+.idea
+Gemfile.lock
+
+# TODO Comment out this rule if you are OK with secrets being uploaded to the repo
+config/initializers/secret_token.rb
+
+# Only include if you have production secrets in this file, which is no longer a Rails default
+# config/secrets.yml
+
+# dotenv
+# TODO Comment out this rule if environment variables can be committed
+.env
+
+## Environment normalization:
+/.bundle
+/vendor/bundle
+
+# these should all be checked in to normalize the environment:
+# Gemfile.lock, .ruby-version, .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc
+
+# if using bower-rails ignore default bower_components path bower.json files
+/vendor/assets/bower_components
+*.bowerrc
+bower.json
+
+# Ignore pow environment settings
+.powenv
+
+# Ignore Byebug command history file.
+.byebug_history

data/.travis.yml

@@ -0,0 +1,12 @@
+language: ruby
+before_install: gem install bundler
+bundler_args: --without yard guard benchmarks
+script: "rake spec"
+rvm:
+  - 2.2.4
+  - 2.3.3
+  - 2.4.0
+  - ruby-head
+matrix:
+  allow_failures:
+    - rvm: ruby-head

data/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+gemspec
+
+group :test do
+  gem 'coveralls', require: false
+end

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 Nikita Bulai
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,118 @@
+# Ruby lib for managing proxies
+[![Build Status](https://travis-ci.org/nbulaj/proxy_fetcher.svg?branch=master)](https://travis-ci.org/nbulaj/proxy_fetcher)
+[![Coverage Status](https://coveralls.io/repos/github/nbulaj/proxy_fetcher/badge.svg)](https://coveralls.io/github/nbulaj/proxy_fetcher)
+[![License](http://img.shields.io/badge/license-MIT-brightgreen.svg)](#license)
+
+This gem can help your Ruby application to make HTTP(S) requests from proxy server, fetching and validating
+current proxy lists from the [HideMyAss](http://hidemyass.com/) service.
+
+## Installation
+
+If using bundler, first add 'proxy_fetcher' to your Gemfile:
+
+```ruby
+gem 'proxy_fetcher', '~> 0.1'
+```
+
+And run:
+
+```sh
+bundle install
+```
+
+Otherwise simply install the gem:
+
+```sh
+gem install proxy_fetcher -v '0.1'
+```
+
+## Example of usage
+
+Get current proxy list:
+
+```ruby
+manager = ProxyFetcher::Manager.new # will immediately load proxy list from the server
+manager.proxies
+
+ #=> [#<ProxyFetcher::Proxy:0x00000002879680 @addr="97.77.104.22", @port="3128", @country="USA", 
+ #     @response_time="5217", @speed="48", @connection_time="100", @type="HTTP", @anonymity="High">, ... ]
+```
+
+Get raw proxy URLs:
+
+```ruby
+manager = ProxyFetcher::Manager.new
+manager.raw_proxies
+
+ # => ["http://97.77.104.22:3128", "http://94.23.205.32:3128", "http://209.79.65.140:8080",
+ #     "http://91.217.42.2:8080", "http://97.77.104.22:80", "http://165.234.102.177:8080", ...]
+```
+
+If `ProxyFetcher::Manager` was already initialized somewhere, you can refresh the proxy list by calling `#refresh_list!` method:
+
+```ruby
+manager.refresh_list!
+
+ #=> [#<ProxyFetcher::Proxy:0x00000002879680 @addr="97.77.104.22", @port="3128", @country="USA", 
+ #     @response_time="5217", @speed="48", @connection_time="100", @type="HTTP", @anonymity="High">, ... ]
+```
+
+Every proxy is a `ProxyFetcher::Proxy` object that has next readers:
+
+* `addr` (IP address)
+* `port`
+* `country` (USA or Brazil for example)
+* `response_time` (5217 for example)
+* `connection_time` (rank from 0 to 100, where 0 — slow, 100 — high)
+* `speed` (rank from 0 to 100, where 0 — slow, 100 — high)
+* `type` (URI schema, HTTP for example)
+* `anonimity` (Low or High +KA for example)
+
+Also you can call next instance method for every Proxy object:
+
+* `connectable?` (whether proxy server is available)
+* `http?` (whether proxy server has a HTTP protocol)
+* `https?` (whether proxy server has a HTTPS protocol)
+* `uri` (returns `URI::Generic` object)
+* `url` (returns a formatted URL like "_http://IP:PORT_" )
+
+If you wanna clear current proxy manager list from dead servers, you can just call `cleanup!` method:
+
+```ruby
+manager.cleanup!
+```
+
+To change open/read timeout for `cleanup!` and `connectable?` methods yu need to change ProxyFetcher::Manager config:
+
+```ruby
+ProxyFetcher::Manager.config.read_timeout = 1 # default is 3
+ProxyFetcher::Manager.config.open_timeout = 1# default is 3
+
+manager = ProxyFetcher::Manager.new
+manager.cleanup!
+```
+
+## Contributing
+
+You are very welcome to help improve ProxyFetcher if you have suggestions for features that other people can use.
+
+To contribute:
+
+1. Fork the project.
+2. Create your feature branch (`git checkout -b my-new-feature`).
+3. Implement your feature or bug fix.
+4. Add documentation for your feature or bug fix.
+5. Run <tt>rake doc:yard</tt>. If your changes are not 100% documented, go back to step 4.
+6. Add tests for your feature or bug fix.
+7. Run `rake` to make sure all tests pass.
+8. Commit your changes (`git commit -am 'Add new feature'`).
+9. Push to the branch (`git push origin my-new-feature`).
+10. Create new pull request.
+
+Thanks.
+
+## License
+
+proxy_fetcher gem is released under the [MIT License](http://www.opensource.org/licenses/MIT).
+
+Copyright (c) 2017 Nikita Bulai (bulajnikita@gmail.com).

data/Rakefile

@@ -0,0 +1,6 @@
+require 'bundler/gem_tasks'
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/lib/proxy_fetcher.rb

@@ -0,0 +1,66 @@
+require 'uri'
+require 'net/http'
+require 'nokogiri'
+
+require 'proxy_fetcher/configuration'
+require 'proxy_fetcher/proxy'
+
+module ProxyFetcher
+  class Manager
+    PROXY_PROVIDER_URL = 'http://proxylist.hidemyass.com/'.freeze
+
+    class << self
+      def config
+        @config ||= ProxyFetcher::Configuration.new
+      end
+    end
+
+    attr_reader :proxies
+
+    # refresh: true - load proxy list from the remote server on initialization
+    # refresh: false - just initialize the class, proxy list will be empty ([])
+    def initialize(refresh: true)
+      if refresh
+        refresh_list!
+      else
+        @proxies = []
+      end
+    end
+
+    # Update current proxy list from the provider
+    def refresh_list!
+      doc = Nokogiri::HTML(load_html(PROXY_PROVIDER_URL))
+      rows = doc.xpath('//table[@id="listable"]/tbody/tr')
+
+      @proxies = rows.map { |row| Proxy.new(row) }
+    end
+
+    alias_method :fetch!, :refresh_list!
+
+    # Clean current proxies list from dead proxies
+    def cleanup!
+      proxies.keep_if(&:connectable?)
+    end
+
+    alias_method :validate!, :cleanup!
+
+    # Just schema + host + port
+    def raw_proxies
+      proxies.map(&:url)
+    end
+
+    # No need to put all the attr_readers
+    def inspect
+      to_s
+    end
+
+    private
+
+    def load_html(url)
+      uri = URI.parse(url)
+      http = Net::HTTP.new(uri.host, uri.port)
+      response = http.get(uri.request_uri)
+      response.body
+    end
+  end
+end

data/lib/proxy_fetcher/configuration.rb

@@ -0,0 +1,10 @@
+module ProxyFetcher
+  class Configuration
+    attr_reader :open_timeout, :read_timeout
+
+    def initialize
+      @open_timeout = 3
+      @read_timeout = 3
+    end
+  end
+end

data/lib/proxy_fetcher/proxy.rb

@@ -0,0 +1,94 @@
+module ProxyFetcher
+  class Proxy
+    attr_reader :addr, :port, :country, :response_time,
+                :connection_time, :speed, :type, :anonimity
+
+    def initialize(html_row)
+      parse_row!(html_row)
+
+      self
+    end
+
+    def connectable?
+      connection = Net::HTTP.new(addr, port)
+      connection.open_timeout = ProxyFetcher::Manager.config.open_timeout
+      connection.read_timeout = ProxyFetcher::Manager.config.read_timeout
+
+      connection.start { |http| return true if http.request_head('/') }
+
+      false
+    rescue Timeout::Error, Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::ECONNABORTED
+      false
+    end
+
+    def http?
+      type.casecmp('http').zero?
+    end
+
+    def https?
+      !http?
+    end
+
+    def uri
+      URI::Generic.build(host: addr, port: port, scheme: type)
+    end
+
+    def url
+      uri.to_s
+    end
+
+    private
+
+    def parse_row!(html)
+      html.xpath('td').each_with_index do |td, index|
+        case index
+        when 1
+          @addr = parse_addr(td)
+        when 2 then
+          @port = Integer(td.content.strip)
+        when 3 then
+          @country = td.content.strip
+        when 4
+          @response_time = parse_response_time(td)
+          @speed = parse_indicator_value(td)
+        when 5
+          @connection_time = parse_indicator_value(td)
+        when 6 then
+          @type = td.content.strip
+        when 7
+          @anonymity = td.content.strip
+        else
+          # nothing
+        end
+      end
+    end
+
+    def parse_addr(html)
+      good = []
+      bytes = []
+      css = html.at_xpath('span/style/text()').to_s
+      css.split.each { |l| good << $1 if l.match(/\.(.+?)\{.*inline/) }
+
+      html.xpath('span/span | span | span/text()').each do |span|
+        if span.is_a?(Nokogiri::XML::Text)
+          bytes << $1 if span.content.strip.match(/\.{0,1}(.+)\.{0,1}/)
+        elsif (span['style'] && span['style'] =~ /inline/) ||
+              (span['class'] && good.include?(span['class'])) ||
+              (span['class'] =~ /^[0-9]/)
+
+          bytes << span.content
+        end
+      end
+
+      bytes.join('.').gsub(/\.+/, '.')
+    end
+
+    def parse_response_time(html)
+      Integer(html.at_xpath('div')['rel'])
+    end
+
+    def parse_indicator_value(html)
+      Integer(html.at('.indicator').attr('style').match(/width: (\d+)%/i)[1])
+    end
+  end
+end

data/lib/proxy_fetcher/version.rb

@@ -0,0 +1,17 @@
+module ProxyFetcher
+  def self.gem_version
+    Gem::Version.new VERSION::STRING
+  end
+
+  module VERSION
+    # Major version number
+    MAJOR = 0
+    # Minor version number
+    MINOR = 1
+    # Smallest version number
+    TINY  = 0
+
+    # Full version number
+    STRING = [MAJOR, MINOR, TINY].compact.join('.')
+  end
+end

data/proxy_fetcher.gemspec

@@ -0,0 +1,23 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), 'lib'))
+
+require 'proxy_fetcher/version'
+
+Gem::Specification.new do |gem|
+  gem.name = 'proxy_fetcher'
+  gem.version = ProxyFetcher.gem_version
+  gem.date = '2017-05-19'
+  gem.summary = 'Ruby gem for dealing with proxy lists '
+  gem.description = 'This gem can help your Ruby application to make HTTP(S) requests ' \
+                    'from proxy server, fetching and validating current proxy lists from the HideMyAss service.'
+  gem.authors = ['Nikita Bulai']
+  gem.email = 'bulajnikita@gmail.com'
+  gem.require_paths = ['lib']
+  gem.files = `git ls-files`.split($RS)
+  gem.homepage = 'http://github.com/nbulaj/proxy_fetcher'
+  gem.license = 'MIT'
+  gem.required_ruby_version = '>= 2.2.2'
+
+  gem.add_runtime_dependency 'nokogiri', '~> 1.6', '>= 1.6'
+
+  gem.add_development_dependency 'rspec', '~> 3.5'
+end

data/spec/proxy_fetcher/manager_spec.rb

@@ -0,0 +1,31 @@
+require 'spec_helper'
+
+describe ProxyFetcher::Manager do
+  it 'loads proxy list on initialization by default' do
+    manager = described_class.new
+    expect(manager.proxies).not_to be_empty
+  end
+
+  it "doesn't load proxy list on initialization if `refresh` argument was set to false" do
+    manager = described_class.new(refresh: false)
+    expect(manager.proxies).to be_empty
+  end
+
+  it 'can returns Proxy objects' do
+    manager = described_class.new
+    expect(manager.proxies).to all(be_a(ProxyFetcher::Proxy))
+  end
+
+  it 'can returns raw proxies' do
+    manager = described_class.new
+    expect(manager.raw_proxies).to all(be_a(String))
+  end
+
+  it 'cleanup proxy list from dead servers' do
+    allow_any_instance_of(ProxyFetcher::Proxy).to receive(:connectable?).and_return(false)
+
+    manager = described_class.new
+
+    expect { manager.cleanup! }.to change { manager.proxies }.to([])
+  end
+end

data/spec/proxy_fetcher/proxy_spec.rb

@@ -0,0 +1,27 @@
+require 'spec_helper'
+
+describe ProxyFetcher::Proxy do
+  before :all do
+    @manager = ProxyFetcher::Manager.new
+  end
+
+  let(:proxy) { @manager.proxies.first }
+
+  it 'checks schema' do
+    expect(proxy.http?).to be_falsey.or(be_truthy)
+    expect(proxy.https?).to be_falsey.or(be_truthy)
+  end
+
+  it 'checks connection status' do
+    allow_any_instance_of(ProxyFetcher::Proxy).to receive(:addr).and_return('192.168.1.1')
+    expect(proxy.connectable?).to be_falsey
+  end
+
+  it 'returns URI::Generic' do
+    expect(proxy.uri).to be_a(URI::Generic)
+  end
+
+  it 'returns URL' do
+    expect(proxy.url).to be_a(String)
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,16 @@
+if ENV['CI'] || ENV['TRAVIS'] || ENV['COVERALLS'] || ENV['JENKINS_URL']
+  require 'coveralls'
+  Coveralls.wear!
+else
+  require 'simplecov'
+  SimpleCov.start
+end
+
+require 'bundler/setup'
+Bundler.setup
+
+require 'proxy_fetcher'
+
+RSpec.configure do |config|
+  config.order = 'random'
+end

metadata

@@ -0,0 +1,92 @@
+--- !ruby/object:Gem::Specification
+name: proxy_fetcher
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Nikita Bulai
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-05-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+description: This gem can help your Ruby application to make HTTP(S) requests from
+  proxy server, fetching and validating current proxy lists from the HideMyAss service.
+email: bulajnikita@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- lib/proxy_fetcher.rb
+- lib/proxy_fetcher/configuration.rb
+- lib/proxy_fetcher/proxy.rb
+- lib/proxy_fetcher/version.rb
+- proxy_fetcher.gemspec
+- spec/proxy_fetcher/manager_spec.rb
+- spec/proxy_fetcher/proxy_spec.rb
+- spec/spec_helper.rb
+homepage: http://github.com/nbulaj/proxy_fetcher
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.2.2
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.3
+signing_key: 
+specification_version: 4
+summary: Ruby gem for dealing with proxy lists
+test_files: []