proxy_utilities 1.4.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 875abd7b5e98703a60cab5058ea2e577202cd830
+  data.tar.gz: 8340e1c4a5f69dee94d3bf9dcaf845b2f00d0fd2
+SHA512:
+  metadata.gz: 396ab14cd9b4661ff347c3385b5461a6eda636da7a6d5b84e1edf74c8308fe0f243ec92ecbbbf0e1d7db588f8b6755d0153084765a33741cf5b57134674f5e1c
+  data.tar.gz: 15508bc4a0d0025a73ec41289586e5178bad7f22fd5358edb8422644fd8fc84bd32b85c6b09debb82081e8f25ea3af29808b8a9ca454b2e042f1525bcd78f718

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.gem

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in proxy_utilities.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 ruby-matze
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,73 @@
+# ProxyUtilities
+
+Gem to get lists of Proxies and check them if they are still available.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'proxy_utilities'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install proxy_utilities
+
+## Usage
+
+This gem has two features at the moment:
+
+- Get a list of proxies.
+- Check if a proxy is available
+
+### Getter
+
+```ruby
+list = ProxyUtilities.get_proxies
+
+#=>[{:ip => "0.0.0.0",:port => 80, :country_code => "US", :anonymity => "transparent", :type => "SOCKS5"}, {...}, ...]
+```
+
+
+### Checker
+ This module checks if the proxy can reach a website over the proxy, also additional websites can be enabled.
+
+ ```ruby
+ checked_proxy = ProxyUtilities.check_proxy("0.0.0.0", 80)
+ #=> {:status => true}
+ # or
+ #=> {:status => false}
+```
+also working:
+```ruby
+checked_proxy = ProxyUtilities.check_proxy("0.0.0.0", "80")
+ ```
+Following additional attributes are supported:
+
+```ruby
+checked_proxy = ProxyUtilities.check_proxy("0.0.0.0", 80, check_websites: true)
+ #=> {:status => true, :websites => {:google => true, :amazon => true}}
+```
+```ruby
+checked_proxy = ProxyUtilities.check_proxy("0.0.0.0", 80, check_websites: true, webistes: ["https://github.com/ruby-matze/proxy_utilities"])
+ #=> {:status => true, :websites => {:google => true, :amazon => true, "https://github.com/ruby-matze/proxy_utilities" => true}
+```
+Checking full lists of proxies like:
+```ruby
+list = [{{:ip => "0.0.0.0", :port => 80, ...}, ...]
+checked_proxy = ProxyUtilities.check_proxy_list(list, check_websites: true, webistes: ["https://github.com/ruby-matze/proxy_utilities"])
+#=> [{:ip => "0.0.0.0", :port => 80, :status => true, :websites => {:google => true, :amazon => true, "https://github.com/ruby-matze/proxy_utilities" => true}, ....]
+```
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/ruby-matze/proxy_utilities.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "proxy_utilities"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/proxy_utilities.rb

@@ -0,0 +1,92 @@
+require "proxy_utilities/version"
+require "proxy_utilities/getter"
+require "proxy_utilities/checker"
+#Give 2 main methods, that are needed to using the game
+module ProxyUtilities
+
+  #Get a list of Proxies
+
+  # Say hi to the world!
+  #
+  # Example:
+  #   >> ProxyUtilities.get_proxies
+  #   => [{ip: "0.0.0.0", port: 80, country_code: "US", anonymity: elite, type: "HTTPS"}]
+  #
+  # Arguments:
+  #   proxybroker: (String)
+  #   proxybroker_limit (Integer)
+  #   proxybroker_type (Array["HTTP", "HTTPS", "SOCKS4", "SOCKS5"])
+  def self.get_proxies()
+    return ProxyUtilities::Getter.get_all()
+  end
+
+  #Check if a specific proxy is active and get some stats.
+
+  #params
+  # ip        ipadress of proxy server   string
+  # port      port of proxy server       string
+  # webistes  array of websites to check Array[string]
+  def self.check_proxy(ip, port, websites: nil, check_websites: false)
+    proxy = ProxyUtilities::Checker.new(ip, port)
+    result = {}
+
+    if proxy.check_online?
+      result[:status] = true
+      if check_websites
+        result[:websites] = {}
+
+        if proxy.check_google?
+          result[:websites][:google] = true
+        else
+          result[:websites][:google] = false
+        end
+
+        if proxy.check_amazon?
+          result[:websites][:amazon] = true
+        else
+          result[:websites][:amazon] = false
+        end
+
+        if websites.present?
+          result[:websites] = result[:websites].merge(proxy.check_any_website(websites))
+        end
+      end
+    else
+      result[:status] = false
+    end
+    return result
+  end
+
+  #Check if a specific proxy is active and get some stats.
+
+  #params
+  # list  form of [{:ip => "0.0.0.0", :port => 80, ....}, ....]
+  # check_websites  if he has to check google and amazon   boolean
+  # webistes  array of websites to check Array[string]
+  def self.check_proxy_list(list, websites: nil, check_websites: false)
+    result = []
+    threads = []
+    poolsize = 10
+    jobs = Queue.new
+    list.each{|p| jobs.push p}
+
+    poolsize.times do
+      threads << Thread.new do
+        begin
+          while proxy = jobs.pop(true)
+            Thread.exit if proxy.nil?
+            if proxy.key?(:ip)  and proxy.key?(:port)
+              begin
+                result << ProxyUtilities.check_proxy(proxy[:ip], proxy[:port], websites: websites, check_websites: check_websites).merge({:ip => proxy[:ip], :port => proxy[:port]})
+              end
+            end
+          end
+        rescue
+          Thread.exit
+        end
+      end
+    end
+    threads.each(&:join)
+    return result
+  end
+end

data/lib/proxy_utilities/checker.rb

@@ -0,0 +1,59 @@
+require 'net/http'
+require 'net/ping'
+module ProxyUtilities
+  class Checker
+
+    def initialize(ip, port)
+      @ip = ip
+      @port = port
+    end
+
+    def check_online?
+      begin
+        check = Net::Ping::External.new(host = @ip, port = @port)
+        check.ping?
+      rescue
+        return false
+      end
+    end
+
+    def check_google?
+      begin
+        url = "http://google.com"
+        proxy_connection = Net::HTTP::Proxy(@ip, @port.to_i)
+        Timeout.timeout(3) do
+          return proxy_connection.start(url).active?
+        end
+      rescue
+        return false
+      end
+    end
+
+    def check_amazon?
+      begin
+        url = "http://amazon.com"
+        proxy_connection = Net::HTTP::Proxy(@ip, @port.to_i)
+        Timeout.timeout(3) do
+          return proxy_connection.start(url).active?
+        end
+      rescue
+        return false
+      end
+    end
+
+    def check_any_website(websites)
+      result = {}
+      proxy_connection = Net::HTTP::Proxy(@ip, @port.to_i)
+      websites.each do |website|
+        begin
+          Timeout.timeout(3) do
+            result[website] = proxy_connection.start(website).active?
+          end
+        rescue
+          next
+        end
+      end
+      return result
+    end
+  end
+end

data/lib/proxy_utilities/getter.rb

@@ -0,0 +1,198 @@
+require 'net/http'
+require 'rubygems'
+require 'nokogiri'
+require 'open-uri'
+require 'json'
+require 'iso_country_codes'
+require 'base64'
+
+module ProxyUtilities
+  module Getter
+
+    #Starting all fetching processes
+    def self.get_all()
+      result = []
+      result += self.get_free_proxy_list_net
+      result += self.get_ssl_proxies_org
+      result += self.get_gather_proxy_com
+      result += self.get_httptunnel_ge
+      result += self.get_proxy_list_org
+      result += self.get_socks_proxy_list_net
+      result += self.get_checker_proxy_net
+      return result.uniq
+    end
+
+    def self.get_free_proxy_list_net
+      result = []
+      url = "https://free-proxy-list.net/"
+      page = Nokogiri::HTML(open(url))
+      rows = page.xpath('//table[@id="proxylisttable"]/tbody/tr')
+
+      rows.each do |row|
+        begin
+          proxy = {}
+          proxy[:ip] = row.xpath('td[1]').text
+          proxy[:port] = Integer(row.xpath('td[2]').text.gsub(/^0+/, ''))
+          proxy[:country_code] = row.xpath('td[3]').text
+          proxy[:anonymity] = row.xpath('td[5]').text.gsub(" proxy", "")
+          if row.xpath('td[7]').text == 'yes'
+            proxy[:type] = 'HTTPS'
+          else
+            proxy[:type] = 'HTTP'
+          end
+          result << proxy
+        end
+      end
+      return result
+    end
+
+    def self.get_ssl_proxies_org
+      result = []
+      url = "https://www.sslproxies.org/"
+      page = Nokogiri::HTML(open(url))
+      rows = page.xpath('//table[@id="proxylisttable"]/tbody/tr')
+
+      rows.each do |row|
+        begin
+          proxy = {}
+          proxy[:ip] = row.xpath('td[1]').text
+          proxy[:port] = Integer(row.xpath('td[2]').text.gsub(/^0+/, ''))
+          proxy[:country_code] = row.xpath('td[3]').text
+          proxy[:anonymity] = row.xpath('td[5]').text.gsub(" proxy", "")
+          if row.xpath('td[7]').text == 'yes'
+            proxy[:type] = 'HTTPS'
+          else
+            proxy[:type] = 'HTTP'
+          end
+          result << proxy
+        end
+      end
+      return result
+    end
+
+    def self.get_gather_proxy_com
+      result = []
+      url = "http://www.gatherproxy.com/"
+      page = Nokogiri::HTML(open(url))
+      rows = page.xpath('//div[@class="proxy-list"]/table/script')
+      javascript = rows.text[/{.+}/im].gsub("gp.insertPrx(", "").gsub(");", "")
+      json = JSON.parse("[" + javascript.split("\r\n").join(",").gsub("               ,                    ", "") + "]")
+      json.each do |row|
+        begin
+          proxy = {}
+          proxy[:ip] = row["PROXY_IP"]
+          proxy[:port] = row['PROXY_PORT'].to_i(16)
+          begin
+            proxy[:country_code] = IsoCountryCodes.search_by_name(row["PROXY_COUNTRY"]).first.alpha2
+          rescue
+            proxy[:country_code] = nil
+          end
+          proxy[:anonymity] = row['PROXY_TYPE'].downcase
+          proxy[:type] = 'HTTP'
+          result << proxy
+        end
+      end
+      return result
+    end
+
+    def self.get_httptunnel_ge
+      result = []
+      url = "http://www.httptunnel.ge/ProxyListForFree.aspx"
+      page = Nokogiri::HTML(open(url))
+      rows = page.xpath('//table[contains(@id, "GridView")]/tr[(count(td)>2)]')
+
+      rows.each do |row|
+        begin
+          proxy = {}
+          proxy[:ip] = row.xpath('td[1]').text.split(":")[0].strip
+          proxy[:port] = Integer(row.xpath('td[1]').text.split(":")[1].strip)
+          proxy[:country_code] = row.xpath('td[8]/img/@title').text
+          transparency = row.xpath('td[5]').text.to_sym
+          proxy[:anonymity] = { A: 'anonymous',  E: 'elite', T: 'transparent', U: 'unknown'}.fetch(transparency, 'Unknown')
+          proxy[:type] = 'HTTP'
+          result << proxy
+        end
+      end
+      return result
+    end
+
+    def self.get_proxy_list_org
+      result = []
+      (1..10).to_a.each do |page_num|
+        begin
+          url = "https://proxy-list.org/english/index.php?p=#{page_num}"
+          page = Nokogiri::HTML(open(url))
+          rows = page.css('.table-wrap .table ul')
+
+          rows.each do |row|
+            begin
+              proxy = {}
+              proxy[:ip] = ::Base64.decode64(rows.first.at_css('li script').text.match(/'(.+)'/)[1]).split(":")[0].strip
+              proxy[:port] = Integer(::Base64.decode64(rows.first.at_css('li script').text.match(/'(.+)'/)[1]).split(":")[1].strip)
+              country_code = row.xpath('li[5]/div/span/span/span[2]').text.scan(/([A-Z]{2}) /)
+              unless country_code.nil?
+                begin
+                  proxy[:country_code] = country_code.first.first.strip
+                rescue
+                  next
+                end
+              else
+                next
+              end
+              proxy[:anonymity] = row.xpath('li[4]').text.strip.downcase
+              proxy[:type] = row.xpath('li[2]').text.strip.upcase
+              if proxy[:type] == "-"
+                proxy[:type] = nil
+              end
+              result << proxy
+            end
+          end
+        end
+      end
+      return result
+    end
+
+    def self.get_socks_proxy_list_net
+      result = []
+      url = "https://www.socks-proxy.net/"
+      page = Nokogiri::HTML(open(url))
+      rows = page.xpath('//table[@id="proxylisttable"]/tbody/tr')
+
+      rows.each do |row|
+        begin
+          proxy = {}
+          proxy[:ip] = row.xpath('td[1]').text
+          proxy[:port] = Integer(row.xpath('td[2]').text.gsub(/^0+/, ''))
+          proxy[:country_code] = row.xpath('td[3]').text
+          proxy[:anonymity] = row.xpath('td[6]').text.downcase
+          proxy[:type] = row.xpath('td[5]').text.upcase
+          result << proxy
+        end
+      end
+      return result
+    end
+
+    def self.get_checker_proxy_net
+      result = []
+      url = "https://checkerproxy.net/api/archive/#{Date.today.strftime('%Y-%m-%d')}"
+      page = Nokogiri::HTML(open(url))
+      rows = JSON.parse(page)
+
+      rows.each do |row|
+        begin
+          proxy = {}
+          proxy[:ip] = row["ip"]
+          proxy[:port] = Integer(row["addr"].split(":")[1].gsub(/^0+/, ''))
+          proxy[:country_code] = row["addr_geo_iso"].upcase
+          anonymities = {0=>"transparent", 1=> "elite", 2=> "anonymous"}
+          proxy[:anonymity] = anonymities[row["kind"]]
+          types = {1=>"HTTP", 2=>"HTTPS", 3=>"SOCKS4", 4=>"SOCKS5"}
+          proxy[:type] = types[row["type"]]
+          result << proxy
+        end
+      end
+      return result
+    end
+
+  end
+end

data/lib/proxy_utilities/version.rb

@@ -0,0 +1,3 @@
+module ProxyUtilities
+  VERSION = "1.4.1"
+end

data/proxy_utilities.gemspec

@@ -0,0 +1,33 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "proxy_utilities/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "proxy_utilities"
+  spec.version       = ProxyUtilities::VERSION
+  spec.authors       = ["ruby-matze"]
+  spec.email         = ["matze@it-dudes.de"]
+
+  spec.summary       = %q{Getting and checking Proxies.}
+  spec.description   = %q{This gem will help you with getting and checking proxies.}
+  spec.homepage      = "https://github.com/ruby-matze/proxy_utilities.git"
+  spec.license       = "MIT"
+
+
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.16"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.7"
+  spec.add_dependency "nokogiri", "~> 1.8", ">= 1.8.2"
+  spec.add_dependency "json", "~> 2.1", ">= 2.1.0"
+  spec.add_dependency "iso_country_codes", "~> 0.7",">= 0.7.8"
+  spec.add_dependency "net-ping", "~> 2.0", ">= 2.0.4"
+end

metadata

@@ -0,0 +1,178 @@
+--- !ruby/object:Gem::Specification
+name: proxy_utilities
+version: !ruby/object:Gem::Version
+  version: 1.4.1
+platform: ruby
+authors:
+- ruby-matze
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-05-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.7'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.8.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.8.2
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+- !ruby/object:Gem::Dependency
+  name: iso_country_codes
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.7.8
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.7.8
+- !ruby/object:Gem::Dependency
+  name: net-ping
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.4
+description: This gem will help you with getting and checking proxies.
+email:
+- matze@it-dudes.de
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/proxy_utilities.rb
+- lib/proxy_utilities/checker.rb
+- lib/proxy_utilities/getter.rb
+- lib/proxy_utilities/version.rb
+- proxy_utilities.gemspec
+homepage: https://github.com/ruby-matze/proxy_utilities.git
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2.1
+signing_key: 
+specification_version: 4
+summary: Getting and checking Proxies.
+test_files: []