proxies-scanner 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in proxies-scanner.gemspec
+gemspec
+
+gem 'work_queue'
+gem 'nokogiri'

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Sam
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# Proxies::Scanner
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'proxies-scanner'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install proxies-scanner
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"

data/bin/proxies-scanner

@@ -0,0 +1,81 @@
+#!/usr/bin/env ruby
+#encoding: utf-8
+
+require 'logger'
+
+$:.unshift(File.dirname(__FILE__) + "/../lib")
+require 'proxies-scanner'
+
+include ProxiesScanner
+
+$logger = Logger.new(STDOUT)
+# $logger.level = Logger::DEBUG
+# $logger.level = Logger::INFO
+# $logger.level = Logger::ERROR
+$logger.level = Logger::FATAL
+
+QUEUE_SIZE_PROXIES = 15
+QUEUE_SIZE_HTTP    = 3
+
+def scan_areas(areas)
+  proxies = []
+  wq = WorkQueue.new(QUEUE_SIZE_HTTP)
+
+  beginning_time = Time.now
+
+  i = 0
+  areas.each do |area|
+    wq.enqueue_b do 
+    i += 1
+      begin
+        s_pos = " [#{i}/#{areas.size}]"
+        print "\033[36m ** HTTP/GET #{area.upcase} proxies #{s_pos.rjust(52, '.')}\033[0m\n"
+        proxies += Proxies.find_by_area(area)
+      rescue SignalException => e
+        raise e
+      rescue Exception => e
+        print "\033[31m~ Error: #{e.message}\033[0m\n"
+      end
+    end
+  end
+  wq.join
+  end_time = Time.now
+
+  print "\033[36m ** #{areas.size} areas scanned in #{((end_time - beginning_time)*1000).round(2)} ms \033[0m\n"
+  puts
+
+  return proxies
+end
+
+
+begin
+  proxies = []
+  if ARGV.size > 0
+    if ARGV[0] = '-f'
+      proxies = Proxies.from_file(ARGV[1])
+    else
+      proxies = scan_areas(ARGV)
+    end
+  else
+    proxies = scan_areas(Proxies::ALL_AREAS)
+  end
+
+  print "\033[32m ** #{proxies.size} proxies founded\033[0m\n"
+
+  wq = WorkQueue.new(QUEUE_SIZE_PROXIES)
+  proxies.each do |p|
+    wq.enqueue_b do
+      p.check
+      print p.format
+    end
+  end
+  wq.join
+
+rescue SignalException => e
+rescue Exception => e
+  puts 'Error:'
+  puts e.message
+  puts e.backtrace
+end
+puts
+$logger.close

data/lib/proxies-scanner/proxies.rb

@@ -0,0 +1,68 @@
+
+
+module ProxiesScanner
+  class Proxies
+
+    ALL_AREAS = [
+      'us', 'fr', 'gb', 'au', 'ca', 'it', 'es', 'hu', 'ng', 'ch', 'se', 'br', 'id',
+      'cn', 'fi', 'be', 'co', 'hr', 'ge', 'gr', 'ie', 'mx', 'mt', 'nl', 'pa', 'ph',
+      'ro', 'sc', 'tw', 'tn', 'tr', 'ua', 've'
+    ]
+
+    def self.find_by_area(area, page=0)
+      $logger.info "Proxies.extract area=#{area}, page=#{page}"
+      proxies = []
+
+      doc = Nokogiri::HTML(open("http://www.proxylists.net/#{area.downcase}_#{page}.html"))
+      table = doc.search('table')[1]
+
+      table.css('tr').each do |tr|
+        td1 = tr.css('td')[0]
+        td2 = tr.css('td')[1]
+
+        unless td1.nil?
+          # extract host
+          js = td1.text.scan(/cape\('(.*)'\)\);/)[0]
+          unless js.nil?
+            js1 = js[0]
+            js2 = URI.decode(js1) # self.document.writeln(\"81.199.34.100\");
+            $logger.debug "Proxies.extract js2: #{js2}"
+            ip = js2.to_s.scan(/ln\("(.*)"\);/)[0][0].to_s
+            $logger.debug "Proxies.extract ip: #{ip}"
+          end
+
+          # extract port
+          unless td2.nil?
+            port = td2.text.strip.to_s.to_i || 8080
+          end
+
+          proxies << Proxy.new(ip, port, area) unless ip.nil?
+        end
+      end
+      return proxies
+    end
+
+    def self.from_file(filename)
+      proxies = []
+      file = nil
+      begin
+        file = File.open(filename, 'r+')
+        file.each_line do |line|
+          next if line[0] == '#'
+          s_ip = line.split(':')[0]
+          s_port = line.split(':')[1]
+          s_area = line.split(':')[2] || '--'
+          $logger.debug "From file extract s_ip: #{s_ip}"
+          $logger.debug "From file extract s_port: #{s_port}"
+          $logger.debug "From file extract s_area: #{s_area}"
+          proxies << Proxy.new(s_ip.strip, s_port.strip.to_i, s_area.strip)
+        end
+      rescue Exception => e
+        $logger.error "From file error: #{e.message}"
+      end
+      file.close unless file.nil?
+      return proxies
+    end
+  end
+
+end

data/lib/proxies-scanner/proxy.rb

@@ -0,0 +1,84 @@
+
+module ProxiesScanner
+  class Proxy
+    attr_accessor :ip, :port, :area, :status, :response_time
+
+    CHECK_URL = 'http://www.google.fr'
+    TIMEOUT   = 10
+
+    STATUS_NOT_TESTED = "N.TESTED"
+    STATUS_OPEN       = "OPEN"
+    STATUS_FILTERED   = "FILTERED"
+    STATUS_TIMEOUT    = "TIME OUT"
+    STATUS_SOCKERROR  = "CLOSED"
+
+    STATUS_STR_MAX_LENGTH = STATUS_NOT_TESTED.length
+
+    def initialize(ip, port, area='--', status=STATUS_NOT_TESTED)
+      @ip, @port, @area, @status = ip, port, area, status
+      @response_time = -1
+    end
+
+    def to_s
+      "#{@ip}:#{@port} [#{@area}] [#{@status}]" 
+    end
+
+    def format
+      ProxyFormater.fetch(self)
+    end
+
+    def check
+      $logger.debug "check proxy: #{self}"
+
+      uri = URI.parse(CHECK_URL)
+      Timeout::timeout(TIMEOUT) do
+
+        beginning_time = Time.now
+        http = Net::HTTP::Proxy @ip, @port
+        body = http.get_response(uri).body
+        end_time = Time.now
+
+        @response_time = (end_time - beginning_time)*1000
+
+        t = Nokogiri::HTML(body).search('title')
+        if !t.nil? && !t[0].nil? && t[0].text.downcase.include?('google')
+          $logger.debug "Proxy TITLE: #{t[0].text}"
+          $logger.debug 'Proxy open'
+          @status = Proxy::STATUS_OPEN
+        else
+          $logger.debug 'Proxy filtered'
+          @status = Proxy::STATUS_FILTERED
+        end
+      end
+
+    rescue SignalException => e
+      raise e
+    rescue Timeout::Error => e
+      @status = Proxy::STATUS_TIMEOUT
+      $logger.info "** Proxies.check: timeout: #{self}"
+    rescue Exception => e
+      @status = Proxy::STATUS_SOCKERROR
+      $logger.error "** Proxies.check: #{e.message} proxy=#{self}"
+    end
+
+  end
+
+  class ProxyFormater
+    SEPARATOR     = '   '
+    COL_IP_SIZE   = '255.255.255.255'.size
+    COL_PORT_SIZE = '99999'.size
+    COL_STATUS_SIZE = Proxy::STATUS_STR_MAX_LENGTH
+
+    def self.fetch(proxy)
+      s_color = "\033[31m"
+      s_color = "\033[32m" if proxy.status == Proxy::STATUS_OPEN
+      s_color = "\033[35m" if proxy.status == Proxy::STATUS_FILTERED
+      s_color = "\033[33m" if proxy.status == Proxy::STATUS_TIMEOUT
+
+      s_time = ""
+      s_time = "response-time: #{(proxy.response_time.round(2)).to_s[0..8].rjust(9)} (ms)" if proxy.response_time > 0
+      "%color #{proxy.ip.rjust(COL_IP_SIZE)}#{SEPARATOR}#{proxy.port.to_s.ljust(COL_PORT_SIZE)}#{SEPARATOR}[#{proxy.area.upcase}]#{SEPARATOR}[#{proxy.status.center(COL_STATUS_SIZE)}]#{SEPARATOR}#{s_time}\033[0m\n".gsub(/%color/, s_color)
+    end
+  end
+
+end

data/lib/proxies-scanner/version.rb

@@ -0,0 +1,3 @@
+module ProxiesScanner
+  VERSION = "0.0.1"
+end

data/lib/proxies-scanner.rb

@@ -0,0 +1,15 @@
+
+require 'net/http'
+require 'uri'
+require 'cgi'
+require 'timeout'
+require 'nokogiri'
+require 'open-uri'
+require 'work_queue'
+require 'logger'
+
+require "proxies-scanner/version"
+require "proxies-scanner/proxies"
+require "proxies-scanner/proxy"
+
+

data/proxies-scanner.gemspec

@@ -0,0 +1,20 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/proxies-scanner/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Sam"]
+  gem.email         = ["samuel@pagedegeek.com"]
+  gem.description   = %q{scan proxies list and check if they are valid}
+  gem.summary       = %q{scan proxies list and check if they are valid}
+  gem.homepage      = "http://www.github.com/pagedegeek/proxies-scanner"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "proxies-scanner"
+  gem.require_paths = ["lib", "bin"]
+  gem.version       = ProxiesScanner::VERSION
+
+  gem.add_dependency "nokogiri"
+  gem.add_dependency "work_queue"
+end

metadata

@@ -0,0 +1,90 @@
+--- !ruby/object:Gem::Specification
+name: proxies-scanner
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Sam
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-07-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: work_queue
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: scan proxies list and check if they are valid
+email:
+- samuel@pagedegeek.com
+executables:
+- proxies-scanner
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/proxies-scanner
+- lib/proxies-scanner.rb
+- lib/proxies-scanner/proxies.rb
+- lib/proxies-scanner/proxy.rb
+- lib/proxies-scanner/version.rb
+- proxies-scanner.gemspec
+homepage: http://www.github.com/pagedegeek/proxies-scanner
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+- bin
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: scan proxies list and check if they are valid
+test_files: []