rhack 0.2.2

data/lib/init.rb

@@ -0,0 +1,36 @@
+# encoding: utf-8
+module HTTPAccessKit
+  include RMTools
+  CONFIG = YAML.load(read(%W(rhack.yml #{File.join(ENV['HOME'], 'rhack.yml')})) || '') || {}
+
+  UAS = if File.file?(uas = CONFIG['ua file'] || File.join(ENV['HOME'], 'ua.txt'))
+           IO.read(uas)/"\n"
+  else   ['Mozilla/5.0 (Windows; U; Windows NT 5.1; ru; rv:1.9.1.8) Gecko/20100202 MRA 5.6 (build 03278) Firefox/3.5.8 (.NET CLR 3.5.30729)'] end
+  
+  L = RMLogger.new(CONFIG.logger || {})
+  
+  config	      = CONFIG.db || File.join(ENV['HOME'], 'db.yml')
+  begin
+  DB	            = ActiveRecord::Base.establish_connection_with config
+  rescue LoadError
+  DB = nil
+  end
+ 
+  cache	      = CONFIG.cache || {}
+  CacheDir	  = cache.dir || File.join(File.dirname(__FILE__), 'cache')
+  CacheTable	= (cache.table || :rhack_cache).to_sym
+  CacheTTL	  = cache.clean ? eval(cache.clean).b : nil
+  
+  RETRY	      = CONFIG['scout retry'] || {}
+  
+  $uas	  ||= UAS
+  $Carier	||= Curl::Multi.new
+  $Carier.pipeline = true
+  
+  def self.update
+    each_child {|c| c.class_eval "include HTTPAccessKit; extend HTTPAccessKit" if !c.in c.children}
+  end
+end
+
+module Curl; include HTTPAccessKit end
+RHACK = HTTPAccessKit

data/lib/rhack.rb

@@ -0,0 +1,16 @@
+# encoding: utf-8
+$KCODE	= 'UTF-8' if RUBY_VERSION < "1.9"
+require 'rmtools_dev' unless defined? RMTools
+here = File.expand_path File.dirname __FILE__
+require File.join(here, 'curb_core.so')
+require 'active_record'
+Dir.glob("#{here}/extensions/**.rb") { |f| require f }
+
+require "#{here}/init" unless defined? RHACK
+RHACK::VERSION = IO.read(File.join here, '..', 'Rakefile').match(/RHACK_VERSION = '(.+?)'/)[1]
+
+require "#{here}/curl-global"
+require "#{here}/scout"
+require "#{here}/frame"
+require "#{here}/words"
+require "#{here}/cache" if RHACK::DB

data/lib/rhack.yml.template

@@ -0,0 +1,19 @@
+ua file: /path/to/ua/list
+cache:
+  dir: /path/to/cache/dir
+  table: hack_cache
+  clean: 30.days
+logger:
+  :out: /path/to/rmlogger/logfile
+scout retry:
+  example.com:
+  - TimeoutError
+db:
+  reconnect: true
+  encoding: utf8
+  username: root
+  adapter: mysql
+  database: dbname
+  pool: 5
+  password:
+  socket: /var/run/mysqld/mysqld.sock

data/lib/rhack/proxy/checker.rb

@@ -0,0 +1,226 @@
+# encoding: utf-8
+require 'rhack'
+require File.join(File.dirname(__FILE__), 'list')
+
+$unres_hosts = (RMTools.read('tmp/unres_hosts') or '')/"\n"
+
+module RHACK
+  module Proxy
+    PROXYLISTFILE = 'log/proxylist.txt'
+    DefaultGet = {:req => 'http://internet.yandex.ru/speed/?len=10&rnd=%s', :expect => {:body => "yandex! "}}
+    DefaultDL = {:req => 'http://internet.yandex.ru/speed/?len=100000', :expect => {:body => "yandex! "*12500}}
+    DefaultPost = {:req => [{'yandex!' => 'yandex!'}, false, 'http://internet.yandex.ru/speed/'], :expect => {:body => "hooray!"}}
+      
+    def DOWANT(pl, opts={})
+      opts[:div] ||= 50
+      opts[:timeout] ||= 5
+      pc = Checker pl, opts
+      begin
+        pc.check
+        L.info "working proxies: #{pc.wpl.size} with average ping #{pc.wpl.values.avg}"
+        pc.charge pc.get_by_ping(opts[:ping] || 0.5)
+        opts[:div] = opts[:dl_div] || 5
+        opts[:timeout] = opts[:dl_timeout]
+        pc.check opts
+        L.info "fast proxies: #{pc.fpl.size} with average speed #{pc.fpl.values.avg.bytes}/s"
+        pc.get_by_speed(opts[:speed] || 50000)
+      rescue
+        $pc = pc
+        raise
+      end
+    end
+    module_function :DOWANT
+
+    class Interceptor < Scout
+      attr_accessor :posted, :ready, :captcha, :engine, :ready, :num
+    end
+
+    class Checker
+      __init__
+      attr_reader :target, :opts, :pl, :ics, :wpl, :fpl
+      
+      def initialize(*argv)
+        @pl, @target, @opts = argv.fetch_opts [[], DefaultGet]
+        @wpl = {} # proxy => ping
+        @fpl  = {} # proxy => Bps
+        @succeed = @failed = 0
+        @printer = TempPrinter self, "succeed: :succeed\nfailed : :failed"
+        if @opts.page
+          Curl.run
+          @page = IB::Page.new(@opts.page, :rt => true, :form => true) if @opts.page.is String
+          Curl.wait
+          @opts.engine = @page.engine
+        end
+        charge @pl
+      end
+      
+      def inspect
+        "<#ProxyChecker @ics: #{@ics.size} @wpl: #{@wpl.size} @fpl: #{@fpl.size}>"
+      end
+      
+      def charge(pl=@pl, target=@target.req)
+        @ics = []
+        GC.start
+        fail_proc = lambda {|c, e|
+          c.on_complete {}
+          c.base.error = e
+        }
+        target = target.find_is(String) if !target.is String
+        pl.each {|pr|
+          sc = Interceptor.new(target, pr, $uas.rand, @opts)
+          sc.http.on_failure(&fail_proc)
+          @ics << sc
+        }
+        self
+      end
+      
+      def check(*argv, &callback)
+        target, query, opts = argv.fetch_opts [@target, @ics], @opts
+        report	  = opts[:report]
+        cond	    = opts[:while]
+        carier	    = opts[:carier]
+        post	    = opts[:post] || carier || target.is(Array)
+        dl	        = opts[:dl] unless post
+        division	= opts[:div]   || 500
+        if !target.req.is(Array) and @page.resto target.req
+          post ||= target.req == :action
+          target.req = @page.host + @page.send(target.req)
+        end
+        if !query[0].is Interceptor
+          @opts.merge!(opts)
+          query = charge(query, target.req)
+        end
+        
+        testrow = lambda {|d|
+          $log << "\n#{report} = #{instance_eval(report).inspect}" if report
+          throw :break if cond and !instance_eval(&cond)
+          d.each {|s| 
+            if post
+              (carier || self).Post(s, target)
+            else 
+              if '%s'.in target.req
+                scoped_target = target.dup
+                scoped_target.req %= rand
+              else
+                scoped_target = target
+              end
+              if dl
+                DL(s, scoped_target)
+              else
+                callback ? Get(s, scoped_target, &callback) : Get(s, scoped_target)
+              end
+            end
+          }
+          Curl.wait
+        }
+        
+        dl ? (@fpl = {}) : (@wpl  = {})
+        @succeed = @failed = 0
+        Curl.execute
+        catch(:break) {
+          query.div(division).each {|d| testrow[d]}
+        }
+        catch(:break) {
+          query.select {|i| i.res.is Array and i.res[0] == Curl::Err::TimeoutError}.div(division).each {|d| testrow[d]}
+        }
+        @printer.end!
+        self
+      end
+      
+      def expected? res, target
+        target.expect[:code] ||= 200
+        !target.expect.find {|k, v| !( v.is(Proc) ? v[res.__send__(k)] : v === res.__send__(k) )}
+      end
+      
+      def DL(scout, target)
+        scout.loadGet(target.req) {|c|
+          res = c.res
+          $log.debug "  #{c.base} returned #{res}"
+          if !res.is Array and expected? res, target
+            @fpl[c.proxy_url] = (res.body.size/(c.total_time - @wpl[scout.proxystr].to_f)).to_i
+            @succeed += 1
+          else 
+            @failed += 1
+          end
+          @printer.p if !$panic
+        }
+      end
+    
+      def Get(scout, target, &callback)
+        if callback
+          scout.loadGet(target.req, &callback)
+        else
+          scout.loadGet(target) {|c|
+            res = c.res
+            $log.debug " #{c.base} returned #{res}"
+            if !res.is Array and expected? res, target
+              @wpl[c.proxy_url] = c.total_time
+              @succeed += 1
+            else 
+              @failed += 1
+            end
+            @printer.p if !$panic
+         }
+        end
+      end
+      
+      def Post(scout, target)
+        scout.loadPost(*target.req) {|c|
+          res = c.res
+          $log.debug "#{c.base} returned #{res}"
+          if !res.is Array and expected? res, target# || (res.code == '303' and @opts.engine.is IB::Wakaba)
+            @wpl[c.proxy_url] = c.total_time
+            @succeed += 1
+          else 
+            @failed += 1
+          end
+          @printer.p if !$panic
+        }
+      end
+      
+      def to_a
+        deprecation "use #to_pl instead."
+        to_pl
+      end
+    
+      def to_pl
+        @wpl.map {|k,v| [v,k]}.sort.map! {|e| e.last/':'}.to_pl
+      end
+      
+      def fastest
+        val = @wpl.values.sort[0]
+        @wpl.find {|k,v| v == val}[0]/':' if val
+      end
+    
+      def get_by_ping(limit, minlen=1)
+        newpl = []
+        case limit
+          when Numeric;   @wpl.each {|pr,lt| newpl << pr/':' if lt < limit}
+          when Range 
+            begin
+              limit.to_a.each {|i|
+                @wpl.each {|pr,lt| newpl << pr/':' if lt < i}
+                break if newpl.size >= minlen
+                newpl.clear
+              }
+            rescue TypeError
+              @wpl.each {|pr,lt| newpl << pr/':' if lt < limit.min}
+              if newpl.size < minlen
+                newpl.clear
+                @wpl.each {|pr,lt| newpl << pr/':' if lt < limit.max}
+              end
+            end
+        end
+        newpl.to_pl
+      end
+      
+      def get_by_speed(min)
+        newpl = []
+        @fpl.each {|pr, sp| newpl << pr/':' if sp >= min}
+        newpl.to_pl
+      end
+      
+    end
+        
+  end
+end

data/lib/rhack/proxy/list.rb

@@ -0,0 +1,196 @@
+# encoding: utf-8
+module RMTools
+    
+  def mkprlist(df='proxy.txt')
+    if df == :last
+      df = Dir.new('log').content.find_all {|e| e[/pl|proxy/]}.sort_by {|e| File.mtime(e)}.last
+    end
+    pl = [].to_pl df
+    if File.file? df
+      IO.readlines(df).each {|s|
+        s = s[%r{^(?![#/])[^#/]+}]
+        pl << s.chomp if s
+      }
+      pl.map! {|e| e/':'}
+    else
+      puts df+' is missed!'
+    end
+    pl
+  end
+  
+  module_function :mkprlist
+end
+
+class String
+  
+  def grabprlist(to)
+    to.concat(parseips.uniq).size
+  end
+  
+end
+
+class Array
+  
+  def to_pl(file=nil)
+    ProxyList self, file
+  end
+  
+  def grabprlist(dest)
+    if $Carier
+      text = ''
+      GetAll( lambda {text.grabprlist(dest)} ) {|res| text << res+"\n"}
+    else
+      getURLs.join("\n").grabprlist(dest)
+    end
+  end
+  
+end
+
+class ProxyList < Array
+  __init__
+  attr_accessor :file
+  attr_reader :name
+  alias :minus :-
+  alias :plus :+
+  
+  def initialize(source=[], file=nil)
+    if source.is String
+      super []
+      load source
+    elsif source.is Array
+      raise ArgumentError, 'second arg must be a string' if file and !file.is String
+      super source
+      fix! if file
+      @name	= sort.hash
+      @name	= File.split(file)[1] if file and @name == 0
+      @file	    = file || "tmp/#{'%0*x'%[12,@name]}.txt"
+      rw @file, map {|i| i*':'+"\n"} if file and !empty?
+    else raise TypeError, "can't create proxylist from #{source.class}"
+    end
+  end
+  
+  def rehash
+    ProxyList self, @file
+  end
+  
+  def ==(pl)
+    if pl.is ProxyList
+      pl.name == @name 
+    else
+      pl == self
+    end
+  end
+  
+  def -(pl)
+    ProxyList minus pl
+  end
+  
+  def +(pl)
+    ProxyList plus pl
+  end
+  
+  def fix!
+    map! {|i|
+      if i.is Array and ip = i[0]
+        port = i[1].to_i
+        [(ip.gsub!(/^(\d+)\.(\d+)\.(\d+)\.(\d+)$/) {|m| sprintf("%d.%d.%d.%d", *$~[1..4].map! {|d| d.to_i})} || ip), port > 0 ? port : i[1]]
+      end
+    }.compact!
+    uniq! || self
+  end
+  
+  def fix
+    dup.fixprlist!
+  end
+  
+  def inspect
+    @name.is(String) ?
+      sprintf("<#ProxyList: %s (%d items)>", @name, size) :
+      sprintf("<#ProxyList:%#0*x (%d items)>", 12, @name, size)
+  end
+  
+  def to_pl
+    self
+  end
+  
+  def _fixed(ip)
+    ip.is String and ip[/^\d+\.\d+\.\d+\.\d+$/] and !ip["\n"]
+  end
+  
+  def ips
+    self[0].is(Array) ?
+      self[0].size > 1 ?
+        find_all	{|i| i[1].to_i > 0 and _fixed i[0]}.firsts :
+        find_all	{|i| _fixed i[0]}.firsts :
+      find_all	{|i| _fixed i}
+  end
+  
+  def standart
+    find_all	{|i| i[1].to_i > 0}
+  end
+  
+  def glypes
+    reject	{|i| i[1].to_i > 0}
+  end
+  
+  def ips_fixed
+    firsts.each {|ip| return false if !_fixed ip }
+    true
+  end
+  
+  def fix_ips
+    ts = []
+    (0...size).to_a.div(size/50).each {|d| ts << Thread.new {d.each {|i|
+    
+          if self[i][0].is Fixnum
+            self[i][0] = self[i][0].to_ip
+            next
+          elsif self[i][1].to_i == 0 or self[i][0][/^\d+\.\d+\.\d+\.\d+$/]
+            next
+          end
+          
+          if (ip = self[i][0][/\d{1,3}[\.\-]\d+[\.\-]\d+[\.\-]\d{1,3}/])
+            self[i][0] = ip.gsub('-', '.')
+          elsif $unres_hosts.has ip
+            self[i] = nil
+          else
+            ip = IPSocket.getaddress(self[i][0]) rescue($unres_hosts << self[i][0]; nil)
+            ip and (self[i][0] = ip) and tick!
+          end
+          
+    }}}
+    ts.joins && ts.clear
+    compact! || self
+  end
+  
+  def valid
+    each {|i| return false if !i.is Array or i.nitems != 2 or !i[0].is String or !(i[1].is String or i[1].is Fixnum)}
+    true
+  end
+  
+  def save(mark=nil)
+    str = map {|i| i * ':' + "\n" if i.is Array}
+    str = "#"*10+" #{puttime} - #{mark}\n#{str}"+"#"*20 if mark
+    rw @file, str
+  end
+  
+  def load(file=@file)
+    if "\n".in file or "</".in file
+      file.grabprlist self
+    elsif File.file?(file)
+      IO.read(file).grabprlist(self)
+    else
+      file.grabprlist self
+    end
+    self
+  end
+  
+  def find_all
+    super
+  end
+  
+  def reject
+    super
+  end
+
+end