rhack 0.2.2

data/lib/rhack/services.rb

@@ -0,0 +1,445 @@
+# encoding: utf-8
+require 'rhack'
+
+module HTTPAccessKit
+
+  class Service
+    attr_accessor :f
+    
+    def initialize(service, frame, *args)
+      @service = service
+      @f = frame || Frame(self.class::URI[service], *args)
+    end
+      
+    def go(*args, &block)
+      __send__(@service, *args, &block) rescue(Curl.reload)
+    end
+        
+    def inspect
+      "<##{self.class.self_name}:#{@service.to_s.camelize} service via #{@f.inspect}>"
+    end
+    
+  end
+  
+  class ServiceError < Exception; end
+  
+  class Yandex < Service
+    __init__
+      
+    unless defined? IGNORE_UPPERCASE
+    URI = {
+        :speller => "http://speller.yandex.net/services/spellservice.json/checkText",
+        :search => "http://www.yandex.ru/yandsearch?lr=213&%s",
+        :weather => "http://pogoda.yandex.ru/%d/details/"
+    }
+    
+    IGNORE_UPPERCASE = 1
+    IGNORE_DIGITS = 2
+    IGNORE_URLS = 4
+    FIND_REPEAT_WORDS = 8
+    IGNORE_LATIN = 16
+    NO_SUGGEST = 32
+    FLAG_LATIN = 128
+    end
+      
+    def initialize(service=:search, frame=nil)
+      ua = UAS.rand
+      ua << " YB/4.2.0" if !ua["YB"]
+      super service, frame, nil, ua, :ck => {
+          "yandexuid"=>"3644005621268702222",
+          "t"=>"p"
+      }, :eval => false
+    end
+      
+    def search(text, opts={}, &block)
+      uri = URI.search % urlencode(opts.merge(:text=>text))
+      @f.run(uri, :proc_result => block) {|page| process page}
+    end
+    
+    def process page
+      page.find('.p1/.cr').map {|n| [n.at('.cs').href, n.at('.cs').text.strip, (n.at('.kk') || n.at('.k7/div')).text.strip]} if page.html.b
+    end
+      
+    def speller(text, opts=23)
+      text = text.split_to_lines(10000)
+      i = 0
+      @f.run({"text" => text[i], "options" => opts}, URI.speller, :json => true) {|pg| 
+        yield pg.hash
+        text[i+=1] && @f.get({"text" => text[i], "options" => opts}, URI.speller, :json => true)
+      }
+    end
+    
+    def fix_content(doc, opts={})
+      nodes = doc.root.text_nodes
+      speller(nodes*". ", opts) {|json|
+        fix = {}
+        json.each {|h| fix[h.word] = h.s[0] if h.s[0]}
+        nodes.each {|n|
+          fixed = false
+          text = n.text
+          fix.each {|k, v| fixed = true if text.gsub!(/\b#{k}\b/, v)}
+          n.text(text) if fixed
+        }
+      }
+      Curl.wait
+    end
+    
+    def weather city=27612, day=nil, &block
+      if city.is String
+        city = CitiesCodes[city] if defined? CitiesCodes
+        raise ServiceError, "can't get weather info for #{city.inspect}:#{city.class}" if !city.is(Fixnum)
+      end
+      @f.get(URI.weather%city, :proc_result => block) {|pg|
+        ary = pg.find('//.b-forecast-details/tbody/tr{_["class"] =~ /t\d/}').map {|e|
+          "#{e.at('.date') ? e.at('.date').text+":\n" : ''} - #{e.at('.t').text} - #{e.at('.data').text} - #{e.at('.wind/img').alt} #{e.at('.wind').text} м/с"
+        }
+        ary = ary[0..11].div(4) + ary[12..-1].div(2)
+        day ? ary[day] : ary
+      }#.res
+    end
+    
+    def self.weather(*args, &block) new(:weather).go *args, &block end
+    def self.search(*args, &block) new.go *args, &block end
+      
+  end
+    
+  class Google < Service
+    __init__
+    URI = {
+        :translate => "http://ajax.googleapis.com/ajax/services/language/translate?v=1.0&q=%s&langpair=%s%%7C%s",
+        :search => "http://ajax.googleapis.com/ajax/services/search/web?v=1.0&hl=ru&q=%s",
+        :detect => "http://ajax.googleapis.com/ajax/services/language/detect?v=1.0&q=%s"
+    }
+    
+    Shortcuts = Hash[*%w{
+    v ru.wikipedia.org в ru.wikipedia.org вики en.wikipedia.org
+    w en.wikipedia.org ев en.wikipedia.org wiki en.wikipedia.org
+    lm lurkmore.ru лм lurkmore.ru
+    wa world-art.ru ва world-art.ru
+    ad anidb.info ад anidb.info
+    ed encyclopediadramatica.com ед encyclopediadramatica.com
+   }]
+    
+    Langs = *%w{
+    af sq am ar hy az eu be bn bh bg my ca chr zh zh-CN zh-TW hr cs da dv nl en eo et tl fi fr gl ka de el gn gu iw hi hu is id iu it ja kn kk km ko ku ky lo lv lt mk ms ml mt mr mn ne no or ps fa pl pt-PT pa ro ru sa sr sd si sk sl es sw sv tg ta tl te th bo tr uk ur uz ug vi
+  	}
+      
+    def initialize(service=:search, frame=nil)
+      super service, frame, :json => true
+    end
+      
+    def search(text, opts={}, &block)
+      text = "site:#{opts[:site]} #{text}" if opts[:site]
+      uri = URI.search % CGI.escape(text)
+      @f.run(uri, :proc_result => block) {|page|
+        if data = page.hash.responseData.b
+          data.results.map! {|res| [res.unescapedUrl, res.titleNoFormatting, res.content]}
+        end
+      }#.res
+    end
+      
+    def detect(text, wait=!block_given?, &block)
+      text = text.is(String) ? text[0...600] : text[0]
+      uri = URI[:detect] % CGI.escape(text)
+      @f.run(uri, :proc_result => block, :wait => wait) {|page|
+        (data = page.hash.responseData.b) && data.language
+      }
+    end
+      
+    def translate(text, to, from=nil, &block)
+      text = text.split_to_blocks(600, :syntax) if !text.is Array
+      if !from
+        if block_given?
+          return detect(text) {|from| yield translate(text, to, from)}
+        else
+          return translate(text, to, detect(text).res)
+        end
+      end
+      res = []
+      i = 0
+      text.each_with_index {|b, j|
+        @f.run(URI.translate%[CGI.escape(text[j]), from, to], :proc_result => block, :wait => false) {|page|
+          res[j] = (data = page.hash.responseData.b and data.translatedText)
+          (i += 1) == text.size ? res*"\n" : :skip
+        }
+      }
+      Curl.wait if !block_given?
+      res*"\n"
+    end
+    
+    def self.search(*args, &block) new.search *args, &block end
+    def self.tr(*args, &block) new(:translate).translate *args, &block end
+    
+  end
+  
+  class Infoseek < Service
+    URI = {:tr => 'http://translation.infoseek.co.jp/'}
+    
+    def initialize frame=nil
+      super :tr, frame, :eval => false
+    end
+    
+    def get_token page
+      @token = page.at('input[name=token]').value
+    end
+    
+    def tr(text, direction=:from_ja, &block)
+      if @token
+        selector = direction.in([:from_ja, :from_jp, :to_en]) ? 1 : 0
+        body = {'ac' => 'Text', 'lng' => 'en', 'original' => text, 'selector' => selector, 'token' => @token, 'submit' => '　翻訳'}
+        @f.run(body, :proc_result => block) {|page| 
+          get_token page
+          page.at('textarea[name=converted]').text
+        }#.res
+      else
+        @f.run(:save_result => !block) {|page| 
+          get_token page
+          tr text, direction, &block
+        }#.res
+      end
+    end
+    
+    def self.tr(*args, &block) new.tr *args, &block end
+    
+  end
+  
+  class Youtube < Service
+    URI = {:info => "http://www.youtube.com/get_video_info?video_id=%s"}
+    attr_reader :track
+    
+    def initialize frame=nil
+      super :dl, frame, :eval => false
+      @f.ss.each {|s| s.timeout=600}
+      require 'open3'
+      require 'mp3info'
+    end
+    
+    def dl(id, fd=nil, &block)
+      if block 
+        info(id) {|lnk| __dl(lnk, fd, block)}
+      else  __dl(info(id), fd)
+      end
+    end
+    
+    def dlmp3(id, mp3=nil)
+      dl(id) {|flv| 
+        if !File.file?(df = mp3||flv.sub(/.flv$/, '.mp3'))
+          Open3.popen3("ffmpeg -i '#{flv}' -ab 262144 -ar 44100 '#{df}'") {|i,o,e|
+            if $verbose
+              t = e.gets2 and t and t[/^size=/] and print t until e.eof?
+              puts "\n#{t}"
+            end
+          }
+        end
+        Mp3Info.open(df, :encoding=>'utf-8') {|mp3| 
+          mp3.tag2.TPE1, mp3.tag2.TIT2 = @track[1..2]
+        }  }
+    end
+
+    def self.dl(id) new.dl(id) end
+    def self.dlmp3(id) new.dlmp3(id) end
+    
+  private
+    def info(id, &block)
+      @f.run(URI.info%[id[/\/watch/] ?
+                               id.parseuri.query.v : 
+                               File.basename(id).till(/[&?]/)],:hash=>true,:proc_result=>block){|p|
+        res = p.hash
+        @track = [res.author, res.creator, res.title]
+        CGI.unescape(res.fmt_url_map).split(/,\d+\|/)[0].after('|')
+      }#.res
+    end
+    
+    def __dl(lnk,fd,block=nil)
+      @f.dl(lnk, fd||"files/youtube/#{@track*' - '}.flv", :auto, 5, &block)
+    end
+  
+  end
+
+  class VK < Service
+    attr_reader :links, :open_links
+    URI = {
+      :people => "http://vkontakte.ru/gsearch.php?from=people&ajax=1",
+      :login => "http://vkontakte.ru/index.php",
+      :id => "http://vkontakte.ru%s"
+    }
+    DefaultParams = Hash[*%w[
+      c[city] 1
+      c[country] 1
+      c[noiphone] 1
+      c[photo] 1
+      c[section] people
+      c[sex] 1
+      c[status] 6
+  ]]
+    @@reloadable = ReloadablePage {
+      if !@title and !@hash
+        L << self 
+        L << @doc
+      end
+      if @hash == false or @hash.nil? && (!@title or @title["Ошибка"])
+        L.info "@title caller.size", binding
+        sleep 2
+      end
+    }
+    def self.com; new end
+  
+    class NotFoundError < Exception; end
+        
+    def initialize frame=nil
+      super :people, frame, {:cp => true, :relvl => 5, :eval => false}, 5
+      @links = []
+      @open_links = []
+      login
+    end
+    
+    def login params={'email'=>'fshm@bk.ru', 'pass'=>'Riddick2', 'expire'=>nil}
+      Curl.run
+      @f[0].cookies.clear
+      @f.get(URI[:login], :json=>nil) {|login_page|
+        login_page.submit('form', @f, params, :json=>nil) {|redirection| 
+          redirection.submit('form', @f, {}, :json=>nil) {|logged|
+            @f.each {|s| s.cookies.replace @f[0].cookies}
+      }}}
+    end
+    
+    def get_links h, pagenum, &block
+      @f.run(h.merge('offset' => pagenum*20), URI[:people], :proc_result=>block, :result=>@@reloadable, :json => true) {|page|
+        ls = Page(page.hash.rows).get_links('.image/a')
+        @links.concat ls
+        ls
+      }
+    end
+    
+    def people(q, *args, &block)
+      age, opts = args.get_opts [17..23]
+      h = DefaultParams.merge('c[q]' => q)
+      h.merge! Hash[opts.map {|k,v| ["c[#{k}]", v]}]
+      h['c[age_from]'], h['c[age_to]'] = age.first, age.last
+      
+      @f.run(h, URI[:people], :proc_result => block, :json => true) {|page|
+      # ответом может быть невнятное требование залогиниться
+        sum = page.hash.summary.sub(/<span.+>/, '')
+        puts sum
+        found = sum[/\d+/]
+        if !found
+          L.warn sum 
+        else
+          @links.concat Page(page.hash.rows).get_links('.image/a')
+          max_page = [50, (found.to_f/20).ceil].min
+          (1...max_page).each {|_|
+            sleep 0.5
+            get_links h, _, &block
+          }
+        end
+      }
+    end
+    
+    def get_people q, *opts
+      @links = []
+      @open_links = []
+      people q, *opts
+      get_pages q
+    end
+    
+    def get_pages q=nil
+      @links.uniq.each {|id| get_page id, q; sleep 1.5}
+    end
+    
+    def get_page id, q=nil
+      q = q ? q.ci.to_re : // unless q.is Regexp
+      id_num = id[/\d+/].to_i
+      @f.get(id, :result=>@@reloadable) {|p|
+        data = p.find('.profileTable//.dataWrap').to_a.b
+        if data
+          L.debug "!p.at('.basicInfo//.alertmsg') data.contents.join('')[/(\\d\\s*){6,}/] data.contents.join('')[q]", binding
+        end
+        if data = p.find('.profileTable//.dataWrap').b and
+           contents = data.to_a.contents.join.b and contents[q]
+          digits = contents[/(\d *){6,9}/]
+          bot = (digits and digits[/^\d{7}$/] and id_num.between 852e5, 893e5)
+          if !bot and !p.at('.basicInfo//.alertmsg') || digits
+            L << "added vk.com#{id}"
+            @open_links << id
+          elsif bot
+            L << "bot #{id_num} detected"
+          else tick!
+          end
+        else tick!
+        end
+      }
+    end
+    
+  end
+  
+  class Mamba < Service
+    attr_reader :links, :open_links
+    @@login, @@pass = %w{AnotherOneUser AyaHirano8}
+    URI = {
+      :people => "http://mamba.ru/?",
+      :login => "http://mamba.ru/tips/?tip=Login",
+      :id => "http://vk.com%s"
+    }
+    DefaultParams = Hash[*%w[
+      c[city] 1
+      c[country] 1
+      c[noiphone] 1
+      c[photo] 1
+      c[section] people
+      c[sex] 1
+      c[status] 6
+  ]]
+        
+    def initialize frame=nil
+      super :people, frame, {:cp=>{
+        "PREV_LOGIN"=>"anotheroneuser", "LOGIN"=>"anotheroneuser", "UID"=>"494809761", "LEVEL"=>"Low", "bar"=>"AShwjUz54RmYnfClOdlMYZylGUU90PUxeFkwlGixrP2ARHDs3A0EbDDxQTEksEm4LPT8FfzpfdiMME1omFz0tVhA5QjcsCgckaSQfIDxI", "s"=>"MJt2J3U9Pnk7Qvpie13lN7rrqmahTrAk", "SECRET"=>"adqH47"},
+        :eval=>false, :timeout=>5, :retry=>['TimeoutError']
+      }, 5
+      @links = []
+      @open_links = []
+    end
+    
+    def login
+      @f.run(URI[:login]) {|p|
+        p.submit('.ap-t-c//form', @f, 'login'=>@@login, 'password'=>@@pass, 'level'=>nil) {
+          @f.each {|s| s.cookies.replace @f[0].cookies}
+        }
+      }
+      Curl.wait
+    end
+    
+    def people
+      # TODO
+      # ... or not TODO?
+    end
+    # seems like NOT... LOL
+    
+  end
+  
+  module Downloaders
+  
+    def letitbit(path, &block)
+      link = ''
+      frame = Frame 'letitbit.net', {:cp => true, :eval => nil}, 1
+      frame.run(path, :wait => !block) {|page1|
+        page1.submit('#ifree_form', frame) {|page2|
+          page2.submit('[action=/download4.php]', frame) {|page3|
+            page3.submit('[action=/download3.php]', frame) {|page4|
+              t = Thread.new {
+                sleep 60
+                frame.run({}, '/ajax/download3.php', 
+                                    :headers => {"Referer" => "http://letitbit.net/download3.php"}
+                             ) {|res|
+                  link << res.html
+                  block[link] if block
+                }
+              }
+              t.join if !block
+      }}}}
+      link
+    end
+    
+    module_function :letitbit
+  end
+  
+end

data/lib/rhack_in.rb

@@ -0,0 +1,2 @@
+require 'rhack'
+class Object; include RHACK end

data/lib/scout.rb

@@ -0,0 +1,591 @@
+# encoding: utf-8
+module Curl
+  
+  def ITT
+    res = nil
+    HTTPAccessKit::Scout('file://').loadGet(__FILE__) {|c| res = yield}
+    loop {if res then break res else sleep 0.01 end}
+  end
+  module_function :ITT
+  
+  class Response
+    __init__
+    attr_reader :header, :code, :body, :hash, :timestamp, :time, :req, :date, :error
+    
+    def to_s
+      str = '<#'
+      if @error
+        str << "#{@error[0].self_name}: #{@error[1]}"
+      else
+        str << (@header[/\d{3}/] == @code.to_s ? @header : "#{@header[/\S+/]} #{@code}") if @header
+        if @hash.location
+          str << ' '+@req.url if $panic
+          str << ' -> '+@hash.location 
+        end
+        str << " (#{@body ? @body.size.bytes : 'No'} Body)"
+        str << " [#{@timestamp}]" if @timestamp
+      end
+      str << '>'
+    end
+    alias :inspect :to_s
+    
+    def initialize(easy)
+      @hash = {}
+      @timestamp = @date = @header = nil
+      if easy.base.error
+        @error = easy.base.error
+      else
+        if headers = easy.header_str || easy.base.headers
+          headers /= "\r\n"
+          @header = headers.shift
+          headers.each {|h|
+            h /= ': '
+            if h[0]
+              h[0].downcase!
+              if h[0] == 'set-cookie'
+                (@hash.cookies ||= []) << h[1]
+              else
+                @hash[h[0]] = h[1]
+              end
+            end
+          }
+          @timestamp = if @hash.date
+              begin
+                @date = @hash.date.to_time
+              rescue => e
+                (@date = Time.now).strftime("%H:%M:%S")
+                L < "Error #{e.class}:#{e.message} with @hash.date = #{@hash.date.inspect}"
+              end
+              @hash.date[/\d\d:\d\d:\d\d/]
+            else
+              (@date = Time.now).strftime("%H:%M:%S")
+            end
+        end
+        @code	= easy.response_code
+        @body	= easy.body_str 
+        @time	= easy.total_time
+      end
+      
+      @req = {}
+      @req.url	      = easy.last_effective_url
+      @req.header	= easy.headers
+      if range = easy.headers.Range and range[/(\d+)-(\d+)/]
+        @req.range = $1.to_i .. $2.to_i
+      end
+      if easy.base and @req.meth = easy.base.last_method and @req.meth == :post
+        @req.body	= easy.post_body
+        @req.mp	  = easy.multipart_form_post?
+      end
+    end
+    
+    def is(klass)
+      if @error
+        klass == Array || klass = Curl::Response
+      else
+        klass == Curl::Response
+      end
+    end
+  
+    def [](key_or_index)
+      @error ? @error[key_or_index] : @hash[key_or_index.downcase]
+    end
+    
+  end
+  
+end
+
+module HTTPAccessKit
+
+  class Cookie
+    __init__
+    
+    def initialize(*args)
+      if args[1].is Scout
+        domain, str, scout = nil, *args
+        ck = str//;\s*/
+        ck[1..-1].each {|par|
+          a = par/'='
+          case a[0]
+            when 'path'; @path = (a[1] == '/') ? // : /^#{Regexp.escape a[1]}/
+            when 'domain'
+              if a[1].ord == ?.
+                domain = a[1][1..-1]
+                @domain = /(^|\.)#{Regexp.escape(domain)}$/
+              else 
+                domain = a[1]
+                @domain = /^#{Regexp.escape(a[1])}$/
+              end
+          end
+        }
+        @name, @value = ck[0]/'='
+        L.debug args if !domain 
+        (scout.cookies[domain || scout.uri.host] ||= {})[@name] = self
+      else
+        @name, cookie = args[0]
+        case cookie
+          when Array; @value, @path, @domain = cookie
+          when Hash; @value, @path, @domain = cookie.value, cookie.path, cookie.domain
+          else @value = args[1].to_s
+        end
+      end
+      @path ||= //
+      @domain ||= //
+      @string = "#{@name}=#{@value}; "
+    end
+    
+    def use(str, uri)
+      str << @string if uri.path[@path] and !uri.root || uri.host[@domain]
+    end
+      
+    def to_s; @value end
+    def inspect; @value.inspect end
+    
+  end
+
+  class Scout
+    __init__
+    attr_accessor	:timeout, :raise_err, :retry
+    attr_accessor	:path, :root, :sld, :proxy
+    attr_reader	    :uri
+    attr_reader	    :webproxy, :last_method, :proxystr, :headers, :body, :http, :error
+    attr_reader	    :cookies, :ua, :refforge, :cookieStore, :cookieProc
+    
+    DefaultHeader = {
+        "Expect"	              => "",
+        "Keep-Alive"	        => "300",
+        "Accept-Charset"	=> "windows-1251,utf-8;q=0.7,*;q=0.7",
+        "Accept-Language"	=> "ru,en-us;q=0.7,en;q=0.3",
+        "Connection"	        => "keep-alive"
+    }
+    
+    class ProxyError < ArgumentError
+      def initialize proxy
+        super "incorrect proxy: %s class %s, must be an Array
+            proxy format: ['127.0.0.1', '80'], [2130706433, 80], ['someproxy.com', :WebproxyModule]"%[proxy.inspect, proxy.class]
+      end
+    end
+    @@retry = RETRY
+    
+    def initialize(*argv)
+      uri, proxy, @ua, @refforge, opts = argv.get_opts ['http://', nil, :rand, 1]
+      raise ProxyError, proxy if proxy and (!webproxy && !proxy.is(Array) or webproxy && !proxy.is(String))
+      'http://' >> uri if uri !~ /^\w+:\/\//
+      if proxy
+        if proxy[1] and proxy[1].to_i == 0
+          @webproxy	= eval("WebProxy::#{proxy[1]}")
+          @proxy	    = proxy[0].parse(:uri).root
+        else 
+          proxy[0]	  = proxy[0].to_ip if proxy[0].is Integer
+          @proxy	  = proxy
+        end
+      end
+      @cookies    	= {}
+      @body       	= {}
+      @num    	    = []
+      @cookieProc	= opts[:cp] || opts[:ck]
+      @raise_err   	= opts[:raise] # no way to use @raise id, it makes any 'raise' call here fail
+      @engine     	= opts[:engine]
+      @timeout    	= opts[:timeout] || $CurlDefaultTimeout || 60
+      @post_proc	= @get_proc = @head_proc = Proc::NULL
+      update uri
+      @retry = opts[:retry] || {}
+      @retry = {@uri.host => @retry} if @retry.is Array
+    end
+    
+    def update(uri)
+      if !uri[/^\w+:\/\//]
+        '/' >> uri if uri[0,1] != '/'
+        @uri = uri.parse:uri
+        return                       
+      end
+      @uri = uri.parse:uri
+      return if @uri.root == @root
+      @root	= @uri.root
+      @sld	  = @root[/[\w-]+\.[a-z]+$/]
+      @path	= @uri.fullpath
+      if @http
+        @http.url = @webproxy ? @proxy : @root
+      else
+        @http = Curl::Easy(@webproxy ? @proxy : @root)
+        @http.base = self       
+      end
+      if @proxy
+        @http.proxy_url = @proxy*':' if !@webproxy
+        @proxystr = @webproxy ? @proxy[0] : @http.proxy_url
+      else @proxystr = 'localhost' 
+      end
+      if @cookieProc.is Hash
+        self.main_cks = @cookieProc
+        @cookieProc = true    
+      end
+      self
+    end
+    
+    def to_s
+      str = "<##{self.class.self_name} @ "
+      if @webproxy
+        str << "#{@proxy} ~ "
+      elsif @proxy
+        str << @proxy*':'+" ~ " 
+      end
+      str << @root+'>'
+    end
+    alias :inspect :to_s
+    
+    def update_res
+      @outdated = false
+      @res = @http.res
+      @headers = nil
+      @res
+    end
+    
+    def res
+      if @res && !@outdated
+             @res
+      else update_res end
+    end
+    
+    def req; res.req   end
+    
+    def dump
+      str = "IP: #{@proxystr}\nRequest: "
+      str << ({"Action"=>@root+@path} + @http.headers).dump+@body.dump+"Response: #{res}"
+      str << "\nReady" if @ready
+      str
+    end
+  
+    def fix(path)
+      path = path.tr ' ', '+'
+      path = expand path if path =~ /^\./
+      if update(path) or @uri.root
+        path = @webproxy.encode(path) if @webproxy
+      else
+        path = @webproxy.encode(@root+path) if @webproxy
+      end
+      path
+    end
+    
+    def expand(uri)
+      if !@webproxy || @http.last_effective_url
+        path = (@http.last_effective_url ? @http.last_effective_url.parse(:uri) : @uri).path
+        return uri.sub(/^(\.\.?\/)?/, File.split(uri =~ /^\.\./ ? File.split(path)[0] : path)[0])
+      end
+      uri
+    end
+
+    def mkBody(params, multipart=nil)
+      if multipart
+        @http.multipart_post_body = params.map {|k, v|
+          v = v.call if v.is Proc
+          if k =~ /^f:/
+            Curl::PostField.file(k[2..-1], "application/octet-stream", 
+                                     "#{randstr(16, :hex)}.jpg", v+randstr )
+          elsif k =~ /^p:/
+            Curl::PostField.file(k[2..-1], "application/octet-stream", 
+                                     File.basename(f), read(v)                   )
+          else
+            Curl::PostField.content(k.to_s, v.to_s)
+          end
+        }
+      else
+        @http.post_body = params.urlencode
+      end
+    end
+    
+    def mkHeader(uri)
+      header = DefaultHeader.dup
+      if @cookieProc
+        cookies = ''
+        main_cks.each_value {|v| v.use cookies, @uri}
+        header['Cookie'] = cookies[0..-3]                                 
+      end
+      if @refforge
+        ref = @uri.root ? uri : (@webproxy ? @http.host : @root)+uri
+        header['Referer'] = ref.match(/(.+)[^\/]*$/)[1]           
+      end
+      header['User-Agent'] = @ua == :rand ? UAS.rand : @ua if @ua
+      header
+    end
+      
+    def ProcCookies(res)
+      ck = []
+      case res
+        when String
+          res.split(/\r?\n/).each {|h|
+            hs = h/': '
+            ck << hs[1] if hs[0] and hs[0].downcase! == 'set-cookie'
+          }
+        when Curl::Response
+          ck = res['cookies']
+      end
+      return if !ck.b
+      ck.each {|c| Cookie(c, self)}
+  #    StoreCookies if @cookieStore
+    end
+
+    def cp_on() @cookieProc = true end
+    def cp_off() @cookieProc = false end
+    
+    def main_cks() @cookies[@uri.host] ||= {} end
+    def main_cks=(cks)
+      @cookies[@uri.host] = @webproxy ? 
+        @webproxy.ck_encode(@root, cks) : 
+        cks.map2 {|k, v| Cookie(k, v)}   
+    end
+    
+    def retry?(err)
+      # exc = ['0chan.ru', '2-ch.ru', 'www.nomer.org', 'nomer.org'].select_in('http://www.nomer.org') = ['www.nomer.org', 'nomer.org']
+      exc = (@@retry.keys + @retry.keys).select_in @root
+      return false if !exc.b
+      # ['www.nomer.org', 'nomer.org'].every |www| 'TiemoutError'.in({'nomer.org' => 'TimeoutError'}[www])} ?
+      !exc.find {|e| !err[0].self_name.in((@@retry[e] || []) + @retry[e])}
+    end
+    
+    def loaded?
+      $Carier.reqs.include? @http
+    end
+    
+    def load!
+      $log <= [$Carier, @http]
+      unless $Carier.add @http
+        $Carier.remove @http
+        $Carier.add @http
+      end
+    rescue RuntimeError => e
+      e.message << ". Failed to load allready loaded? easy handler: Bad file descriptor" unless Curl::Err::CurlError === e
+      raise e
+    end
+    
+    def load(path=@path, headers={}, not_redir=1, relvl=10)
+      @http.path = path = fix(path)
+      @http.headers = mkHeader(path).merge!(headers)
+      if not_redir.b
+        @http.follow_location = false
+      else
+        @http.follow_location = true
+        @http.max_redirects = relvl
+      end
+      @http.timeout = @timeout
+
+      @http.on_complete {|c|
+        @error = nil
+        @outdated = true
+        ProcCookies c.res if @cookieProc
+        yield c if block_given?
+      }
+      @http.on_failure {|c, e|
+        @http.on_complete {}
+        @outdated = true
+        @error = e
+        if retry? e
+          L.debug "#{e[0]} -> reloading scout"
+          #load uri, headers, not_redir, relvl, &callback
+          load! # all params including post_body are still set
+        else
+          L.debug "#{e[0]} -> not reloading scout"
+          raise *e if @raise_err
+        end
+      } if !@http.on_failure
+      
+      load!
+    end
+
+    def loadPost(*argv, &callback)
+      hash, multipart, uri, opts = argv.get_opts [@body, false, @path], 
+                                           :headers => {}, :redir => false, :relvl => 2
+      mkBody hash, multipart.b
+      @last_method	= :post
+      if block_given?
+        @post_proc	= callback
+      elsif @http.callback != @post_proc
+        callback = @post_proc 
+      end
+      load(uri, opts[:headers], !opts[:redir], opts[:relvl], &callback)
+    end
+    
+    def loadGet(*argv, &callback)
+      uri, opts = argv.get_opts [@path], 
+                     :headers => {}, :redir => false, :relvl => 2
+      @http.get	    = true
+      @last_method	= :get
+      if block_given?
+        @get_proc	  = callback
+      elsif @http.callback != @get_proc
+        callback = @get_proc 
+      end
+      load(uri, opts[:headers], !opts[:redir], opts[:relvl], &callback)
+    end
+    
+    def loadHead(*argv, &callback)
+      uri, emulate, headers = argv.get_opts [@path, :if_retry]
+      @http.head	  = true if emulate != :always
+      @last_method	= :head
+      if block_given?
+        @head_proc	= callback
+      elsif @http.callback != @head_proc
+        callback = @head_proc 
+      end
+      emu = lambda {
+        @headers = ''
+        $log << @headers
+        @http.on_header {|h|
+          $log << @headers
+          @headers << h
+          h == "\r\n" ? 0 : h.size
+        }
+        @http.get	    = true
+        load(uri, headers) {|c| c.on_header; callback[c]}
+      }
+      if emulate != :always
+        load(uri, headers) {|c|
+          if !@error and c.res.code != 200 and emulate == :if_retry
+            $log << @headers
+            emu.call
+          else
+            callback[c]
+          end
+        } 
+      else emu.call
+      end
+    end
+    
+  end
+  
+  class PickError < IndexError
+    def initialize
+      super "can't get scout from empty squad" end
+  end
+
+  class ScoutSquad < Array
+    __init__
+    
+    def initialize(*args)
+      raise ArgumentError, "can't create empty squad" if (num = args.pop) < 1
+      proxies = nil
+      super []
+      if args[0].is Scout
+        s = args[0]
+      else
+        if !args[0].is String
+          args.unshift ''
+          if (opts = args[-1]).is Hash and (opts[:cp] || opts[:ck]).is Hash
+            L.warn "it's useless to setup cookies for untargeted squad!"
+          end
+        end
+        if args[1] and args[1][0].is Array
+          proxies = args[1]
+          args[1] = proxies.shift
+        end
+        self[0] = s = Scout(*args)
+        num -=1
+      end
+      num.times {|i| 
+        self << Scout(s.root+s.path, (proxies ? proxies[i] : s.proxy), s.ua, s.refforge, :ck => s.main_cks, :raise => s.raise_err, :timeout => s.timeout, :retry => s.retry)
+      }
+    end
+    
+    def update uri, forced=nil
+      each {|s| return L.warn "failed to update scout loaded? with url: #{s.http.url}" if s.loaded?} if !forced
+      each {|s| s.update uri}
+    end
+    
+    def untargeted
+      first.root == 'http://'
+    end
+      
+    def rand
+      raise PickError if !b
+      # to_a because reject returns object of this class
+      if scout = to_a.rand {|_|!_.loaded?}; scout
+      else # Curl should run here, otherwise `next'/`rand'-recursion will cause stack overflow
+        raise "Curl must run in order to use ScoutSquad#rand" if !Curl.status
+        Curl.wait
+        self.rand 
+      end 
+    end
+      
+    def next
+      raise PickError if !b
+      if scout = find {|_|!_.loaded?}; scout
+      else # Curl should run here, otherwise `next'/`rand'-recursion will cause stack overflow
+        raise "Curl must run in order to use ScoutSquad#next" if !Curl.status
+        Curl.wait
+        self.next
+      end 
+    end
+    
+    def to_s
+      str = '<#ScoutSquad @ '
+      if b
+        if first.webproxy
+          str << "#{first.proxy} ~ "
+        elsif first.proxy
+          str << first.proxy*':'+" ~ " 
+        end
+        str << "#{untargeted ? "no target" : first.root} "
+      end
+      str << "x#{size}>"
+    end
+    alias :inspect :to_s
+    
+  end
+  
+end
+  
+  ### Global scope shortcut methods ###
+  
+module RMTools
+
+  def Get(uri, opts={})
+    raise ArgumentError, "Local uri passed to Get function" if uri[0,1] == '/'
+    $log.debug "Protocol-less uri passed to Get function" if !uri[/^\w+:\/\//]
+    headers	  = opts[:headers]	  || opts[:h]	  || {}
+    proxy	      = opts[:proxy]	    || opts[:pr]	|| $CurlGetProxy
+    ret_body	= opts.fetch(:ret_body, opts.fetch(:b, 1)).b
+    wait	      = opts.fetch(:wait, opts.fetch(:w, !block_given?)).b
+    s	        = HTTPAccessKit::Scout(uri, proxy, opts)
+    buf	    = ret_body ? '' : s.http.res
+    s.raise_err	    ||= opts[:e]
+    s.http.timeout ||= opts[:t]
+    s.loadGet(headers) {|c|
+      if ret_body
+        buf << c.body_str
+      else
+        buf.load_from c.res
+      end
+      yield buf if block_given?
+    }
+    if wait
+      ($CarierThread and $CarierThread.status) ? Curl.wait : $Carier.perform
+    end
+    buf
+  end
+  module_function :Get
+  
+end
+  
+module Enumerable
+  
+  def GetAll(on_count=nil, default_domain=nil, &callback)
+    if on_count
+      len = size
+      counter = 0
+      send(resto(:each_value) ? :each_value : :each) {|uri|
+        uri = File.join(default_domain, uri) if default_domain and (uri[0,1] == '/' or !uri[/^https?:/])
+        Get(uri) {|buf|
+          callback.arity > 1 ?
+            callback.call(buf, counter) :
+            callback.call(buf)
+          if (counter += 1) == len
+            on_count.arity > 0	?
+              on_count.call(buf)	:
+              on_count.call
+          end
+        }
+      }
+    else send(resto(:each_value) ? :each_value : :each) {|uri|
+          Get(uri, &callback)          }
+    end
+  end
+
+end