rhack 0.4.1 → 1.0.0.rc4

data/lib/rhack/js/johnson.rb

@@ -0,0 +1,71 @@
+# encoding: utf-8
+module Johnson
+  begin
+    require 'johnson'
+  rescue LoadError
+    Enabled = false
+  else
+    if VERSION <= "2.0.0" and RUBY_VERSION > "1.9"
+      Enabled = false
+    else Enabled = true
+    end
+  end
+  ### JavaScript interface DOM emulation ###
+  
+  class Runtime
+    cattr_accessor :browser
+    attr_accessor :thread_id
+    BROWSER_PATH = File.expand_path "../browser", __FILE__
+    
+    class << self
+      
+      def runtime_set?(opts)
+        !opts[:eval].b or (@@browser and @@browser.thread_id == Curl.carier_thread.object_id)
+      end
+    
+      # CarierThread breaks if Multi has no work && CarierThread
+      # is joined so itwon't last forever.
+      #
+      # Johnson is not thread safe =>
+      # Runtime created in this thread will become unusable after
+      # CarierThread dies. 
+      #
+      # So we don't use Curl.wait until Carier haven't got whole
+      # request for this Runtime.
+      def set_browser_for_curl(opts)
+        unless runtime_set? opts
+          if Curl.status
+            Curl.recall
+            Curl.debug 'recalled'
+          end
+          if opts[:thread_safe].b
+            @@browser = new_browser(opts[:jq])
+            L.debug "#@@browser initialized in #{Thread.current}\nmain: #{Thread.main}; carier: #{Curl.carier_thread}"
+          else
+            L.debug 'about to run carier'
+            Curl.execute {@@browser = new_browser(opts[:jq])
+                               L.debug "#@@browser initialized in #{Thread.current}\nmain: #{Thread.main}; carier: #{Curl.carier_thread}"}
+            sleep 0.01 until runtime_set? opts
+          end
+        end
+      end
+    
+      def new_browser(jq=false)
+        rt = new
+        %w{xmlw3cdom_1 xmlw3cdom_2 xmlsax env}.concat(jq ? ['jquery'] : []).each {|f|
+          path = "#{BROWSER_PATH}/#{f}.js"
+          rt.evaluate IO.read(path), path, 1 
+        }
+        rt.document = ''
+        rt
+      end
+    
+    end
+    
+    def document=(html)
+      evaluate "var document = new DOMDocument(#{html.to_doc.to_xhtml.inspect})"
+    end
+    
+  end
+  
+end

data/lib/rhack/page.rb

@@ -0,0 +1,263 @@
+# encoding: utf-8
+module RHACK
+
+    # Frame( ScoutSquad( Curl::Multi <- Scout( Curl API ), Scout, ... ) ) => 
+    # Curl -> Johnson::Runtime -> XML::Document => Page( XML::Document ), Page, ... 
+    
+    class Page
+    # for debug, just enable L#debug, don't write tons of chaotic log-lines 
+    __init__
+    attr_writer :title
+    attr_reader :html, :loc, :hash, :doc, :js, :curl_res, :failed
+    # result of page processing been made in frame context
+    attr_accessor :res
+    # for johnson
+    @@ignore = /google|_gat|tracker|adver/i
+      
+    def initialize(obj='', loc=Hash.new(''), js=Johnson::Runtime.browser||Johnson::Runtime.new)
+      loc = loc.parse:uri if !loc.is Hash
+      @js = js
+      if obj.is Curl::Easy or obj.kinda Scout
+        c = obj.kinda(Scout) ? obj.http : obj
+        @html = ''
+        # just (c, loc) would pass to #process opts variable that returns '' on any key
+        process(c, loc.b || {})
+      else
+        @html = obj
+        @loc = loc
+      end
+    end
+    
+    def empty?
+      !(@hash.nil? ? @html : @hash).b
+    end
+        
+    def inspect
+      if !@hash.nil?
+        "<#FramePage (#{@hash ? @hash.inspect.size.bytes : 'failed to parse'}) #{@json ? 'json' : 'params hash'}>"
+      else
+        "<#FramePage #{@html.b ? "#{@failed ? @curl_res.header : '«'+title(false)+'»'} (#{@html.size.bytes}" : '(empty'})#{' js enabled' if @js and @doc and @hash.nil?}>"
+      end
+    end
+    
+    def html!(encoding='UTF-8')
+      @html.force_encoding(encoding)
+    end
+    
+    # We can then alternate #process in Page subclasses
+    # Frame doesn't mind about value returned by #process
+    def process(c, opts={})
+      @loc = c.last_effective_url.parse:uri
+      @curl_res = c.res
+      L.debug "#{@loc.fullpath} -> #{@curl_res}"
+      if @curl_res.code == 200
+        body = @curl_res.body
+        if opts[:json]
+          @json = true
+          @hash = begin; body.from_json
+          rescue StandardError
+            false 
+          end
+          if !@hash or @hash.is String
+            L.debug "failed to get json from #{c.last_effective_url}, take a look at my @doc for info; my object_id is #{object_id}"
+            @html = body; to_doc
+            @hash = false
+          end
+          
+        elsif opts[:hash]
+          if body.inline
+            @hash = body.to_params
+          else
+            @hash = false
+            L.debug "failed to get params hash from #{c.last_effective_url}, take a look at my @doc for info; my object_id is #{object_id}"
+            @html = body; to_doc
+          end
+          
+        else
+          @html = body.xml_to_utf
+          to_doc
+          if opts[:eval]
+            load_scripts opts[:load_scripts]
+            eval_js
+          end
+        end
+      elsif !(opts[:json] or opts[:hash])
+        @html = @curl_res.body
+        @failed = @curl_res.code
+      end
+      self
+    end
+    
+    def eval_js(frame=nil)
+      eval_string "document.location = window.location = #{@loc.to_json};
+      document.URL = document.baseURI = document.documentURI = location.href;
+      document.domain = location.host;"
+      find("script").each {|n|
+        L.debug n.text.strip
+        if text = n.text.strip.b
+          js[:write_output] = ''
+          eval_string text
+          if res = js[:write_output].b then n.after res end
+          n.remove!
+        elsif frame and n.src
+          eval_string frame.get_cached expand_link n.src
+        end
+      }
+    end
+    
+    def eval_string(str)
+      @js ||= Johnson::Runtime.new
+      L.debug "#{@js} evaluating in #{Thread.current}\nmain: #{Thread.main}; carier: #{Curl.carier_thread}"
+      begin
+        @js.evaluate(str)
+      rescue Johnson::Error => e
+        L.warn e.message
+        L.debug {
+          if m = e.message.match(/(\w+) is undefined|([\w.]+) is not a function/)
+            L.clr.hl! str, /\b#{m[1] || m[2]}\b/
+          end
+          "\n\t#{str}"
+        }
+      end
+    end
+    
+    def to_doc
+      @doc = @html.to_doc :forceutf
+    end
+    
+    def title(full=true)
+      if @hash.nil? and !@failed and @html.b
+        if full
+          to_doc unless defined? @doc
+          if @doc.title.b
+            @title = @doc.title
+          else
+            @title = @loc.href
+            @doc.at('head').prepend XML::Node('title', @title) if @doc.at('head')
+            @title
+          end
+        else
+          title true unless defined? @title
+          if RUBY_VERSION < '1.9' and @title.cyr? and UTF2ANSI[@title].size > 40
+            @short_title = ANSI2UTF[UTF2ANSI[@title][/.{1,30}\S*/][0..38]]+'…'
+          elsif @title.size > 40
+            @short_title = @title[/.{1,30}\S*/][0..38]+'…'
+          else
+            @short_title = @title
+          end
+        end
+      else
+        @loc.href
+      end
+    end
+      
+    def find(xp) (@doc || to_doc).find xp end
+    
+    def at(xp) (@doc || to_doc).at xp end
+    
+    def url() @loc.href end
+    alias :href :url
+    
+    def get_srcs(links='img')
+      begin
+        links = find(links).map {|e| e.src} if links.is String
+      rescue XML::Error
+        links = [links]
+      end
+      links.map {|link| expand_link link}.uniq
+    end
+    
+    def get_src(link='img')
+      begin
+        link = at(link) && at(link).src if link.is String
+      rescue XML::Error; nil
+      end
+      expand_link link if link
+    end
+    
+    def get_links(links='a')
+      begin
+        links = find(links).map {|e| e.href}.b || find(links+'//a').map {|e| e.href} if links.is String
+      rescue XML::Error
+        links = [links]
+      end
+      links.map {|link| expand_link link}.uniq
+    end
+    
+    def get_link(link='a')
+      begin
+        link = at(link) && (at(link).href || at(link+'//a').href) if link.is String
+      rescue XML::Error; nil
+      end
+      expand_link link if link
+    end
+    alias :get_hrefs :get_links
+    alias :links :get_links
+    alias :get_href :get_link
+    alias :link :get_link
+    alias :srcs :get_srcs
+    alias :src :get_src
+    
+    def expand_link(link)
+      case link
+        when /^\w+:\/\// then link
+        when /^\/\// then @loc.protocol+link
+        when /^\// then @loc.root+link
+        else File.join((@loc.path.b ? File.dirname(@loc.path) : @loc.root), link)
+      end
+    end
+    
+    def form(form='form', hash={}, opts={})
+      form = "[action=#{@loc.path.inspect}]" if form == :self
+      if form.is String
+             form_node = at form
+             raise XML::Error, "Can't find form by xpath `#{form}` on page #{inspect}" if !form_node or form_node.name != 'form'
+      else form_node = form
+      end
+      hash = form_node.inputs_all.merge!(hash)
+      action = expand_link(form_node.action || @loc.path)
+      if form_node['method'].downcase == 'post'
+        [hash, form_node.enctype =~ /multipart/, action, opts]
+      else
+        action = "#{action}#{action['?'] ? '&' : '?'}#{hash.urlencode}" if hash.b
+        [action, opts]
+      end
+    end
+      
+    def submit(form, frame, hash={}, opts={}, &callback)
+      (opts[:headers] ||= {}).Referer ||= @loc.href if @loc
+      query = form(form, hash, opts)
+      
+      curr_target, new_target = frame.loc.href, (query[2] || query[0])
+      if need_retargeting = (frame.static && curr_target != new_target)
+        frame.retarget new_target
+      end
+      page = frame.exec(*query, &callback)
+      frame.retarget curr_target, :forced if need_retargeting
+      page
+    end
+    
+    def load_scripts(frame)
+      frame && frame.get_cached(*get_srcs("script[src]")).each {|js| eval_string js}
+    end
+    
+  end
+
+  # using reprocessing of page in case of non-200 response:
+  # page_class = ReloadablePage do
+  #   @res and @res.code != 200
+  # end
+  def ReloadablePage(&reload_condition)
+    rp = Class.new Page
+    rp.send :define_method, :process do |curl, opts|
+      super(curl, opts || {})
+      if curl.instance_eval &reload_condition
+        curl.retry!
+        nil # in case of reload_condition.call super's callback will not proceed
+      else self
+      end
+    end
+    rp
+  end
+  
+end

data/lib/rhack/proxy.rb

@@ -0,0 +1,3 @@
+require 'rhack'
+require 'rhack/proxy/list'
+require 'rhack/proxy/checker'

data/lib/rhack/proxy/checker.rb

@@ -67,7 +67,7 @@ module RHACK
         }
         target = target.find_is(String) if !target.is String
         pl.each {|pr|
-          sc = Interceptor.new(target, pr, $uas.rand, @opts)
+          sc = Interceptor.new(target, pr, RHACK.useragents.rand, @opts)
           sc.http.on_failure(&fail_proc)
           @ics << sc
         }

data/lib/rhack/scout.rb

@@ -0,0 +1,342 @@
+# encoding: utf-8
+module RHACK
+
+  class Scout
+    __init__
+    attr_accessor	:timeout, :raise_err, :retry
+    attr_accessor	:path, :root, :sld, :proxy
+    attr_reader	    :uri
+    attr_reader	    :webproxy, :last_method, :proxystr, :headers, :body, :http, :error
+    attr_reader	    :cookies, :ua, :refforge, :cookieStore, :cookieProc
+    
+    DefaultHeader = {
+        "Expect"	              => "",
+        "Keep-Alive"	        => "300",
+        "Accept-Charset"	=> "windows-1251,utf-8;q=0.7,*;q=0.7",
+        "Accept-Language"	=> "ru,en-us;q=0.7,en;q=0.3",
+        "Connection"	        => "keep-alive"
+    }
+    
+    class ProxyError < ArgumentError
+      def initialize proxy
+        super "incorrect proxy: %s class %s, must be an Array
+            proxy format: ['127.0.0.1', '80'], [2130706433, 80], ['someproxy.com', :WebproxyModule]"%[proxy.inspect, proxy.class]
+      end
+    end
+    
+    def initialize(*argv)
+      uri, proxy, @ua, @refforge, opts = argv.get_opts ['http://', nil, :rand, 1]
+      raise ProxyError, proxy if proxy and (!webproxy && !proxy.is(Array) or webproxy && !proxy.is(String))
+      'http://' >> uri if uri !~ /^\w+:\/\//
+      if proxy
+        if proxy[1] and proxy[1].to_i == 0
+          @webproxy	= eval("WebProxy::#{proxy[1]}")
+          @proxy	    = proxy[0].parse(:uri).root
+        else 
+          proxy[0]	  = proxy[0].to_ip if proxy[0].is Integer
+          @proxy	  = proxy
+        end
+      end
+      @cookies    	= {}
+      @body       	= {}
+      @num    	    = []
+      @cookieProc	= opts[:cp] || opts[:ck]
+      @raise_err   	= opts[:raise] # no way to use @raise id, it makes any 'raise' call here fail
+      @engine     	= opts[:engine]
+      @timeout    	= opts[:timeout] || @@timeout || 60
+      @post_proc	= @get_proc = @head_proc = @put_proc = @delete_proc = Proc::NULL
+      update uri
+      @retry = opts[:retry] || {}
+      @retry = {@uri.host => @retry} if @retry.is Array
+      
+      @http.cacert = @@cacert
+    end
+    
+    def update(uri)
+      if !uri[/^\w+:\/\//]
+        '/' >> uri if uri[0,1] != '/'
+        @uri = uri.parse:uri
+        return                       
+      end
+      @uri = uri.parse:uri
+      return if @uri.root == @root
+      @root	= @uri.root
+      @sld	  = @root[/[\w-]+\.[a-z]+$/]
+      @path	= @uri.fullpath
+      if @http
+        @http.url = @webproxy ? @proxy : @root
+      else
+        @http = Curl::Easy(@webproxy ? @proxy : @root)
+        @http.base = self       
+      end
+      if @proxy
+        @http.proxy_url = @proxy*':' if !@webproxy
+        @proxystr = @webproxy ? @proxy[0] : @http.proxy_url
+      else @proxystr = 'localhost' 
+      end
+      if @cookieProc.is Hash
+        self.main_cks = @cookieProc
+        @cookieProc = true    
+      end
+      self
+    end
+    
+    def to_s
+      str = "<##{self.class.self_name} @ "
+      if @webproxy
+        str << "#{@proxy} ~ "
+      elsif @proxy
+        str << @proxy*':'+" ~ " 
+      end
+      str << @root+'>'
+    end
+    alias :inspect :to_s
+    
+    def update_res
+      @outdated = false
+      @res = @http.res
+      @headers = nil
+      @res
+    end
+    
+    def res
+      if @res && !@outdated
+             @res
+      else update_res end
+    end
+    
+    def req; res.req   end
+    
+    def dump
+      str = "IP: #{@proxystr}\nRequest: "
+      str << ({"Action"=>@root+@path} + @http.headers).dump+@body.dump+"Response: #{res}"
+      str << "\nReady" if @ready
+      str
+    end
+  
+    def fix(path)
+      path = path.tr ' ', '+'
+      path = expand path if path =~ /^\./
+      if update(path) or @uri.root
+        path = @webproxy.encode(path) if @webproxy
+      else
+        path = @webproxy.encode(@root+path) if @webproxy
+      end
+      path
+    end
+    
+    def expand(uri)
+      if !@webproxy || @http.last_effective_url
+        path = (@http.last_effective_url ? @http.last_effective_url.parse(:uri) : @uri).path
+        return uri.sub(/^(\.\.?\/)?/, File.split(uri =~ /^\.\./ ? File.split(path)[0] : path)[0])
+      end
+      uri
+    end
+
+    def mkBody(params, multipart=nil)
+      if multipart
+        @http.multipart_post_body = @body = params.map {|k, v|
+          v = v.call if v.is Proc
+          if k =~ /^f:/
+            Curl::PostField.file(k[2..-1], "application/octet-stream", 
+                                     "#{randstr(16, :hex)}.jpg", v+randstr )
+          elsif k =~ /^p:/
+            Curl::PostField.file(k[2..-1], "application/octet-stream", 
+                                     File.basename(f), read(v)                   )
+          else
+            Curl::PostField.content(k.to_s, v.to_s)
+          end
+        }
+      else
+        @http.post_body = @body = params.urlencode
+      end
+    end
+    
+    def mkHeader(uri)
+      header = DefaultHeader.dup
+      if @cookieProc
+        cookies = ''
+        main_cks.each {|k, v| main_cks.delete k if v.use(cookies, @uri) == :expired}
+        header['Cookie'] = cookies[0..-3]                                 
+      end
+      if @refforge
+        ref = @uri.root ? uri : (@webproxy ? @http.host : @root)+uri
+        header['Referer'] = ref.match(/(.+)[^\/]*$/)[1]           
+      end
+      header['User-Agent'] = @ua == :rand ? RHACK.useragents.rand : @ua if @ua
+      header
+    end
+      
+    def ProcCookies(res)
+      ck = []
+      case res
+        when String
+          res.split(/\r?\n/).each {|h|
+            hs = h/': '
+            ck << hs[1] if hs[0] and hs[0].downcase! == 'set-cookie'
+          }
+        when Curl::Response
+          ck = res['cookies']
+      end
+      return if !ck.b
+      ck.each {|c| Cookie(c, self)}
+  #    StoreCookies if @cookieStore
+    end
+
+    def cp_on() @cookieProc = true end
+    def cp_off() @cookieProc = false end
+    
+    def main_cks() @cookies[@uri.host] ||= {} end
+    def main_cks=(cks)
+      @cookies[@uri.host] = @webproxy ? 
+        @webproxy.ck_encode(@root, cks) : 
+        cks.map2 {|k, v| Cookie(k, v)}   
+    end
+    
+    def retry?(err)
+      # exc = ['0chan.ru', '2-ch.ru', 'www.nomer.org', 'nomer.org'].select_in('http://www.nomer.org') = ['www.nomer.org', 'nomer.org']
+      exc = (@@retry.keys + @retry.keys).select_in @root
+      return false if !exc.b
+      # ['www.nomer.org', 'nomer.org'].every {|www| 'TimeoutError'.in({'nomer.org' => 'TimeoutError'}[www])} ?
+      exc.no? {|e| err[0].self_name.in((@@retry[e] || []) + @retry[e])}
+    end
+    
+    def loaded?
+      Curl.carier.reqs.include? @http
+    end
+    
+    def load!
+      unless Curl.carier.add @http
+        Curl.carier.remove @http
+        Curl.сarier.add @http
+      end
+    rescue RuntimeError => e
+      e.message << ". Failed to load allready loaded? easy handler: Bad file descriptor" unless Curl::Err::CurlError === e
+      raise e
+    end
+    
+    def load(path=@path, headers={}, not_redir=1, relvl=10, &callback)
+      @http.path = path = fix(path)
+      @http.headers = mkHeader(path).merge!(headers)
+      @http.timeout = @timeout
+
+      @http.on_complete {|c|
+        @error = nil
+        @outdated = true
+        ProcCookies c.res if @cookieProc
+        # We cannot just cancel on_complete in on_redirect block
+        # because loadGet will immediately reset on_complete back
+        if c.res.code.in(300..399) and !not_redir.b and (relvl -= 1) > -1 and loc = c.res.hash.location
+          loadGet(loc, headers: headers, relvl: relvl, redir: true, &callback)
+        elsif block_given?
+          yield c
+        end
+      }
+      @http.on_failure {|c, e|
+        if e[0] == Curl::Err::CurlOK
+          @error = e
+          # TODO: где-то в сорцах on_failure вызывается по коду 0, видимо из-за стороннего условия, а не должен
+          L.log << "Got Curl::Err::CurlOK, response was: #{c.res}"
+        else
+          @http.on_complete &Proc::NULL
+          @outdated = true
+          if retry? e
+            L.debug "#{e[0]} -> reloading scout"
+            #load uri, headers, not_redir, relvl, &callback
+            load! # all params including post_body are still set
+          else
+            L.debug "#{e[0]} -> not reloading scout"
+            raise *e if @raise_err
+          end
+        end
+      } if !@http.on_failure
+      
+      load!
+    end
+    
+    def loadGet(*argv, &callback)
+      uri, opts = argv.get_opts [@path], 
+                     :headers => {}, :redir => false, :relvl => 2
+      @http.get	    = true
+      @last_method	= :get
+      if block_given?
+        @get_proc	  = callback
+      else#if @http.callback != @get_proc
+        callback = @get_proc 
+      end
+      load(uri, opts[:headers], !opts[:redir], opts[:relvl], &callback)
+    end
+
+    def loadDelete(*argv, &callback)
+      uri, opts = argv.get_opts [@path], 
+                     :headers => {}, :redir => false, :relvl => 2
+      @http.delete = true
+      @last_method	= :delete
+      if block_given?
+        @delete_proc	= callback
+      else#if @http.callback != @post_proc
+        callback = @delete_proc 
+      end
+      load(uri, opts[:headers], !opts[:redir], opts[:relvl], &callback)
+    end
+
+    def loadPost(*argv, &callback)
+      hash, multipart, uri, opts = argv.get_opts [@body, @http.multipart_form_post?, @path], :headers => {}, :redir => false, :relvl => 2
+      @http.delete = false
+      mkBody hash, multipart.b
+      @last_method	= :post
+      if block_given?
+        @post_proc	= callback
+      else#if @http.callback != @post_proc
+        callback = @post_proc 
+      end
+      load(uri, opts[:headers], !opts[:redir], opts[:relvl], &callback)
+    end
+
+    def loadPut(*argv, &callback)
+      body_or_file, uri, opts = argv.get_opts [@body, @path], 
+                             :headers => {}, :redir => false, :relvl => 2
+      @http.delete = false
+      @http.put_data = @body = body_or_file
+      @last_method	= :put
+      if block_given?
+        @put_proc	= callback
+      else#if @http.callback != @post_proc
+        callback = @put_proc 
+      end
+      load(uri, opts[:headers], !opts[:redir], opts[:relvl], &callback)
+    end
+    
+    def loadHead(*argv, &callback)
+      uri, emulate, headers = argv.get_opts [@path, :if_retry]
+      @http.head	  = true if emulate != :always
+      @last_method	= :head
+      if block_given?
+        @head_proc	= callback
+      else#if @http.callback != @head_proc
+        callback = @head_proc 
+      end
+      emu = lambda {
+        @headers = ''
+        @http.on_header {|h|
+          @headers << h
+          h == "\r\n" ? 0 : h.size
+        }
+        @http.get	    = true
+        load(uri, headers) {|c| c.on_header; callback[c]}
+      }
+      if emulate != :always
+        load(uri, headers) {|c|
+          if !@error and c.res.code != 200 and emulate == :if_retry
+            emu.call
+          else
+            callback[c]
+          end
+        } 
+      else emu.call
+      end
+    end
+    
+  end
+  
+end