rhack 0.3.3 → 0.4.0

data/lib/curl-global.rb

@@ -1,9 +1,10 @@
 # encoding: utf-8
 module Curl
   
-  def execute
-    # если ты ищешь, откуда оно запускается, то это наверное
-    # lib/extensions/johnson.rb:37:in `set_browser_for_curl'
+  def execute(unless_allready=false)
+    if unless_allready and Curl.status
+      return L.debug "Non-nil status! Avoid executing"
+    end
     if $CarierThread and s = $CarierThread.status
       L.debug "Carier thread allready started and has status #{s}"
     else
@@ -33,13 +34,16 @@ module Curl
         } unless error
         error
       }
+      # until main thread has sleep a bit, $CarierThread will have status "run", 
+      # no matter whether it's idling or performing requests
+      sleep 0.001
     end
   end
   alias :run :execute
   module_function :execute, :run
   
   def wait
-    if $CarierThread
+    if $CarierThread and $CarierThread.status
       if !(within = Thread.current == $CarierThread)
         # We can't set `perform' timeout lesser than 1 second in the curl binding
         # because in that case thread status would always be "run"
@@ -84,7 +88,9 @@ module Curl
         end
       end
     else
-      L.debug "No thread to wait"
+      L < "No thread to wait. I guess I should create one"
+      execute
+      wait
     end
   end
   module_function :wait

data/lib/frame.rb

@@ -21,11 +21,12 @@ module HTTPAccessKit
 
   class Frame
     __init__
-    attr_reader :loc, :static, :ss, :opts
+    attr_reader :loc, :static, :ss, :opts, :use_cache, :write_to
+    @@cache = {}
     
     def initialize *args
       args << 10 unless args[-1].is Fixnum
-      args[-2] = {} unless args[-2].is Hash
+      args.insert -2, {} unless args[-2].is Hash
       @opts = {:eval => Johnson::Enabled, :redir => true, :cp => true, :result => Page}.merge!(args[-2].kinda(Hash) ? args[-2] : {})
       args[-2] = @opts
       if args[0].is String
@@ -39,8 +40,7 @@ module HTTPAccessKit
         @static = false
       end
       @ss  = ScoutSquad *args
-      @pages = []
-      Curl.run unless Curl.status
+      Curl.run :unless_allready
     end
     
     def retarget to, forced=nil
@@ -48,23 +48,40 @@ module HTTPAccessKit
       @ss.update to, forced
       @loc = to.parse:uri
     end
-    
-    def target=to
-      retarget to
-    end
+    alias :target= :retarget
     
     def next() @ss.next end
     def rand() @ss.rand end
     def each(&block) @ss.each &block end
     def [](i) @ss[i] end
     
+    def copy_cookies! i=0
+      @ss.each {|s| s.cookies.replace @ss[i].cookies}
+    end
+    
+    def use_cache! opts={}
+      if opts == false
+        @use_cache = false
+      else
+        @@cache = opts[:pages].kinda(Hash) ? opts[:pages] : opts[:pages].map_hash {|p| [p.href, p]} if opts[:pages]
+        #@write_to = opts[:write_to] if :write_to.in opts
+        @use_cache = true
+      end
+    end
+    
+    def drop_cache! use=nil
+      @@cache.clear
+      GC.start
+      @use_cache = use if use.in [true, false]      
+    end
+    
     def inspect
       "<#Frame @ #{@ss.untargeted ? 'no target' : @loc.root}: #{'scout'.x @ss.size}#{', static' if @static}, cookies #{@ss[0].cookieProc ? 'on' : 'off'}>"
     end
     
     # opts are :eval, :json, :hash, :wait, :proc_result, :save_result, :load_scripts, 
     # :zip, :thread_safe, :result, :stream, :raw, :xhr + any opts for Scouts in one hash
-    def get *args, &callback
+    def exec *args, &callback
       many, order, orders, with_opts = interpret_request *args
       L.log({:many => many, :order => order, :orders => orders, :with_opts => with_opts})
       
@@ -79,7 +96,7 @@ module HTTPAccessKit
       if many then	exec_many orders, with_opts, &callback
       else 	          exec_one order, with_opts, &callback    end
     end
-    alias :exec :get
+    alias :get :exec
     alias :run :get
     
     def interpret_request(*args)
@@ -431,28 +448,46 @@ module HTTPAccessKit
       uris.map! {|u| validate u}
     end
     
-    def exec_one(order, opts)
+    def run_callbacks!(page, opts, &callback)
+      if callback
+        yres = callback.call page
+        if opts[:save_result] or :proc_result.in opts
+          page.res = yres
+        end
+        if opts[:proc_result].is Proc and yres != :skip
+          opts[:proc_result].call yres
+        end
+      end
+    end
+    
+    # TODO: found why/how IO on callbacks breaks +curl.res.body+ content and how to fix or how to avoid it
+    def exec_one(order, opts, &callback)
+      if @use_cache and order[0] == :loadGet and page = @@cache[order[1]]
+        run_callbacks! page, opts, &callback
+        res = opts[:wait] && (opts[:save_result] or :proc_result.in opts) ? page.res : page
+        return res
+      end
       # must result in Page (default) or it's subclass
       page = opts[:result].new
-      # if no spare scouts can be found, squad simply waits for all callbacks to complete
+      # if no spare scouts can be found, squad simply waits for first callbacks to complete
       s = @ss.next
-      #s.raise_err = true#  Зачем это тут? Можно добавлять :raise=>1 фрейму при запиле
       s.send(*(order << opts)) {|curl|
+      #   there is a problem with storing html on disk
+        if order[0] == :loadGet and @write_to
+      #     sometimes  (about 2% for 100-threads-dling) when this string is calling
+      #     no matter what +curl.res.body+ has contained here
+          RMTools.rw @write_to+'/'+order[-2].sub(/^[a-z]+:\/\//, ''), curl.res.body.xml_to_utf
+        end
         if opts[:raw]
           yield curl
-        elsif page.process(curl, opts) and block_given?
-          yres = yield page
-          if opts[:save_result] or :proc_result.in opts
-            page.res = yres
-          end
-          if opts[:proc_result].is Proc and yres != :skip
-            opts[:proc_result].call yres
-          end
+      #   here +curl.res.body+ become empty
+        elsif page.process(curl, opts)
+          @@cache[page.href] = page if order[0] == :loadGet and @use_cache
+          run_callbacks! page, opts, &callback
         end
       }
       if opts[:wait]
         opts[:thread_safe] ? $Carier.perform : Curl.wait
-        # почему бы не уменьшить бойлерплейт в сервисах и не возвращать res сразу?
         (opts[:save_result] or :proc_result.in opts) ? page.res : page
       else page
       end
@@ -530,39 +565,49 @@ module HTTPAccessKit
     # for debug, just enable L#debug, don't write tons of chaotic log-lines 
     __init__
     # res here is result of page processing made in frame context
-    attr_accessor :title, :res
-    attr_reader :html, :loc, :hash, :doc, :js
+    attr_writer :title
+    attr_reader :html, :loc, :hash, :doc, :js, :curl_res, :failed
+    attr_accessor :res
     @@ignore = /google|_gat|tracker|adver/i
       
     def initialize(obj='', loc=Hash.new(''), js=$JSRuntime||Johnson::Runtime.new)
       loc = loc.parse:uri if !loc.is Hash
       @js = js
       if obj.is Curl::Easy or obj.kinda Scout
-        c = obj.kinda(Scout) ? obj.http : html
+        c = obj.kinda(Scout) ? obj.http : obj
         @html = ''
         # just (c, loc) would pass to #process opts variable that returns '' on any key
-        process(c, loc.b || {}) 
+        process(c, loc.b || {})
       else
         @html = obj
         @loc = loc
       end
     end
+    
+    def empty?
+      !(@hash.nil? ? @html : @hash).b
+    end
         
     def inspect
       if !@hash.nil?
         "<#FramePage (#{@hash ? @hash.inspect.size.bytes : 'failed to parse'}) #{@json ? 'json' : 'params hash'}>"
       else
-        "<#FramePage #{@html.b ? "«#{@title}» (#{@html.size.bytes}" : '(empty'})#{' js enabled' if @js and @doc and @hash.nil?}>"
+        "<#FramePage #{@html.b ? "#{@failed ? @curl_res.header : '«'+title(false)+'»'} (#{@html.size.bytes}" : '(empty'})#{' js enabled' if @js and @doc and @hash.nil?}>"
       end
     end
     
+    def html!(encoding='UTF-8')
+      @html.force_encoding(encoding)
+    end
+    
     # We can then alternate #process in Page subclasses
     # Frame doesn't mind about value returned by #process
     def process(c, opts={})
       @loc = c.last_effective_url.parse:uri
-      L.debug "#{@loc.fullpath} -> #{c.res}"
-      if c.res.code == 200
-        body = c.res.body
+      @curl_res = c.res
+      L.debug "#{@loc.fullpath} -> #{@curl_res}"
+      if @curl_res.code == 200
+        body = @curl_res.body
         if opts[:json]
           @json = true
           @hash = begin; body.from_json
@@ -577,7 +622,7 @@ module HTTPAccessKit
           
         elsif opts[:hash]
           if body.inline
-            @hash = body.to_hash
+            @hash = body.to_params
           else
             @hash = false
             L.debug "failed to get params hash from #{c.last_effective_url}, take a look at my @doc for info; my object_id is #{object_id}"
@@ -585,12 +630,16 @@ module HTTPAccessKit
           end
           
         else
-          @html = body; to_doc
+          @html = body.xml_to_utf
+          to_doc
           if opts[:eval]
             load_scripts opts[:load_scripts]
             eval_js
           end
         end
+      elsif !(opts[:json] or opts[:hash])
+        @html = @curl_res.body
+        @failed = @curl_res.code
       end
       self
     end
@@ -630,17 +679,32 @@ module HTTPAccessKit
     
     def to_doc
       @doc = @html.to_doc :forceutf
-      if !(@title = @doc.title.b)
-        @title = @loc.href
-        @doc.at('head').prepend XML::Node('title', @title) if @doc.at('head')
-      else
-        if @title.cyr? and UTF2ANSI[@title].size > 40
-          @title = ANSI2UTF[UTF2ANSI[@title][/.{1,30}\S*/]]+'…'
-        elsif @title.size > 40
-          @title = @title[/.{1,30}\S*/]+'…'
+    end
+    
+    def title(full=true)
+      if @hash.nil? and !@failed and @html.b
+        if full
+          to_doc unless defined? @doc
+          if @doc.title.b
+            @title = @doc.title
+          else
+            @title = @loc.href
+            @doc.at('head').prepend XML::Node('title', @title) if @doc.at('head')
+            @title
+          end
+        else
+          title true unless defined? @title
+          if RUBY_VERSION < '1.9' and @title.cyr? and UTF2ANSI[@title].size > 40
+            @short_title = ANSI2UTF[UTF2ANSI[@title][/.{1,30}\S*/][0..38]]+'…'
+          elsif @title.size > 40
+            @short_title = @title[/.{1,30}\S*/][0..38]+'…'
+          else
+            @short_title = @title
+          end
         end
+      else
+        @loc.href
       end
-      @doc
     end
       
     def find(xp) (@doc || to_doc).find xp end
@@ -717,7 +781,7 @@ module HTTPAccessKit
     end
       
     def submit(form, frame, hash={}, opts={}, &callback)
-      (opts[:header] ||= {}).Referer ||= @loc.href if @loc
+      (opts[:headers] ||= {}).Referer ||= @loc.href if @loc
       query = form(form, hash, opts)
       
       curr_target, new_target = frame.loc.href, (query[2] || query[0])

data/lib/rhack.yml.template

@@ -1,19 +1,19 @@
-ua file: /path/to/ua/list
-cache:
-  dir: /path/to/cache/dir
-  table: hack_cache
-  clean: 30.days
-logger:
-  :out: /path/to/rmlogger/logfile
-scout retry:
-  example.com:
-  - TimeoutError
-db:
-  reconnect: true
-  encoding: utf8
-  username: root
-  adapter: mysql
-  database: dbname
-  pool: 5
-  password:
-  socket: /var/run/mysqld/mysqld.sock
+#ua file: db/useragents.txt 
+#logger:
+#  :out: log/rhack.log
+#scout retry: {host => [Curl::Error subclass, ], }
+#  example.com:
+#  - TimeoutError
+#db: # defaults file @ to RAILS_PATH/config/RAILS_ENV.yml
+#  reconnect: true
+#  encoding: utf8
+#  username: root
+#  adapter: mysql
+#  database: dbname
+#  pool: 5
+#  password:
+#  socket: /var/run/mysqld/mysqld.sock
+#cache: # deprecated
+#  dir: /path/to/cache/dir
+#  table: hack_cache
+#  clean: 30.days

data/lib/rhack/services.rb

@@ -13,6 +13,17 @@ module HTTPAccessKit
       # first argument should be a string so that frame won't be static
       @f = frame || Frame(self.class::URI[service] || self.class::URI[:login], *args)
     end
+    
+    # Usable only for sync requests
+    def login(*)
+      Curl.run
+      @f[0].cookies.clear
+      json, wait, @f.opts[:json], @f.opts[:wait] = @f.opts[:json], @f.opts[:wait], false, true
+      yield @f.get(self.class::URI[:login])
+      @f.get(self.class::URI[:home]) if self.class::URI[:home]
+      @f.opts[:json], @f.opts[:wait] = json, wait
+      @f.copy_cookies!
+    end
       
     def go(*args, &block)
       __send__(@service, *args, &block) 
@@ -307,13 +318,9 @@ module HTTPAccessKit
     end
     
     def login params={'email'=>'fshm@bk.ru', 'pass'=>'Riddick2', 'expire'=>nil}
-      Curl.run
-      @f[0].cookies.clear
-      @f.get(URI[:login], :json=>nil) {|login_page|
-        login_page.submit('form', @f, params, :json=>nil) {|redirection| 
-          redirection.submit('form', @f, {}, :json=>nil) {|logged|
-            @f.each {|s| s.cookies.replace @f[0].cookies}
-      }}}
+      super {|login_page|
+        login_page.submit('form', @f, params).submit('form', @f, {})
+      }
     end
     
     def get_links h, pagenum, &block

data/lib/scout.rb

@@ -114,7 +114,7 @@ module HTTPAccessKit
         }
         @name, @value = ck[0].split('=', 2)
         #@value.gsub!(/^['"]|['"]$/, '')
-        L.debug args if !@domain
+        #L.debug args if !@domain
         (scout.cookies[scout.uri.host] ||= {})[@name] = self
       else
         @name, cookie = args[0]
@@ -337,7 +337,7 @@ module HTTPAccessKit
       # exc = ['0chan.ru', '2-ch.ru', 'www.nomer.org', 'nomer.org'].select_in('http://www.nomer.org') = ['www.nomer.org', 'nomer.org']
       exc = (@@retry.keys + @retry.keys).select_in @root
       return false if !exc.b
-      # ['www.nomer.org', 'nomer.org'].every |www| 'TimeoutError'.in({'nomer.org' => 'TimeoutError'}[www])} ?
+      # ['www.nomer.org', 'nomer.org'].every {|www| 'TimeoutError'.in({'nomer.org' => 'TimeoutError'}[www])} ?
       exc.no? {|e| err[0].self_name.in((@@retry[e] || []) + @retry[e])}
     end
     
@@ -346,7 +346,6 @@ module HTTPAccessKit
     end
     
     def load!
-      $log <= [$Carier, @http]
       unless $Carier.add @http
         $Carier.remove @http
         $Carier.add @http
@@ -427,9 +426,7 @@ module HTTPAccessKit
       end
       emu = lambda {
         @headers = ''
-        $log << @headers
         @http.on_header {|h|
-          $log << @headers
           @headers << h
           h == "\r\n" ? 0 : h.size
         }
@@ -439,7 +436,6 @@ module HTTPAccessKit
       if emulate != :always
         load(uri, headers) {|c|
           if !@error and c.res.code != 200 and emulate == :if_retry
-            $log << @headers
             emu.call
           else
             callback[c]
@@ -499,7 +495,8 @@ module HTTPAccessKit
       if scout = to_a.rand {|_|!_.loaded?}; scout
       else # Curl should run here, otherwise `next'/`rand'-recursion will cause stack overflow
         raise "Curl must run in order to use ScoutSquad#rand" if !Curl.status
-        Curl.wait
+        #Curl.wait
+        loop {sleep 1; break if $Carier.reqs.size < size}
         self.rand 
       end 
     end
@@ -509,7 +506,8 @@ module HTTPAccessKit
       if scout = find {|_|!_.loaded?}; scout
       else # Curl should run here, otherwise `next'/`rand'-recursion will cause stack overflow
         raise "Curl must run in order to use ScoutSquad#next" if !Curl.status
-        Curl.wait
+        #Curl.wait
+        loop {sleep 1; break if $Carier.reqs.size < size}
         self.next
       end 
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rhack
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.4.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-08-07 00:00:00.000000000 Z
+date: 2013-01-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rmtools
@@ -18,7 +18,7 @@ dependencies:
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 1.2.12
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 1.2.12
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -105,20 +105,28 @@ extra_rdoc_files:
 files:
 - ext/curb/curb_errors.c
 - ext/curb/curb_errors.h
+- ext/curb/curb_errors.o
 - ext/curb/Makefile
 - ext/curb/curb_macros.h
 - ext/curb/curb_multi.c
 - ext/curb/curb_multi.h
+- ext/curb/curb_multi.o
 - ext/curb/curb_upload.c
 - ext/curb/curb_upload.h
+- ext/curb/curb_upload.o
 - ext/curb/curb_config.h
 - ext/curb/extconf.rb
 - ext/curb/curb.c
 - ext/curb/curb.h
+- ext/curb/curb.o
+- ext/curb/curb_core.so
 - ext/curb/curb_easy.c
 - ext/curb/curb_easy.h
+- ext/curb/curb_easy.o
+- ext/curb/mkmf.log
 - ext/curb/curb_postfield.c
 - ext/curb/curb_postfield.h
+- ext/curb/curb_postfield.o
 - ext/curb-original/curb_errors.c
 - ext/curb-original/curb_errors.h
 - ext/curb-original/curb_macros.h
@@ -165,7 +173,7 @@ files:
 - ./Gemfile
 - ./History.txt
 - .gemtest
-homepage: http://2ch.ru
+homepage: https://github.com/tinbka/rhack
 licenses: []
 post_install_message: 
 rdoc_options:
@@ -181,7 +189,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -702291569
+      hash: -84270947
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: