rhack 1.3.1 → 1.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8f5a539c0ca1c92416d50ffd4609d08a69d900fe
-  data.tar.gz: fee2e42ae02b21af929a1b0b417360adc043562d
+  metadata.gz: 40e4929e52203b0121c9ca2eac2f289f656ff543
+  data.tar.gz: abb0fe53f2f8e43836473fdc8d7ae517fcd63602
 SHA512:
-  metadata.gz: bfda297b6b465393b6749bac236e32bd967216dbad68c4383fc3bdd1e9cbc5602efcb57513ce9c01b9b6c6416ad312b10e150a802b717932e471594744e8765b
-  data.tar.gz: e4e762cd0e9e858cd20e9bef50c0a9fb4d97b506361879b5efc49cad3d939ebb35b2e3245144fd2e9f01a7ddc033200b656d71a5e4ffa53172e84d468376d3c4
+  metadata.gz: 23e4c2b43ee95212d0c8069ed7e2c731dc1878ec4bd4f7b8984bb0dfea55f1cd496ad161c098d2a0551a49d995a6cf4faca8b0041fc891d9f3fc3b42bfce5243
+  data.tar.gz: b6e3fce288abf2aaa51cf7910b1b99c90e0c673f13bf6541e8a047b496d83fe6d90b5d11379f8d8e6610a9ad94c6d8834d58c9b0c8cfa2fd171d43852791d987

data/lib/rhack/clients/base.rb

@@ -11,6 +11,7 @@ module RHACK
     class_attribute :frame_defaults, :instance_writer => false
     class_attribute :accounts, :instance_writer => false
     class_attribute :routes, :instance_writer => false
+    class_attribute :rootpath, :instance_writer => false
     
     self.frame_defaults = {}
     self.accounts = {}
@@ -25,33 +26,50 @@ module RHACK
         }
       end
       
+      def method_missing(method, *args, &block)
+        if personal_instance_methods.include? method
+          return new.__send__(method, *args, &block)
+        end
+        super
+      end
+      
     private
       
+      def root(value=nil)
+        if value
+          value = 'http://' + value if value !~ /^\w+:/
+          self.rootpath = value
+        else
+          self.rootpath
+        end
+      end
+      alias :host :root
+      
       # Set routes map
       def map(dict)
         # URI is deprecated # backward compatibility
         if defined? URI and URI.is Hash
           URI.merge! dict.map_hash {|k, v| [k.to_sym, v.freeze]}
         end
-        routes.merge! dict.map_hash {|k, v| [k.to_sym, v.freeze]}
+        self.routes += dict.map_hash {|k, v| [k.to_sym, v.freeze]}
       end
       
       # Set default Frame options
       def frame(dict)
-        frame_defaults.merge! dict
+        self.frame_defaults += dict
       end
       
       # Set usable accounts
       # @ dict : {symbol => {symbol => string, ...}}
       def accounts(dict)
-        accounts.merge! dict
+        self.accounts += dict
       end
       
     end
     
     def initialize(*args)
-      service, opts = args.get_opts [:api]
-      @service = service
+      service, opts = args.get_opts [routes.include?(:api) ? :api : nil]
+      @service = service # Deprectated. Use different classes to implement different services.
       # first argument should be a string so that frame won't be static
       if opts.is_a?(Frame)
         @f = opts
@@ -60,9 +78,13 @@ module RHACK
         if self.class.const_defined? :Result
           opts[:result] = self.class::Result
         end
-        @f = Frame(route(service) || route(:login), opts)
+        @f = Frame(rootpath || route(service) || route(:login), opts)
       end
     end
+        
+    def inspect
+      "<##{self.class.name}#{":#{@service.to_s.camelize} service" if @service} via #{@f.inspect}>"
+    end
     
     
     # Usable only for sync requests
@@ -91,15 +113,17 @@ module RHACK
         @f.get(url) {|next_page| scrape!(next_page)}
       end
     end
-        
-    def inspect
-      "<##{self.class.self_name}:#{@service.to_s.camelize} service via #{@f.inspect}>"
-    end
+    
     
     # shortcuts to class variables #
     
     def route(name)
-      routes[name]
+      if url = routes[name]
+        if url !~ /^\w+:/
+          url = File.join rootpath, url
+        end
+        url
+      end
     end
     alias :url :route
     # URI is deprecated # backward compatibility

data/lib/rhack/frame.rb

@@ -90,17 +90,36 @@ module RHACK
     
     def inspect
       sssize = @ss.size
-      "<#Frame @ #{@ss.untargeted ? 'no target' : @loc.root}: #{sssize} #{sssize == 1 ? 'scout' : 'scouts'}#{', static'+(' => '+@static.protocol if @static.is(Hash)) if @static}, cookies #{@ss[0].cookieProc ? 'on' : 'off'}>"
+      "<#Frame @ #{@ss.untargeted ? 'no target' : @loc.root}: #{sssize} #{sssize == 1 ? 'scout' : 'scouts'}#{', static'+(' => '+@static.protocol if @static.is(Hash)) if @static}, cookies #{@ss[0].cookie_enabled ? 'on' : 'off'}>"
     end
     
     # All opts going in one hash.
     # Opts for Frame:
-    #   :wait, :proc_result, :save_result, :zip, :thread_safe, :result, :stream, :raw, :xhr, :content_type
-    # Opts passed to Page:
+    #   :wait, :sync, :thread_safe, :raw, :proc_result, :save_result, :zip, :result, :stream
+    #   ... processed and passed to Scout:
+    #     :xhr, :content_type, :auth
+    # Opts passed to result:
     #   :xml, :html, :json, :hash, :eval, :load_scripts
-    # Opts for Scout:
+    # Opts passed to Scout:
     #   :headers, :redir, :relvl
-    # TODO: describe options
+    #
+    # @ :result : враппер результата исполнения; по умолчанию Page, для Client — если определён — Result; при асинхронном вызове будет возвращён незамедлительно
+    # @ &callback : в него будет передан инстанс result, а его результат будет записан в result#res (по умолчанию это ссылка на себя)
+    # @ :thread_safe : не использовать луп исполнения Curl::Multi#perform, а вызывать #perform прямо в этом треде; если установлен, то невозможно прерывание исполнения клавиатурой (продолжит работать, выполняя колбеки, в фоне), и невозможно задавать больше параллельных реквестов, чем разрешено параллельных соединений (просто застрянет)
+    # @ :sync : остановить (Thread#kill) perform-loop после исполнения всех запросов; подразумевает wait=true; при вызове одиночного реквеста подразумевает thread_safe=true
+    # @ :wait : ждать исполнения всех реквестов
+    # @ :save_result: возвращает #res для каждого инстанса result вместо самого инстанса; если не задан :proc_result, то подразумевает wait=true
+    # @ :proc_result: Proc, в который будет передан result#res, если задан также &callback; служит для создания вложенных блоков для клиентов; если =nil, то подразумевает wait=true
+    # @ :raw : сохраняем *только* тело ответа, без хедеров, без отладочной инфы в #res
+    # @ :raw + :sync : подразумевает save_result=true
+    # @ :xhr, :content_type, :auth : формируют хедеры X-Requested-With, Content-Type, Authorization для передачи в Scout
+    # @ :xhr : boolean
+    # @ :content_type : symbol<extension>  |  raw string
+    # @ :auth : "<username>:<password>"
+    # 
+    # @ :zip, :stream и все опции для result : deprecated
+    #
+    # TODO: Семантически разделить синхронное и асинхронное выполнение запросов (не важно, серии или отдельных), с учётом, что асинхронность по сути своей перегружена и требуется, например, в очередях сообщений, но не в синхронных контроллерах Rails
     def exec *args, &callback
       many, order, orders, with_opts = interpret_request *args
       L.log({:many => many, :order => order, :orders => orders, :with_opts => with_opts})
@@ -113,8 +132,15 @@ module RHACK
       # if we aren't said explicitly about the opposite
       Johnson::Runtime.set_browser_for_curl with_opts
       
-      if many then	exec_many orders, with_opts, &callback
-      else 	          exec_one order, with_opts, &callback    end
+      if many
+        result = exec_many orders, with_opts, &callback
+      else
+        result = exec_one order, with_opts, &callback
+      end
+      if with_opts[:sync]
+        Curl.stop
+      end
+      result
     end
     alias :get :exec
     alias :run :get
@@ -218,7 +244,13 @@ module RHACK
                     
       opts[:eval] = false if opts[:json] or opts[:hash] or opts[:raw]
       opts[:load_scripts] = self if opts[:load_scripts]
-      opts[:stream] = true if opts[:raw]
+      opts[:save_result] = true if opts[:wait] and opts[:raw]
+      
+      if orders
+        opts[:thread_safe] = false if @ss.size < orders.size
+      else
+        opts[:thread_safe] = true if opts[:sync]
+      end
       
       (opts[:headers] ||= {})['X-Requested-With'] = 'XMLHttpRequest' if opts[:xhr]
       if opts[:content_type]
@@ -233,6 +265,9 @@ module RHACK
           (opts[:headers] ||= {})['Content-Type'] = opts[:content_type]
         end
       end
+      if opts[:auth]
+        (opts[:headers] ||= {})['Authorization'] = "Basic #{Base64.encode64(opts[:auth])}".chop
+      end
       
       [many, order, orders, opts]
     end
@@ -331,7 +366,7 @@ module RHACK
       # if no spare scouts can be found, squad simply waits for first callbacks to complete
       s = @ss.next
       s.http.on_failure {|curl, error|
-        if s.process_failure(*error)
+        s.process_failure(*error) {
           # curl itself has decided not to retry a request
           if opts[:raw]
             page.res = s.error
@@ -339,14 +374,14 @@ module RHACK
             run_callbacks! page, opts, &callback
             # nothing to do here if process returns nil or false
           end
-        end
+        }
       }
       s.send(*(order << opts)) {|curl|
       #   there is a problem with storing html on disk
         if order[0] == :loadGet and @write_to
       #     sometimes  (about 2% for 100-threads-dling) when this string is calling
       #     no matter what +curl.res.body+ has contained here
-          RMTools.rw @write_to+'/'+order[-2].sub(/^[a-z]+:\/\//, ''), curl.res.body.xml_to_utf
+          RMTools.rw @write_to+'/'+order[-2].sub(/^\w+:\/\//, ''), curl.res.body.xml_to_utf
         end
         if opts[:raw]
           page.res = block_given? ? yield(curl) : curl.body_str

data/lib/rhack/scout.rb

@@ -7,7 +7,7 @@ module RHACK
     attr_accessor	:path, :root, :sld, :proxy
     attr_reader	    :uri
     attr_reader	    :webproxy, :last_method, :proxystr, :headers, :body, :http, :error
-    attr_reader	    :cookies, :ua, :refforge, :cookieStore, :cookieProc
+    attr_reader	    :cookies, :ua, :refforge, :cookies_enabled
     
     DefaultHeader = {
         "Expect"	              => "",
@@ -40,7 +40,7 @@ module RHACK
       @cookies    	= {}
       @body       	= {}
       @num    	    = []
-      @cookieProc	= opts[:cp] || opts[:ck]
+      @cookies_enabled	= opts[:cp] || opts[:ck]
       @raise_err   	= opts[:raise] # no way to use @raise id, it makes any 'raise' call here fail
       @engine     	= opts[:engine]
       @timeout    	= opts[:timeout] || @@timeout || 60
@@ -81,9 +81,9 @@ module RHACK
         @proxystr = @webproxy ? @proxy[0] : @http.proxy_url
       else @proxystr = 'localhost' 
       end
-      if @cookieProc.is Hash
-        self.main_cks = @cookieProc
-        @cookieProc = true    
+      if @cookies_enabled.is Hash
+        self.main_cks = @cookies_enabled
+        @cookies_enabled = true    
       end
       self
     end
@@ -169,7 +169,7 @@ module RHACK
     
     def mkHeader(uri)
       header = DefaultHeader.dup
-      if @cookieProc
+      if @cookies_enabled
         cookies = ''
         main_cks.each {|k, v| main_cks.delete k if v.use(cookies, @uri) == :expired}
         header['Cookie'] = cookies[0..-3]                                 
@@ -182,7 +182,7 @@ module RHACK
       header
     end
       
-    def ProcCookies(res)
+    def process_cookies(res)
       ck = []
       case res
         when String
@@ -195,11 +195,10 @@ module RHACK
       end
       return if !ck.b
       ck.each {|c| Cookie(c, self)}
-  #    StoreCookies if @cookieStore
     end
 
-    def cp_on() @cookieProc = true end
-    def cp_off() @cookieProc = false end
+    def cp_on() @cookies_enabled = true end
+    def cp_off() @cookies_enabled = false end
     
     def main_cks() @cookies[@uri.host] ||= {} end
     def main_cks=(cks)
@@ -231,13 +230,18 @@ module RHACK
       Curl.carier.reqs.include? @http
     end
     
+    # Scout must not be reused until not only response will have come,
+    # but callback will have been processed, too.
+    # Otherwise, #retry! may not work as expected:
+    # if a scout gets callback as a block argument, then it may re-run not original callback,
+    # but it's copy with another scope.
     def available?
-      !loaded?
+      !loaded? and !@busy
     end
     
     # - if curl should retry request based on Curl::Err class only
     #   => false
-    def process_failure(curl_err, message)
+    def process_failure(curl_err, message, &callback)
       @error = curl_err.new message
       #@error = [curl_err, message] # old
       @http.outdate!
@@ -247,33 +251,35 @@ module RHACK
       if retry? curl_err
         L.debug "#{curl_err} -> reloading scout"
         retry!
-        false
       else
         L.debug "#{curl_err} -> not reloading scout"
         raise @error if @raise_err
         #raise *@error if @raise_err # old
-        true
+        yield if block_given?
+        # Now, we assume that data of this @http have been copied or will not be used anymore,
+        # thus the scout can be reused.
+        @busy = false
       end
     end
     
     def load!
       unless Curl.carier.add @http
+        L.warn "#{self}##{object_id}: Failed to add Curl::Easy##{@http.object_id} to Curl::Multi##{Curl.carier.object_id}. Trying to remove it and re-add."
         Curl.carier.remove @http
         Curl.сarier.add @http
       end
     rescue RuntimeError => e
       e.message << ". Failed to load allready loaded? easy handler: Bad file descriptor" unless Curl::Err::CurlError === e
-      L.warn "#{e.inspect}: #{e.message}"
+      L.warn "#{self}##{object_id}: #{e.inspect}: #{e.message}"
       if loaded?
         Curl.carier.remove @http
       end
       sleep 1
       load!
-      #e.message << ". Failed to load allready loaded? easy handler: Bad file descriptor" unless Curl::Err::CurlError === e
-      #raise e
     end
     
     def load(path=@path, headers={}, not_redir=1, relvl=10, &callback)
+      @busy = true
       # cache preprocessed data for one time so we can do #retry
       @__path = path
       @__headers = headers
@@ -286,21 +292,28 @@ module RHACK
       @http.timeout = @timeout
 
       @http.on_complete {|curl| # = @http
-        # > Carier.requests--
+        # @http has already been removed when a request had complete,
+        # but this callback may occure wherever in a serial queue of curl callbacks.
         @error = nil
         # While not outdated, Curl::Response here may contain pointers on freed
         # memory, thus throwing exception on #to_s and #inspect
         @http.outdate!
         res = @http.res
-        ProcCookies res if @cookieProc
-        # We cannot just cancel on_complete in on_redirect block
-        # because loadGet will immediately reset on_complete back
+        process_cookies res if @cookies_enabled
+        # We cannot just cancel on_complete in on_redirect block,
+        # because loadGet should (and will) immediately reset on_complete back.
         if res.code.in(300..399) and !not_redir.b and (relvl -= 1) > -1 and loc = res.hash.location
           loadGet(loc, headers: headers, relvl: relvl, redir: true, &callback)
-        elsif block_given?
-          yield @http
+        else
+          yield @http if block_given?
+          # Now, we assume that data of this @http have been copied or will not be used anymore,
+          # thus the scout can be reused.
+          @busy = false
+          @http.on_failure &Proc::NULL
         end
       }
+      # Curl::Err::* (TCP/IP level) exception callback.
+      # May be set out there.
       @http.on_failure {|curl, error|
         process_failure(*error)
       } unless @http.on_failure

data/lib/rhack/version.rb

@@ -1,3 +1,3 @@
 module RHACK
-  VERSION = '1.3.1'
+  VERSION = '1.3.3'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rhack
 version: !ruby/object:Gem::Version
-  version: 1.3.1
+  version: 1.3.3
 platform: ruby
 authors:
 - Sergey Baev
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-27 00:00:00.000000000 Z
+date: 2014-11-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rmtools