rhack 1.2.10 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 98ec5c6f4bcfb4bbc988ac69fc4148a7042d348c
-  data.tar.gz: d12c3aaad05efbc0df476d790213d0bcd9065beb
+  metadata.gz: ccd52c86276421476a67a4b379fcb6275bd6f85e
+  data.tar.gz: 46efe342223575cb26eceddca4970173b7f32cdf
 SHA512:
-  metadata.gz: 8eef5e956ef5b444c33e6ddb28c56e9e4c8e9c2cdaec9ced9124569e461a6aed26bff7a913e8b02009105428171aa30fd5bd9a88e5829e4c7ee9a91abd78bb28
-  data.tar.gz: c40ca66013388dbd0459d4b1d97cb8c9ebce03ff942d225b77affedbf4dfb76c897691305737c5e8b9e7bec719c603738a7c518b464e8b253d51bd259c6873f1
+  metadata.gz: af9bbd50acdc9ee228bc3ccede8fc31da56dd762d39a96f85c847e3c223167f2aff0fff6f4ef30ee68e7d7b3fa91e51d74cf670c7b7968bd9d4ec70861ca3ae9
+  data.tar.gz: 7d0f7418a4af8d3ff2bcbfab5ec26fbca85ad0664570242255fee2822ffbf3c528ab8e4ba9979d2a545b6599a77bd80f15ab6aa2f24297bb947fdc134a7457b0

data/lib/rhack.rb

@@ -10,7 +10,6 @@ require "rhack/js/johnson"
 module RHACK
   mattr_reader :config, :redis, :useragents
   
-  Dir.chdir ENV['APP_ROOT'] if ENV['APP_ROOT']
   cfgfile = Dir['{config/,}rhack.yml'].first
   @@config = cfgfile ? YAML.load(IO.read(cfgfile)) : {}
   

data/lib/rhack/curl/response.rb

@@ -8,7 +8,7 @@ module Curl
     def to_s
       str = '<#'
       if @error
-        str << "#{@error[0].self_name}: #{@error[1]}"
+        str << "#{@error.name}: #{@error.message}"
       else
         str << (@header[/\d{3}/] == @code.to_s ? @header : "#{@header[/\S+/]} #{@code}") if @header
         if @hash.location
@@ -77,14 +77,16 @@ module Curl
     
     def is(klass)
       if @error
-        klass == Array || klass = Curl::Response
+        $log.warn "obsolete comparison with Array", caller: 1 if klass == Array
+        klass == Array || klass = Curl::Response # obsolete
       else
         klass == Curl::Response
       end
     end
   
-    def [](key_or_index)
-      @error ? @error[key_or_index] : @hash[key_or_index.downcase]
+    def [](key)
+      #@error ? @error[key_or_index] : @hash[key_or_index.downcase] # old
+      @hash[key.downcase]
     end
     
     alias :headers :hash

data/lib/rhack/frame.rb

@@ -22,6 +22,7 @@ module RHACK
   class Frame
     __init__
     attr_reader :loc, :static, :ss, :opts, :use_cache, :write_to
+    alias options opts
     @@cache = {}
     
     def initialize *args
@@ -56,7 +57,7 @@ module RHACK
       @ss.update to, forced
       update_loc to
     end
-    alias :target= :retarget
+    alias target= retarget
     
     def anchor
       retarget @loc.href
@@ -222,7 +223,7 @@ module RHACK
       (opts[:headers] ||= {})['X-Requested-With'] = 'XMLHttpRequest' if opts[:xhr]
       if opts[:content_type]
         if opts[:content_type].is Symbol
-          if mime_type = Mime::Types.of(opts[:content_type])[0]
+          if mime_type = MIME::Types.of(opts[:content_type])[0]
             (opts[:headers] ||= {})['Content-Type'] = mime_type.content_type
           else
             raise ArgumentError, "failed to detect Mime::Type by extension: #{opts[:content_type]}
@@ -329,6 +330,15 @@ module RHACK
       page = opts[:result].new
       # if no spare scouts can be found, squad simply waits for first callbacks to complete
       s = @ss.next
+      s.http.on_failure {|curl, error|
+        s.process_failure(*error)
+        if opts[:raw]
+          page.res = s.error
+        elsif page.process(curl, opts)
+          run_callbacks! page, opts, &callback
+          # nothing to do here if process returns nil or false
+        end
+      }
       s.send(*(order << opts)) {|curl|
       #   there is a problem with storing html on disk
         if order[0] == :loadGet and @write_to
@@ -343,7 +353,7 @@ module RHACK
         elsif page.process(curl, opts)
           @@cache[page.href] = page if order[0] == :loadGet and @use_cache
           run_callbacks! page, opts, &callback
-        # nothing to do here if process returns nil or false
+          # nothing to do here if process returns nil or false
         end
       }
       # > Carier.requests++

data/lib/rhack/page.rb

@@ -72,8 +72,8 @@ module RHACK
         
     def inspect
       sz = size
-      if !@data.nil?
-        "<##{self.class.name} (#{@data == false ? 'failed to parse' : sz.bytes}) #{@json ? 'json' : 'url params'}>"
+      if @json or @hash
+        "<##{self.class.name} (#{@data ? sz.bytes : 'failed to parse'}) #{@json ? 'json' : 'url params'}>"
       else
         "<##{self.class.name} #{sz == 0 ? '(empty)' : "#{@failed ? @curl_res.header : '«'+title(false)+'»'} (#{sz.bytes})"}#{' js enabled' if @js and @doc}>"
       end
@@ -91,7 +91,7 @@ module RHACK
     
     # override this in a subclass
     def failed?(*)
-      @curl_res.code != 200
+      @curl_res.error or @curl_res.code != 200
     end
     
     # override this in a subclass
@@ -103,14 +103,6 @@ module RHACK
     # MUST return self if successful
     # MAY return false otherwise
     def parse(opts={})
-      if failed?
-        failed!
-        if opts[:json] or opts[:hash]
-          @data = false
-        end
-        return self
-      end
-      
       if opts[:json]
         parse_json opts
       elsif opts[:hash]
@@ -120,15 +112,23 @@ module RHACK
       else
         parse_html opts
       end
-      
       self
     end
     
   private
     
+    # @failed means failure cause
+    # MUST return false if you don't want this client
+    # to call main callback in case of failure
+    # MUST return true otherwise
     def failed!
-      @body = @curl_res.body
-      @failed = @curl_res.code
+      if @curl_res.error
+        @failed = @curl_res.error
+      else
+        @body = @curl_res.body
+        @failed = @curl_res.code
+      end
+      false
     end
     
     def log_failed(action)
@@ -171,7 +171,9 @@ module RHACK
       end
     end
     
+    # urlencoded
     def parse_hash(*)
+      @hash = true
       if @curl_res.body.inline
         @data = @curl_res.body.to_params
       else
@@ -184,15 +186,22 @@ module RHACK
   public
     
     # We can then alternate #process in Page subclasses
-    # Frame doesn't mind about value returned by #process
-    def process(c, opts={})
-      @loc = c.last_effective_url.parse:uri
-      @curl = c
-      @curl_res = c.res
+    # Frame doesn't mind about the value returned by #process
+    # unless it is nil or false
+    def process(curl, opts={})
+      @loc = curl.last_effective_url.parse :uri
+      @curl = curl
+      @curl_res = curl.res
       
-      if retry?
-        c.retry!
-        return # callback will not proceed
+      if failed?
+        should_proceed = failed! # false by default
+        if retry?
+          curl.retry!
+          return false # callback will not proceed
+        end
+        unless should_proceed
+          return false # nor callback or retry will not proceed
+        end
       end
       
       L.debug "#{@loc.fullpath} -> #{@curl_res}"
@@ -519,11 +528,7 @@ module RHACK
     # MUST return self if successful
     # MAY return false otherwise
     def parse(opts={})
-      if failed?
-        failed!
-      else
-        parse_xml opts
-      end
+      parse_xml opts
       self
     end
     
@@ -537,11 +542,7 @@ module RHACK
     # MUST return self if successful
     # MAY return false otherwise
     def parse(opts={})
-      if failed?
-        failed!
-      else
-        parse_html opts
-      end
+      parse_html opts
       self
     end
     
@@ -555,11 +556,7 @@ module RHACK
     # MUST return self if successful
     # MAY return false otherwise
     def parse(opts={})
-      if failed?
-        failed!
-      else
-        parse_json opts
-      end
+      parse_json opts
       self
     end
     
@@ -573,11 +570,7 @@ module RHACK
     # MUST return self if successful
     # MAY return false otherwise
     def parse(opts={})
-      if failed?
-        failed!
-      else
-        parse_hash opts
-      end
+      parse_hash opts
       self
     end
     

data/lib/rhack/scout.rb

@@ -142,7 +142,7 @@ module RHACK
             name = v.is(Hash) && v[:name] ||
               File.basename(path)
             content_type = v.is(Hash) && v[:content_type].to_s ||
-              (Mime::Types.of(path)[0] || {}).content_type ||
+              (MIME::Types.of(path)[0] || {}).content_type ||
               "application/octet-stream"
             Curl::PostField.file(k, type, name, read(path))
           else
@@ -235,6 +235,23 @@ module RHACK
       !loaded?
     end
     
+    def process_failure(curl_err, message)
+      @error = curl_err.new message
+      #@error = [curl_err, message] # old
+      @http.outdate!
+      # we must clean @http.on_complete, otherwise
+      # it would run right after this function and with broken data
+      @http.on_complete &Proc::NULL
+      if retry? curl_err
+        L.debug "#{curl_err} -> reloading scout"
+        retry!
+      else
+        L.debug "#{curl_err} -> not reloading scout"
+        raise @error if @raise_err
+        #raise *@error if @raise_err # old
+      end
+    end
+    
     def load!
       unless Curl.carier.add @http
         Curl.carier.remove @http
@@ -253,7 +270,7 @@ module RHACK
     end
     
     def load(path=@path, headers={}, not_redir=1, relvl=10, &callback)
-      # cache preprocessed data for one time for we can do #retry
+      # cache preprocessed data for one time so we can do #retry
       @__path = path
       @__headers = headers
       @__not_redir = not_redir
@@ -264,36 +281,25 @@ module RHACK
       @http.headers = mkHeader(path).merge!(headers)
       @http.timeout = @timeout
 
-      @http.on_complete {|c|
+      @http.on_complete {|curl| # = @http
         # > Carier.requests--
         @error = nil
         # While not outdated, Curl::Response here may contain pointers on freed
         # memory, thus throwing exception on #to_s and #inspect
-        c.outdate!
-        ProcCookies c.res if @cookieProc
+        @http.outdate!
+        res = @http.res
+        ProcCookies res if @cookieProc
         # We cannot just cancel on_complete in on_redirect block
         # because loadGet will immediately reset on_complete back
-        if c.res.code.in(300..399) and !not_redir.b and (relvl -= 1) > -1 and loc = c.res.hash.location
+        if res.code.in(300..399) and !not_redir.b and (relvl -= 1) > -1 and loc = res.hash.location
           loadGet(loc, headers: headers, relvl: relvl, redir: true, &callback)
         elsif block_given?
-          yield c
+          yield @http
         end
       }
-      @http.on_failure {|c, e|
-        eclass = e[0]
-        @error = e
-        c.outdate!
-        # we must clean @http.on_complete, otherwise
-        # it would run right after this function and with broken data
-        @http.on_complete &Proc::NULL
-        if retry? eclass
-          L.debug "#{eclass} -> reloading scout"
-          retry!
-        else
-          L.debug "#{eclass} -> not reloading scout"
-          raise *e if @raise_err
-        end
-      } if !@http.on_failure
+      @http.on_failure {|curl, error|
+        process_failure(*error)
+      } unless @http.on_failure
       
       load!
     end

data/lib/rhack/version.rb

@@ -1,3 +1,3 @@
 module RHACK
-  VERSION = '1.2.10'
+  VERSION = '1.3.0'
 end

data/rhack.gemspec

@@ -16,13 +16,9 @@ Gem::Specification.new do |spec|
   spec.files         = `git ls-files`.split($/)
   spec.require_paths = ["lib"]
   
-  spec.add_runtime_dependency "activesupport", "~> 3"
   #spec.add_runtime_dependency "redis"
-  spec.add_runtime_dependency "rmtools", "~> 2.3"
+  spec.add_runtime_dependency "rmtools", "~> 2.4"
   spec.add_runtime_dependency "libxml-ruby", "~> 2.7"
   
   spec.extensions << 'ext/curb/extconf.rb'
-
-  spec.add_development_dependency "bundler", "~> 0"
-  spec.add_development_dependency "rake", "~> 0"
 end

metadata

@@ -1,43 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: rhack
 version: !ruby/object:Gem::Version
-  version: 1.2.10
+  version: 1.3.0
 platform: ruby
 authors:
 - Sergey Baev
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-06 00:00:00.000000000 Z
+date: 2014-10-15 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: activesupport
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '3'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '3'
 - !ruby/object:Gem::Dependency
   name: rmtools
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '2.3'
+        version: '2.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '2.3'
+        version: '2.4'
 - !ruby/object:Gem::Dependency
   name: libxml-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -52,34 +38,6 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '2.7'
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '0'
 description: 'RHACK is Ruby Http ACcess Kit: curl-based web-client framework created
   for developing web-scrapers/bots.\n\nFeatures:\nAsynchronous, still EventMachine
   independent\nFast as on simple queries as on high load\n3 levels of flexible configuration\nWeb-client