rhack 1.1.0 → 1.1.4

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    MmU2Y2RiMzNmOTM0ZjUwYzkzM2VlNGEyNGQzZjcwMTI2NjdmNTk0YQ==
+  data.tar.gz: !binary |-
+    ZWYxYTBkMDI5Njg5NDhiMzgwYTU0MGRkZGY5MWJjNjA3ZjE5OGUxZQ==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    NTgxOTYyYTQwYjYwYTkzNTgyZGFmZWMzOGVmZTE1MjA2M2E5YzAzM2Q1ZTYy
+    ZjYyYTM3NjkzYTE5OGZjODQ3ZmU0Njc5NzY4MTEzYjE5MDI3MzdiYzc4N2Uy
+    ODFhNmIyMjQyOTE3NTFmZmIwN2VjMTE0ZjllMzY3ZDhmZDg1Mzg=
+  data.tar.gz: !binary |-
+    YmUyM2IyNjBiMWFkNTlmZWUwZWIyZjUxZmIxYmNkYjc3OTliN2Q1ZmMxYzM2
+    ZmIzM2Q4MWFhNGZjYjE1NmNkMjcwZGQ5YzAyNjNjNGQ4NGM5N2VmMTQ2MGY5
+    ODZkMGJjMGJlOGEyMjIxYjc0NjMyZGY0ZjY1MzRkMjQwNmZlNDQ=

data/README.md

@@ -31,6 +31,25 @@ It's still randomly documented since it's just my working tool.
 
 ### CHANGES
 
+##### Version 1.1.4
+
+* ::Frame
+  * Moved `Curl.execute` from *initialize* to *on after request added*
+  
+* ::ScoutSquad
+  * Finally stabilized #next and #rand time management for parallel recursive execution
+
+##### Version 1.1.3
+
+* ::Frame
+  * Added #anchor
+  
+* ::Scout
+  * Fixed #update
+  * Catch weird Curl::Err::CurlOK being thrown on some pages
+
+* Fixed some exceptions messages
+
 ##### Version 1.1.0
 
 * ::OAuthClient < ::Client
@@ -58,6 +77,7 @@ It's still randomly documented since it's just my working tool.
   
 * ::Service
   * Is renamed to Client what is more sensible. RHACK::Service is still usable as alias
+  * require 'rhack/clients' <-> require 'rhack/services'
 
 * Structural changes
   * Updated and documented rhack.yml.template that now lies in <gemdir>/config

data/ext/curb/curb_multi.c

@@ -71,14 +71,13 @@ rb_hash_clear_i(VALUE key, VALUE value, VALUE dummy) {
 }
 
 static void curl_multi_free(ruby_curl_multi *rbcm) {
-
-  if (rbcm && !rbcm->requests == Qnil && rb_type(rbcm->requests) == T_HASH && RHASH_LEN(rbcm->requests) > 0) {
-
+  //if (rbcm && !rbcm->requests == Qnil && rb_type(rbcm->requests) == T_HASH && RHASH_LEN(rbcm->requests) > 0) {
+  if (rbcm && rb_type(rbcm->requests) == T_HASH && RHASH_LEN(rbcm->requests) > 0) {
     rb_hash_foreach( rbcm->requests, (int (*)())curl_multi_flush_easy, (VALUE)rbcm );
-
     rb_hash_foreach(rbcm->requests, rb_hash_clear_i, 0); //rb_hash_clear(rbcm->requests);
     rbcm->requests = Qnil;
   }
+  
   curl_multi_cleanup(rbcm->handle);
   free(rbcm);
 }
@@ -179,10 +178,10 @@ static VALUE ruby_curl_multi_idle(VALUE self) {
   
   Data_Get_Struct(self, ruby_curl_multi, rbcm);
   
-  if ( FIX2INT( rb_funcall(rbcm->requests, rb_intern("length"), 0) ) == 0 ) {
-    return Qtrue;
-  } else {
+  if (RHASH_LEN(rbcm->requests))
     return Qfalse;
+  } else {
+    return Qtrue;
   }
 }
 
@@ -627,6 +626,7 @@ static void rb_curl_multi_idle_perform(VALUE self, ruby_curl_multi *rbcm) {
   create_crt_fd(&fdexcep, &crt_fdexcep);
 #endif
   
+  // sleep while no requests
   do {
 #ifdef HAVE_RB_THREAD_BLOCKING_REGION
     fdset_args.maxfd = 0;
@@ -641,7 +641,7 @@ static void rb_curl_multi_idle_perform(VALUE self, ruby_curl_multi *rbcm) {
     if (rc == -1)
       rb_raise(rb_eRuntimeError, "select(): %s", strerror(errno));
     
-  } while (!(RHASH_TBL(rbcm->requests)->num_entries));
+  } while (!RHASH_LEN(rbcm->requests));
   
 #ifdef _WIN32
   cleanup_crt_fd(&fdread, &crt_fdread);

data/lib/rhack/clients.rb

@@ -0,0 +1,10 @@
+require 'rhack'
+require 'rhack/clients/base'
+require 'rhack/clients/storage'
+require 'rhack/clients/oauth'
+
+module RHACK
+  for name in [:Service, :ServiceError]
+    autoload name, 'rhack/clients/compatibility'
+  end
+end

data/lib/rhack/{services → clients}/oauth.rb

@@ -95,7 +95,7 @@ module RHACK
         response_type: 'code', 
         client_id: OAUTH(:id),
         state: state
-      }.merge(url_params).to_params
+      }.merge(url_params).urlencode
     end
     
     # @ url_params : {:code, :state, ...}
@@ -121,7 +121,7 @@ module RHACK
         grant_type: 'authorization_code',
         client_id: OAUTH(:id), 
         client_secret: OAUTH(:secret)
-      }.merge(url_params).to_params, raw: true, proc_result: block) {|curl|
+      }.merge(url_params).urlencode, raw: true, proc_result: block) {|curl|
         L.debug curl.res
         L.debug curl.res.body
         # TODO: refactor parse type selector: raw, json, hash, xml...
@@ -145,7 +145,7 @@ module RHACK
         grant_type: 'client_credentials',
         client_id: OAUTH(:id), 
         client_secret: OAUTH(:secret)
-      }.to_params, raw: true, proc_result: block) {|curl|
+      }.urlencode, raw: true, proc_result: block) {|curl|
         if curl.res.code == 200
           body = curl.res.body
           hash = '{['[body[0]] ? body.from_json(symbolize_keys: true) : body.to_params
@@ -194,7 +194,7 @@ module RHACK
       
       L.debug state_params
       action += '?' if !action['?']
-      action += action_params.to_params
+      action += action_params.urlencode
       L.debug [action_data, action, token]
       opts = {proc_result: block, headers: {'Referer' => nil}, result: CodeIndiffirentPage}.merge(opts)
       # TODO: option to 

data/lib/rhack/frame.rb

@@ -27,10 +27,11 @@ module RHACK
     def initialize *args
       args << 10 unless args[-1].is Fixnum
       args.insert -2, {} unless args[-2].is Hash
-      if scouts = args[-2][:scouts]
+      opts = args[-2]
+      if scouts = (opts[:scouts] || opts[:threads])
         args[-1] = scouts
       end
-      @opts = {:eval => Johnson::Enabled, :redir => true, :cp => true, :result => Page}.merge!(args[-2])
+      @opts = {:eval => Johnson::Enabled, :redir => true, :cp => true, :result => Page}.merge!(opts)
       args[-2] = @opts
       if args[0].is String
         url = args[0]
@@ -41,7 +42,6 @@ module RHACK
         @static = false
       end
       @ss  = ScoutSquad *args
-      Curl.run :unless_allready
     end
     
     def update_loc url
@@ -57,6 +57,10 @@ module RHACK
     end
     alias :target= :retarget
     
+    def anchor
+      retarget @loc.href
+    end
+    
     def next() @ss.next end
     def rand() @ss.rand end
     def each(&block) @ss.each &block end
@@ -205,17 +209,17 @@ module RHACK
           if @static
             if @static.is Hash
               if loc.host != @loc.host and !@static.host
-                raise TargetError, "unable to get #{url} by static frame [#{@static.protocol}://]#{@loc.host}, you should first update it with new target"
+                raise TargetError, "unable to get #{url} by a static frame [#{@static.protocol}://]#{@loc.host}, you should first update it with a new target"
               end
             else
-              raise TargetError, "unable to get #{url} by static frame #{@loc.root}, you should first update it with new target"
+              raise TargetError, "unable to get #{url} by a static frame #{@loc.root}, you should first update it with a new target"
             end
           end
           @loc.root, @loc.host, @loc.protocol = loc.root, loc.host, loc.protocol
           url
         elsif !loc.root
           if !@static
-            raise TargetError, "undefined root for query #{url}, use :static option as Hash to set default protocol and host, or as True to allow using previously used root"
+            raise TargetError, "undefined root for query #{url}, use :static option as Hash to set a default protocol and host, or as True to allow using previously used root"
           elsif @static.is Hash
             # targeting relatively to default values (from @static hash)
             @loc.protocol = @static.protocol
@@ -223,7 +227,7 @@ module RHACK
             @loc.root = @loc.protocol+'://'+@loc.host
           end
           if !@loc.host
-            raise TargetError, "undefined host for query #{url}, use :host parameter of :static option to set default host"
+            raise TargetError, "undefined host for query #{url}, use :host parameter of :static option to set a default host"
           end
           File.join @loc.root, url
         else url
@@ -238,6 +242,15 @@ module RHACK
       urls.map! {|u| validate u}
     end
     
+    # Feature of :proc_result in that, if you running synchronously,
+    # result of #run will be, for conviniency, `page.res` instead of `page`
+    #
+    # If you only need to transfer &block through a stack of frame callbacks
+    # just add &block to the needed #run call
+    #
+    # If you want a method to be processable as in async-mode with &block passed
+    # as in sync-mode with no &block passed
+    # pass :save_result => !block to the topmost #run call
     def run_callbacks!(page, opts, &callback)
       # if no callback must have run then page.res is equal to the page
       # so we can get the page as result of a sync as well as an async request
@@ -280,12 +293,16 @@ module RHACK
         end
         if opts[:raw]
           page.res = yield curl
-      #   here +curl.res.body+ become empty
+      #   here +curl.res.body+ becomes empty
         elsif page.process(curl, opts)
           @@cache[page.href] = page if order[0] == :loadGet and @use_cache
           run_callbacks! page, opts, &callback
         end
       }
+      # > Carier.requests++
+      unless opts[:wait] and opts[:thread_safe] or opts[:exec] == false
+        Curl.execute :unless_already
+      end
       if opts[:wait]
         opts[:thread_safe] ? Curl.carier.perform : Curl.wait
         (opts[:save_result] or :proc_result.in opts) ? page.res : page
@@ -302,11 +319,16 @@ module RHACK
         end
         pages = orders.zip(with_opts[:ranges]).send(iterator) {|order, range| 
           (with_opts[:headers] ||= {}).Range = "bytes=#{range.begin}-#{range.end}"
-          exec_one order, with_opts, &callback
+          exec_one order, with_opts.merge(:exec => false), &callback
         }
       else
+        # если ss.next будет не хватать скаутов, то он сам запустит курл
+        # правда, это с :thread_safe никак не вяжется
         pages = orders.send(iterator) {|order| exec_one order, with_opts, &callback }
       end
+      unless w and with_opts[:thread_safe] or opts[:exec] == false
+        Curl.execute :unless_already
+      end
       with_opts[:thread_safe] ? Curl.carier.perform : Curl.wait if w
       with_opts[:stream] || pages
     end

data/lib/rhack/scout.rb

@@ -54,7 +54,7 @@ module RHACK
     
     def update(uri)
       if !uri[/^\w+:\/\//]
-        '/' >> uri if uri[0,1] != '/'
+        uri = '/' + uri if uri[0,1] != '/'
         @uri = uri.parse:uri
         return                       
       end
@@ -198,6 +198,10 @@ module RHACK
       Curl.carier.reqs.include? @http
     end
     
+    def available?
+      !loaded?
+    end
+    
     def load!
       unless Curl.carier.add @http
         Curl.carier.remove @http
@@ -214,7 +218,10 @@ module RHACK
       @http.timeout = @timeout
 
       @http.on_complete {|c|
+        # > Carier.requests--
         @error = nil
+        # While not outdated, Curl::Response here may contain pointers on freed
+        # memory, thus throwing exception on #to_s and #inspect
         c.outdate!
         ProcCookies c.res if @cookieProc
         # We cannot just cancel on_complete in on_redirect block
@@ -226,10 +233,11 @@ module RHACK
         end
       }
       @http.on_failure {|c, e|
+        @error = e
         if e[0] == Curl::Err::CurlOK
-          @error = e
-          # TODO: где-то в сорцах on_failure вызывается по коду 0, видимо из-за стороннего условия, а не должен
-          L.log << "Got Curl::Err::CurlOK, response was: #{c.res}"
+          # в сорцах on_failure не вызывается по коду 0, это какой-то глюк
+          # в любом случае такой поворот не означает ошибки
+          L.warn "Got Curl::Err::CurlOK, response was: #{c.res}"
         else
           @http.on_complete &Proc::NULL
           c.outdate!

data/lib/rhack/scout_squad.rb

@@ -34,47 +34,44 @@ module RHACK
     end
     
     def update uri, forced=nil
-      each {|s| return L.warn "failed to update scout loaded? with url: #{s.http.url}" if s.loaded?} if !forced
+      each {|s| return L.warn "failed to update loaded scout with url: #{s.http.url}" if s.loaded?} if !forced
       each {|s| s.update uri}
     end
     
     def untargeted
       first.root == 'http://'
     end
+    
+    def wait_for_available
+      Curl.execute :unless_already
+      # Carier.requests освобождаются ещё до колбека,
+      # но колбеки выполняются последовательно,
+      # поэтому здесь мы можем усыплять тред,
+      # но только если это не тред самого Carier
+      if Curl.carier_thread == Thread.current
+        Curl.wait # runs Multi#perform
+      else
+        sleep 1
+      end
+    end
       
     def rand
       raise PickError if !b
-      # to_a because reject returns object of this class
-      if scout = to_a.rand {|_|!_.loaded?}; scout
-      else # Curl should run here, otherwise `next'/`rand'-recursion will cause stack overflow
-        unless Curl.status
-          L.log "Curl must run in order to use ScoutSquad#rand; setting Carier Thread"
-          Curl.execute
-          #raise "Curl must run in order to use ScoutSquad#rand"
-        end
-        #Curl.wait
-        loop {
-          sleep 1
-          break if Curl.carier.reqs.size < size
-        }
+      # to_a because Array#reject returns object of this class
+      if scout = to_a.rand_by_available?
+        scout
+      else
+        wait_for_available
         self.rand 
       end 
     end
       
     def next
       raise PickError if !b
-      if scout = find {|_|!_.loaded?}; scout
-      else # Curl should run here, otherwise `next'/`rand'-recursion will cause stack overflow
-        unless Curl.status
-          L.log "Curl must run in order to use ScoutSquad#next; setting Carier Thread"
-          Curl.execute :unless_allready
-          #raise "Curl must run in order to use ScoutSquad#next"
-        end
-        #Curl.wait
-        loop {
-          sleep 1
-          break if Curl.carier.reqs.size < size
-        }
+      if scout = to_a.find_available?
+        scout
+      else
+        wait_for_available
         self.next
       end 
     end

data/lib/rhack/services.rb

@@ -1,10 +1 @@
-require 'rhack'
-require 'rhack/services/base'
-require 'rhack/services/storage'
-require 'rhack/services/oauth'
-
-module RHACK
-  for name in [:Service, :ServiceError]
-    autoload name, 'rhack/services/compatibility'
-  end
-end
+require 'rhack/clients'

data/lib/rhack/version.rb

@@ -1,3 +1,3 @@
 module RHACK
-  VERSION = '1.1.0'
+  VERSION = '1.1.4'
 end

metadata

@@ -1,20 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: rhack
 version: !ruby/object:Gem::Version
-  version: 1.1.0
-  prerelease: 
+  version: 1.1.4
 platform: ruby
 authors:
 - Sergey Baev
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-06-07 00:00:00.000000000 Z
+date: 2013-07-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -22,7 +20,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -30,7 +27,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -38,7 +34,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -46,7 +41,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rmtools
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -54,7 +48,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -62,7 +55,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: libxml-ruby
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -70,7 +62,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -78,7 +69,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -86,7 +76,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -94,7 +83,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -102,7 +90,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -156,6 +143,12 @@ files:
 - ext/curb/curb_upload.h
 - ext/curb/extconf.rb
 - lib/rhack.rb
+- lib/rhack/clients.rb
+- lib/rhack/clients/base.rb
+- lib/rhack/clients/compatibility.rb
+- lib/rhack/clients/examples.rb
+- lib/rhack/clients/oauth.rb
+- lib/rhack/clients/storage.rb
 - lib/rhack/cookie.rb
 - lib/rhack/curl.rb
 - lib/rhack/curl/easy.rb
@@ -179,11 +172,6 @@ files:
 - lib/rhack/scout.rb
 - lib/rhack/scout_squad.rb
 - lib/rhack/services.rb
-- lib/rhack/services/base.rb
-- lib/rhack/services/compatibility.rb
-- lib/rhack/services/examples.rb
-- lib/rhack/services/oauth.rb
-- lib/rhack/services/storage.rb
 - lib/rhack/storage.rb
 - lib/rhack/version.rb
 - lib/rhack_in.rb
@@ -194,26 +182,25 @@ files:
 homepage: https://github.com/tinbka/rhack
 licenses:
 - MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 2.0.3
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Curl-based web-client framework created for developing web-scrapers/bots
 test_files: []