rhack 1.1.6 → 1.1.7

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NWNkY2FhMDI2NTg4YWMyOTNkNGRkMzJhYjc1NWFhYTU2Yzc3NzAzYg==
+    YzliZTlhMjBjZjM2MzE4ZjNkZDc0MDQzYjg3MDM5NGZmZjdmMjUwMA==
   data.tar.gz: !binary |-
-    ODM0MDBmYmYyNjQxY2Q1M2YzYjhmYjFmNDY2N2QzNTRkZDQxNDA1Ng==
+    OGFmNGYyN2Y0MTRkODYzNGRlNzc3NDY1ZTg3MTU5ZTBhZTdiN2FjZQ==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    ZGEyMzYxYzY0Y2U0Y2JhZjA5NjMzNDQ1NWU3YzA5NDllNWU3NWNiODU2ZTNl
-    OTZmNGQ1ZWIzZTdiOTUyNGM3N2U4ZTJiMDBkMzY2MDA5N2VmNDBjNGI5NDE4
-    Mjk4NWM0NTE3MzViNTlkYzQ3YzE2ZTdiMzVmNjA1YWI0ZDhjZDY=
+    NGQyMjRiNDZmZjRhMWMzZGUwZGE3NjhlMmE2ODYzOGM2NTBkN2JhYThjMDMw
+    ZmZmODg3OThhMzFlNjU1ZjQ2MzI3ZjliY2ZlN2IyNDNiMGE0OTgxZTQxMDU2
+    OGRhYzlhNzgxYzY5ZGEyN2Y1MmZkNjVlZTA0OWJjMThjMGMyNTQ=
   data.tar.gz: !binary |-
-    ODVhN2NlNDQzMzFmNzhiZGI5MDY3YWRmOTAyZWM0MDdkOTU0NjQ1MTc5ZmIw
-    YzcxM2M0Y2U5NTkxYWRkNGYxZDU0YTZjNTEyNmJiMjA1YjI2NjI3YmQ3NjM4
-    ODRmYWUzMTFiOGM1YmJlOWQzYTMwM2M3OTQyNjVkNjlmYTU1ZWM=
+    NTQ1ODZhMWY1NTIwZjcxZDM3YzVlY2ZiNTE3NDE2YmVkZDI1M2M4ODYyZjY3
+    OGM0OTM4NjNjZjU5NzNhMTJjYzMyODA0YTNiOWE1NjFiMmM3MWUyN2MxNTY1
+    NzU2OWE2YTU0M2RmM2VhM2FkMGE0YjhjMzU3ZDNkYWEyN2M5MmY=

data/README.md

@@ -31,6 +31,11 @@ It's still randomly documented since it's just my working tool.
 
 ### CHANGES
 
+##### Version 1.1.7
+
+* ::Page
+  * Fixed #expand_link for partial links
+
 ##### Version 1.1.6
 
 * ::Frame

data/ext/curb/curb_multi.c

@@ -71,12 +71,8 @@ rb_hash_clear_i(VALUE key, VALUE value, VALUE dummy) {
 }
 
 static void curl_multi_free(ruby_curl_multi *rbcm) {
-  if (rbcm) {
-    ruby_log((!rbcm->requests == Qnil) ? "true" : "false");
-    ruby_log_obj(rbcm->requests);
-  }
-  if (rbcm && !rbcm->requests == Qnil && rb_type(rbcm->requests) == T_HASH && RHASH_LEN(rbcm->requests) > 0) {
-  //if (rbcm && rb_type(rbcm->requests) == T_HASH && RHASH_LEN(rbcm->requests) > 0) {
+  //if (rbcm && !rbcm->requests == Qnil && rb_type(rbcm->requests) == T_HASH && RHASH_LEN(rbcm->requests) > 0) {
+  if (rbcm && rb_type(rbcm->requests) == T_HASH && RHASH_LEN(rbcm->requests) > 0) {
     rb_hash_foreach( rbcm->requests, (int (*)())curl_multi_flush_easy, (VALUE)rbcm );
     rb_hash_foreach(rbcm->requests, rb_hash_clear_i, 0); //rb_hash_clear(rbcm->requests);
     rbcm->requests = Qnil;

data/lib/rhack/curl/global.rb

@@ -44,6 +44,7 @@ module Curl
     end
     alias :run :execute
     
+    # TODO: вместо статуса sleep/run использовать глобальную класс-переменную
     def wait
       if @@carier_thread and @@carier_thread.status
         unless within = Thread.current == @@carier_thread

data/lib/rhack/page.rb

@@ -1,10 +1,35 @@
 # encoding: utf-8
-module RHACK
+class Object
+  unless defined? one_argument_is_a?
+    alias :one_argument_is_a? :is_a? 
+    def is_a?(class1, *other_classes)
+      one_argument_is_a? class1 or other_classes.any? {|cl| one_argument_is_a? cl}
+    end
+  end
+end
 
-    # Frame( ScoutSquad( Curl::Multi <- Scout( Curl API ), Scout, ... ) ) => 
-    # Curl -> Johnson::Runtime -> XML::Document => Page( XML::Document ), Page, ... 
+module RHACK
+        
+  class JsonString < String
+    __init__
+    attr_reader :source
+    
+    def initialize(source)
+      @source = source
+      super source.to_json
+    end
     
-    class Page
+    def inspect
+      "#<RHACK::JsonString(#{@source.inspect})>"
+    end
+  end
+      
+  class ScrapeError < ArgumentError; end
+  class NodeNotFound < ScrapeError; end
+
+  # Frame( ScoutSquad( Curl::Multi <- Scout( Curl API ), Scout, ... ) ) => 
+  # Curl -> Johnson::Runtime -> XML::Document => Page( XML::Document ), Page, ... 
+  class Page
     # for debug, just enable L#debug, don't write tons of chaotic log-lines 
     __init__
     attr_writer :title
@@ -44,6 +69,9 @@ module RHACK
       @html.force_encoding(encoding)
     end
     
+    def url() @loc.href end
+    alias :href :url
+    
     # We can then alternate #process in Page subclasses
     # Frame doesn't mind about value returned by #process
     def process(c, opts={})
@@ -150,63 +178,169 @@ module RHACK
         @loc.href
       end
     end
-      
-    def find(xp) (@doc || to_doc).find xp end
     
-    def at(xp) (@doc || to_doc).at xp end
     
-    def url() @loc.href end
-    alias :href :url
+    # HELPERS #
     
-    def get_srcs(links='img')
-      begin
-        links = find(links).map {|e| e.src} if links.is String
-      rescue XML::Error
-        links = [links]
+    # hook to create even-looked lines defining a hash in my Verdana 10px, e.g.
+    # dict key1: value1, ...
+    #       key2: value2, ...
+    def dict(hash)
+      hash.is_a?(Hash) ? hash : Hash[hash]
+    end
+    
+    # maps {'firstname lastname' => tuple} into {:firstname => tuple[0], :lastname => tuple[1]}
+    def flatten_dict(hash)
+      result = {}
+      hash.each {|k, v|
+        if k.is String and k[' ']
+          k.split(' ').each_with_index {|k_unit, k_idx|
+            result[k_unit.to_sym] = v[k_idx]
+          }
+        else
+          result[k.to_sym] = v
+        end
+      }
+      result
+    end
+    
+    # makes a relative path being on this page into an absolute path
+    def expand_link(link)
+      case link
+        when /^\w+:\/\// then link
+        when /^\/\// then @loc.protocol + ':' + link
+        when /^\// then @loc.root + link
+        when /^\?/ then File.join(@loc.root, @loc.path) + link
+        when /^#/ then File.join(@loc.root, @loc.fullpath) + link
+        else File.join @loc.root, File.dirname(@loc.path), link
       end
-      links.map {|link| expand_link link}.uniq
     end
     
-    def get_src(link='img')
-      begin
-        link = at(link) && at(link).src if link.is String
-      rescue XML::Error; nil
+    
+    # FINDERS #
+      
+  private
+      
+    def node_is_missing!(selector, options)
+      missing = options[:missing]
+      if missing.is Proc
+        missing.call(selector)
+      elsif missing
+        if missing.is String
+          message %= {selector: selector}
+        end
+        raise NodeNotFound, missing
+      end
+    end
+      
+    def preprocess_search_result(preresult, preprocess)
+      if preprocess.is_a? Proc
+        preprocess.call(preresult)
+      elsif preprocess.is_a? Symbol
+        __send__(preprocess, preresult)
+      else
+        preresult
+      end
+    end
+      
+    def preprocess_search_results(preresult, preprocess)
+      if preprocess.is_a? Proc
+        preresult.map(&preprocess)
+      elsif preprocess.is_a? Symbol
+        preresult.map {|node| __send__(preprocess, node)}
+      else
+        preresult
       end
-      expand_link link if link
     end
     
-    def get_links(links='a')
-      begin
-        links = find(links).map {|e| e.href}.b || find(links+'//a').map {|e| e.href} if links.is String
-      rescue XML::Error
-        links = [links]
+    def __at(xp) (@doc || to_doc).at xp end
+    
+    def __find(xp) (@doc || to_doc).find xp end
+    
+  public
+    
+    def at(selector_or_node, options={})
+      if selector_or_node and preresult = selector_or_node.is_a?(XML::Node) ? 
+          selector_or_node : __at(selector_or_node)
+          
+        preresult = preprocess_search_result(preresult, options[:preprocess])
+        block_given? ? yield(preresult) : preresult
+      else
+        node_is_missing!(selector_or_node, options)
+        preresult
       end
-      links.map {|link| expand_link link}.uniq
     end
+    alias :first :at
     
-    def get_link(link='a')
-      begin
-        link = at(link) && (at(link).href || at(link+'//a').href) if link.is String
-      rescue XML::Error; nil
+    def find(selector_or_nodes, options={}, &foreach)
+      preresult = selector_or_nodes.is_a?(XML::XPath::Object, Array) ?
+        selector_or_nodes : __find(selector_or_nodes)
+        
+      if preresult.size > 0
+        preresult = preprocess_search_results(preresult, options[:preprocess])
+        foreach ? preresult.each(&foreach) : preresult
+      else
+        node_is_missing!(selector_or_nodes, options)
+        preresult
       end
-      expand_link link if link
     end
-    alias :get_hrefs :get_links
-    alias :links :get_links
-    alias :get_href :get_link
-    alias :link :get_link
-    alias :srcs :get_srcs
+    alias :all :find
+    
+    
+    # FINDERS PREPROCESSORS #
+    
+    def text(selector_or_node, options={})
+      if node = at(selector_or_node, options)
+        txt = node.text.strip
+        block_given? ? yield(txt) : txt
+      end
+    end
+    
+    def texts(hash, options={})
+      hash.map_values {|selector_or_node|
+        text(selector_or_node, options)
+      }
+    end
+    
+    def get_src(selector_or_node='img', options={}, &onfound)
+      at(selector_or_node, options.merge(:preprocess => lambda {|node|
+        if src = node.src
+          expand_link src
+        end
+      })) {|href| onfound && src ? onfound.call(src) : src}
+    end
     alias :src :get_src
     
-    def expand_link(link)
-      case link
-        when /^\w+:\/\// then link
-        when /^\/\// then @loc.protocol+link
-        when /^\// then @loc.root+link
-        else File.join((@loc.path.b ? File.dirname(@loc.path) : @loc.root), link)
+    def get_link(selector_or_node='a', options={}, &onfound)
+      at(selector_or_node, options.merge(:preprocess => lambda {|node|
+        unless href = node.href
+          if node = node.find('a')
+            href = node.href
+          end
+        end
+        if href
+          expand_link href
+        end
+      })) {|href| onfound && href ? onfound.call(href) : href}
+    end
+    alias :link :get_link
+    alias :get_href :get_link
+    
+    def map(selector_or_nodes, options={}, &mapper)
+      mapping = find(selector_or_nodes, options.merge(:preprocess => mapper))
+      unless options[:compact] == false
+        mapping = mapping.to_a.compact
       end
+      mapping
     end
     
+    def map_json(selector_or_nodes, options={}, &mapper)
+      JsonString map(selector_or_nodes, options, &mapper)
+    end
+    
+    
+    # FORMS #
+    
     def form(form='form', hash={}, opts={})
       form = "[action=#{@loc.path.inspect}]" if form == :self
       if form.is String
@@ -237,6 +371,47 @@ module RHACK
       page
     end
     
+    
+    # OLD #
+    
+    # TODO: make into same form as #get_src and #map
+    def get_srcs(links='img')
+      begin
+        links = find(links).map {|e| e.src} if links.is String
+      rescue XML::Error
+        links = [links]
+      end
+      links.map {|link| expand_link link}.uniq
+    end
+    alias :srcs :get_srcs
+    
+    #def get_src(link='img')
+    #  begin
+    #    link = at(link) && at(link).src if link.is String
+    #  rescue XML::Error; nil
+    #  end
+    #  expand_link link if link
+    #end
+    
+    def get_links(links='a')
+      begin
+        links = find(links).map {|e| e.href}.b || find(links+'//a').map {|e| e.href} if links.is String
+      rescue XML::Error
+        links = [links]
+      end
+      links.map {|link| expand_link link}.uniq
+    end
+    alias :get_hrefs :get_links
+    alias :links :get_links
+    
+    #def get_link(link='a')
+    #  begin
+    #    link = at(link) && (at(link).href || at(link+'//a').href) if link.is String
+    #  rescue XML::Error; nil
+    #  end
+    #  expand_link link if link
+    #end
+    
     def load_scripts(frame)
       frame && frame.get_cached(*get_srcs("script[src]")).each {|js| eval_string js}
     end

data/lib/rhack/version.rb

@@ -1,3 +1,3 @@
 module RHACK
-  VERSION = '1.1.6'
+  VERSION = '1.1.7'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rhack
 version: !ruby/object:Gem::Version
-  version: 1.1.6
+  version: 1.1.7
 platform: ruby
 authors:
 - Sergey Baev
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-03 00:00:00.000000000 Z
+date: 2013-08-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport