pstuteville-scrubyt 0.4.31

data/lib/scrubyt/output/result_node.rb

@@ -0,0 +1,145 @@
+module Scrubyt
+  class ResultNode < Array
+    OUTPUT_OPTIONS = [:write_text]
+
+    attr_accessor :name, :result, :options, :generated_by_leaf
+
+    def initialize(name, result=nil, options={})
+      @name = name
+      @result = result
+      @options = options
+    end
+
+    def write_text
+      @options[:write_text].nil? ? @generated_by_leaf : @options[:write_text]
+    end
+
+    def has_content?
+      return true if result.is_a? String
+      write_text || (inject(false) { |one_child_has_content, child| one_child_has_content || child.has_content? })
+    end
+
+    def to_s
+      return "" if result.nil?
+      text = (@result.is_a? String) ? @result : @result.inner_html.gsub(/<.*?>/, '')
+      text = SharedUtils.unescape_entities(text)
+      text.strip!
+      if (@options[:default] && ((text == '') || (text == @options[:default])))
+        @options[:default]
+      else
+        text
+      end
+    end
+    
+    def inspect
+      to_s
+    end
+
+    def to_libxml
+      libxml_node = XML::Node.new(name)
+      self.each { |child| libxml_node << child.to_libxml if child.has_content? }
+      libxml_node << to_s if write_text
+      libxml_node
+    end
+
+    #note: see ruby_extensions.rb for String#write
+    def to_xml
+      to_xml_lines.join("\n")
+    end
+
+    def to_hash(delimiter=',')
+      result = []
+      flat_hash_inner = lambda {|e, hash|
+        hash[e.name.to_sym] = hash[e.name.to_sym] ? hash[e.name.to_sym] + delimiter + e.to_s : e.to_s  if ((e.write_text && !e.to_s.empty?) || e.options[:default])
+        e.each {|c| flat_hash_inner.call(c, hash)  }
+        hash
+      }
+      self.each {|e| result << flat_hash_inner.call(e, {}) }
+      result
+    end
+    
+    def to_flat_hash()
+      hash_result = self.to_hash('@@@@@@')
+      merged_hash = hash_result.delete_at 0
+      hash_result.each do |hash|
+        merged_hash.keys.each do |key|
+          merged_hash[key] += "@@@@@@#{hash[key]}"
+        end
+      end
+      result_sets = merged_hash.values.map!{|x| x.split('@@@@@@')}.transpose
+      final_result = []
+
+      result_sets.each do |rs|
+        temp_result = {}
+        merged_hash.keys.each do |k|
+          temp_result[k] = rs[merged_hash.keys.index(k)]
+        end
+        final_result << temp_result
+      end
+      final_result
+    end
+
+    def to_flat_xml(delimiter=nil)
+      lines = []
+      hash_result = delimiter ? self.to_hash(delimiter) : self.to_hash
+      merged_hash = hash_result.delete_at 0
+
+      hash_result.each do |hash|
+        merged_hash.keys.each do |key|
+          merged_hash[key] += "#{delimiter}#{hash[key]}"
+        end
+      end
+
+      if delimiter
+        result_sets = merged_hash.values.map!{|x| x.split(delimiter)}.transpose
+        final_result = []
+
+        result_sets.each do |rs|
+          temp_result = {}
+          merged_hash.keys.each do |k|
+            temp_result[k] = rs[merged_hash.keys.index(k)]
+          end
+          final_result << temp_result
+        end
+        hash_result = final_result
+      end
+
+      hash_result.each do |hash|
+        lines << "<item>"
+        hash.each do |key, value|
+          xml_tag = key.to_s
+          value = '' if value == '#empty#'
+          lines << "  <#{xml_tag}>#{REXML::Text.normalize(value)}</#{xml_tag}>"
+        end
+        lines << "</item>"
+      end
+      return lines.join("\n")
+
+    end
+
+    def to_xml_lines
+      lines = []
+      children = self.select{ |child| child.has_content? }
+      if children.empty?
+        if result.is_a? String
+          lines << "<#{name}>#{result}</#{name}>"
+        elsif write_text && !to_s.empty?
+          lines << "<#{name}>#{ERB::Util.html_escape(to_s)}</#{name}>"
+        else
+          if @options[:default]
+            lines << "<#{name}>#{@options[:default]}</#{name}>"
+          else
+            lines << "<#{name}/>"
+          end
+        end
+      else
+        lines << "<#{name}>"
+        lines << "  #{ERB::Util.html_escape(to_s)}" if write_text && !to_s.empty?
+        children.each do |child|
+          lines.push(*child.to_xml_lines.map{ |line| "  #{line}" })
+        end
+        lines << "</#{name}>"
+      end
+    end
+  end
+end

data/lib/scrubyt/output/scrubyt_result.rb

@@ -0,0 +1,42 @@
+module Scrubyt
+  class ScrubytResult < ResultNode
+    attr_accessor :root_patterns, :source_file, :source_proc
+
+    def export
+      #Temporary solution; the real one will be back later - or not     
+     result = <<-EXPLANATION
+     
+     === Extractor tree ===
+     
+     export() is not working at the moment, due to the removal or ParseTree, ruby2ruby and RubyInline.
+     For now, in case you are using examples, you can replace them by hand based on the output below.
+     So if your pattern in the learning extractor looks like
+     
+     book "Ruby Cookbook" 
+     
+     and you see the following below:
+     
+     [book] /table[1]/tr/td[2]
+     
+     then replace "Ruby Cookbook" with "/table[1]/tr/td[2]" (and all the other XPaths) and you are ready!
+     
+     EXPLANATION
+     
+     tree_builder = lambda do |node, level|        
+       result += current_level = ("   " * (level == 0 ? 0 : level-1) + 
+                                  "|\n" * (level == 0 ? 0 : 1) +
+                                  "   " * (level == 0 ? 0 : level-1) + 
+                                 "+-- " * (level == 0 ? 0 : 1) + 
+                                 "[#{node.name}]")
+       result += " #{node.filters[0].xpath}" if node.type == :tree
+       result += "\n"                                 
+                                 
+       node.children.each {|c| tree_builder[c, level+1]}
+     end
+     
+     tree_builder[root_patterns[0],0]
+          
+     result += "\n"
+    end
+  end
+end

data/lib/scrubyt/utils/compound_example_lookup.rb

@@ -0,0 +1,50 @@
+module Scrubyt
+  #=<tt>Lookup of compund examples</tt>
+  #There are two types of string examples in scRUBYt! right now:
+  #the simple example and the compound example.
+  #
+  #This class is responsible for finding elements matched by compound examples.
+  #In the futre probably more sophisticated matching algorithms will be added
+  #(e.g. match the n-th which matches the text, or element that matches the
+  #text but also contains a specific attribute etc.)
+  class CompoundExampleLookup
+    def self.find_node_from_compund_example(doc, compound_example, next_link=false, index = 0)
+      @partial_results = []
+      self.lookup_compound_example(doc, compound_example, index)
+    end
+
+private
+    #Lookup the first element which is matched by this compund example
+    #
+    #A compound example is specified with :contains, :begins_with and
+    #:ends_with descriptors - which can be both regexps or strings
+    #
+    #Example:
+    #
+    #flight_info :begins_with => 'Arrival', :contains => /\d\d-\d+/, :ends_with => '20:00'
+    def self.lookup_compound_example(doc, compound_example, index)
+      compound_example.each do |k,v|
+        v = Regexp.escape(v) if v.is_a? String
+        case k
+          when :contains
+            v = /#{v}/
+          when :begins_with
+            v = /^\s*#{v}/
+          when :ends_with
+            v = /#{v}\s*$/
+        end
+        if (@partial_results.empty?)
+          @partial_results = SharedUtils.traverse_for_match(doc, v)
+        else
+          refine_partial_results(v)
+        end
+      end
+      @partial_results[index]
+    end
+
+    def self.refine_partial_results(regexp)
+      @partial_results = @partial_results.select {|pr| pr.inner_html.gsub(/<.*?>/, '') =~ regexp}
+    end
+
+  end #End of class CompoundExampleLookup
+end #End of module Scrubyt

data/lib/scrubyt/utils/ruby_extensions.rb

@@ -0,0 +1,85 @@
+class Module
+  def option_reader(key_default_hash)
+    key_default_hash.each do |key, default|
+      define_method(key) {
+        if @options[key].nil?
+          if default.is_a? Proc
+            instance_eval(&default)
+          else
+            default
+          end
+        else
+          @options[key]
+        end
+      }
+    end
+  end
+  
+  def option_writer(*keys)
+    keys.each do |key|
+      define_method("#{key.to_s}=".to_sym) { |value|
+        @options[key] = value
+      }
+    end
+  end
+  
+  def option(key, default=nil, writable=false)
+    option_reader(key => default)
+    option_writer(key) if writable
+  end
+  
+  def option_accessor(key_default_hash)
+    key_default_hash.each do |key, default|
+      option(key, default, true)
+    end
+  end
+end
+
+class Range
+  def <=>(other)
+    self.begin <=> other.begin
+  end
+  
+  def +(amount)
+   (self.begin + amount)..(self.end + amount)
+  end
+  
+  def -(amount)
+   (self.begin - amount)..(self.end - amount)
+  end
+end
+
+module Math
+  def self.min(a, b)
+    a < b ? a : b
+  end
+  
+  def self.max(a, b)
+    a > b ? a : b
+  end
+end
+
+#dec 16: Dropped - causes some errors w/ Rails
+#just some hack here to allow current examples' syntax:
+#table_data.to_xml.write(open('result.xml', 'w'), 1)
+#class String
+#  def write(stringio, add_indent=0)
+#    stringio.write((self.split("\n").collect { |line| ('  ' * add_indent) + line }).join("\n"))
+#  end
+#end
+
+#hack to simulate ancestor::tag selector of XPAth
+module Hpricot
+  class Elem
+    def ancestors(tag = nil)
+      element=self
+      path=Hpricot::Elements.new
+      while element.class != Hpricot::Doc do
+        return element if (tag && (tag ==element.name))
+        path.push element
+        element = element.parent
+      end
+      path
+    end
+  end
+end

data/lib/scrubyt/utils/shared_utils.rb

@@ -0,0 +1,58 @@
+module Scrubyt
+  ##
+  #=<tt>Utilities shared between the other utility classes (XPathUtils, SimpleExampleLookup,...)</tt>
+  #
+  class SharedUtils
+    #Entities to replace - need to make this more complete, or install htmlentities or similar package
+    ENTITIES = {
+        'quot'      => '"',
+        'apos'      => "'",
+        'amp'       => '&',
+        'lt'        => '<',
+        'gt'        => '>',
+        'nbsp'      => ' '}
+
+    def self.prepare_text_for_comparison(text)
+      unescape_entities text
+      text.strip!
+      text
+    end
+
+    #Unescape the entities in the HTML!
+    def self.unescape_entities(text)
+      ENTITIES.each {|e,s| text.gsub!(/\&#{e};/) {"#{s}"} }
+      text
+    end
+
+    #Entry point for finding the elements specified by examples
+    def self.traverse_for_match(node, regexp)
+      results = []
+      traverse_for_match_inner = lambda { |node, regexp|
+        ft = prepare_text_for_comparison(node.inner_html.gsub(/<.*?>/, ''))
+        if ft =~ regexp
+          node.instance_eval do
+            @match_data = $~
+            def match_data
+              @match_data
+            end
+          end
+          results << node
+          results.delete node.parent if node.is_a? Hpricot::Elem
+        end
+        node.children.each { |child| traverse_for_match_inner.call(child, regexp) if (child.is_a? Hpricot::Elem) } if ! node.children.nil?
+      }
+      traverse_for_match_inner.call(node,regexp)
+      results
+    end
+
+    def self.get_backtrace
+      begin
+        raise
+      rescue Exception => ex
+        backtrace = ex.backtrace
+      end
+      backtrace.slice!(0)
+      backtrace
+    end
+  end #end of class SharedUtils
+end #end of module Scrubyt

data/lib/scrubyt/utils/simple_example_lookup.rb

@@ -0,0 +1,40 @@
+module Scrubyt
+  #=<tt>Lookup of simple examples</tt>
+  #There are two types of string examples in scRUBYt! right now:
+  #the simple example and the compound example.
+  #
+  #This class is responsible for finding elements matched by simple examples.
+  #In the futre probably more sophisticated matching algorithms will be added
+  #(e.g. match the n-th which matches the text, or element that matches the
+  #text but also contains a specific attribute etc.)
+  class SimpleExampleLookup
+    #From the example text defined by the user, find the lowest possible node which contains the text 'text'.
+    #The text can be also a mixed content text, e.g.
+    #
+    # <a>Bon <b>nuit</b>, monsieur!</a>
+    #
+    #In this case, <a>'s text is considered to be "Bon nuit, monsieur"
+    def self.find_node_from_text(doc, text, next_link=false, index = 0)
+      text.gsub!('»', '&#187;')
+      #Process immediate attribute extraction (like "go to google.com/@href")
+      if text =~ /.+\/@.+$/
+        text = text.scan(/^(.+?)\/@.+$/)[0][0]
+      elsif text =~ /.+\[\d+\]$/
+        res = text.scan(/(.+)\[(\d+)\]$/)
+        text = res[0][0]
+        index = res[0][1].to_i
+      elsif text =~ /.+\[.+\]$/
+        final_element_name = text.scan(/^(.+?)\[/)[0][0]
+        text = text.scan(/\[(.+?)\]/)[0][0]
+      end
+      if final_element_name
+        text = Regexp.escape(text) if text.is_a? String
+        result = SharedUtils.traverse_for_match(doc,/#{text}/)[index]
+        result = XPathUtils.traverse_up_until_name(result,final_element_name)
+      else
+        text = Regexp.escape(text) if text.is_a? String
+        result = SharedUtils.traverse_for_match(doc,/^#{text}$/)[index]
+      end
+    end
+  end #End of class SimpleExampleLookup
+end #End of module Scrubyt

data/lib/scrubyt/utils/xpathutils.rb

@@ -0,0 +1,202 @@
+require 'rubygems'
+require 'hpricot'
+
+module Scrubyt
+  ##
+  #=<tt>Various XPath utility functions</tt>
+  class XPathUtils
+
+    #Find the LCA (Lowest Common Ancestor) of two nodes
+    def self.lowest_common_ancestor(node1, node2)
+      path1 = traverse_up(node1)
+      path2 = traverse_up(node2)
+      return node1.parent if path1 == path2
+
+      closure = nil
+      while (!path1.empty? && !path2.empty?)
+	    closure = path1.pop
+	    return closure.parent if (closure != path2.pop)
+      end
+      path1.size > path2.size ? path1.last.parent : path2.last.parent
+    end
+
+    ##
+    #Generate XPath for the given node
+    #
+    #*parameters*
+    #
+    #_node_ - The node we are looking up the XPath for
+    #
+    #_stopnode_ - The Xpath generation is stopped and the XPath that
+    #was generated so far is returned if this node is reached.
+    #
+    #_write_indices_ - whether the index inside the parent shuold be
+    #added, as in html[1]/body[1]/table[2]/tr[1]/td[8]
+    def self.generate_XPath(node, stopnode=nil, write_indices=false)
+      path = []
+      indices = []
+      found = false
+      while !node.nil? && node.class != Hpricot::Doc do
+        if node == stopnode
+          found = true
+          break
+        end
+        path.push node.name
+        indices.push find_index(node) if write_indices
+        node = node.parent
+      end
+      #This condition ensures that if there is a stopnode, and we did not found it along the way,
+      #we return nil (since the stopnode is not contained in the path at all)
+      return nil if stopnode != nil && !found
+      result = ""
+      if write_indices
+        path.reverse.zip(indices.reverse).each { |node,index| result += "#{node}[#{index}]/" }
+      else
+        path.reverse.each{ |node| result += "#{node}/" }
+      end
+      "/" + result.chop
+    end
+
+    #Generate an XPath of the node with indices, relatively to the given
+    #relative_root.
+    #
+    #For example if the elem's absolute XPath is /a/b/c,
+    #and the relative root's Xpath is a/b, the result of the function will
+    #be /c.
+    def self.generate_relative_XPath( elem,relative_root )
+      return nil if (elem == relative_root)
+      generate_XPath(elem, relative_root, true)
+    end
+
+    #Generate a generalized XPath (i.e. without indices) of the node,
+    #relatively to the given relative_root.
+    #
+    #For example if the elem's absolute XPath is /a[1]/b[3]/c[5],
+    #and the relative root's Xpath is a[1]/b[3], the result of the function will
+    #be /c.
+    def self.generate_generalized_relative_XPath( elem,relative_root )
+      return nil if (elem == relative_root)
+      generate_XPath(elem, relative_root, false)
+    end
+
+    #Find an image based on the src of the example
+    #
+    #*parameters*
+    #
+    #_doc_ - The containing document
+    #
+    #_example_ - The value of the src attribute of the img tag
+    #This is convenient, since if the users rigth-clicks an image and
+    #copies image location, this string will be copied to the clipboard
+    #and thus can be easily pasted as an examle
+    #
+    #_index_ - there might be more images with the same src on the page -
+    #most typically the user will need the 0th - but if this is not the
+    #case, there is the possibility to override this
+    def self.find_image(doc, example, index=0)
+      if example =~ /\.(jpg|png|gif|jpeg)(\[\d+\])$/
+        res = example.scan(/(.+)\[(\d+)\]$/)
+        example = res[0][0]
+        index = res[0][1].to_i
+      end
+      (doc/"//img[@src='#{example}']")[index]
+    end
+
+    ##
+    #Used to find the parent of a node with the given name - for example
+    #find the <form> node which is the parent of the <input> node
+    def self.traverse_up_until_name(node, name)
+      while node.class != Hpricot::Doc do
+        #raise "The element is nil! This probably means the widget with the specified name ('#{name}') does not exist" unless node
+        return nil unless node
+        break if node.name == name
+        node = node.parent
+      end
+      node
+    end
+
+    ##
+    #Used when automatically looking up href attributes (for detail or next links)
+    #If the detail pattern did not extract a link, we first look up it's
+    #children - and if we don't find a link, traverse up
+    def self.find_nearest_node_with_attribute(node, attribute)
+      @node = nil
+      return node if node.is_a? Hpricot::Elem and node[attribute]
+      first_child_node_with_attribute(node, attribute)
+      first_parent_node_with_attribute(node, attribute) if !@node
+      @node
+    end
+
+    ##
+    #Generalre relative XPath from two XPaths: a parent one, (which points higher in the tree),
+    #and a child one. The result of the method is the relative XPath of the node pointed to
+    #by the second XPath to the node pointed to by the firs XPath.
+    def self.generate_relative_XPath_from_XPaths(parent_xpath, child_xpath)
+      original_child_xpath_parts = child_xpath.split('/').reject{|s|s==""}
+      pairs = to_general_XPath(child_xpath).split('/').reject{|s|s==""}.zip to_general_XPath(parent_xpath).split('/').reject{|s|s==""}
+      i = 0
+      pairs.each_with_index do |pair,index|
+        i = index
+        break if pair[0] != pair[1]
+      end
+      "/" + original_child_xpath_parts[i..-1].join('/')
+    end
+
+    def self.to_full_XPath(doc, xpath, generalize)
+      elem = doc/xpath
+      elem = elem.map[0] if elem.is_a? Hpricot::Elements
+      XPathUtils.generate_XPath(elem, nil, generalize)
+    end
+
+private
+    #Find the index of the child inside the parent
+    #For example:
+    #
+    #         tr
+    #      /  |   \
+    #    td   td   td
+    #    0    1    2
+    #
+    #The last row contains the indices of the td's from the
+    #tow above.
+    #
+    #Note that in classic XPath, the indices start with 1 (rather
+    #than 0).
+    def self.find_index(node)
+     c = 0
+     node.parent.children.each do |child|
+       if child.class == Hpricot::Elem
+         c += 1 if (child.name == node.name)
+         break if (node == child)
+       end
+     end
+     c
+    end
+
+    def self.traverse_up(node, stopnode=nil)
+      path = []
+      while node.class != Hpricot::Doc do
+        break if node == stopnode
+        path.push node
+        node = node.parent
+      end
+    path
+    end
+
+    def self.first_child_node_with_attribute(node, attribute)
+      return if !node.instance_of? Hpricot::Elem || @node
+      @node = node if node.attributes[attribute]
+      node.children.each  { |child| first_child_node_with_attribute(child, attribute) }
+    end
+
+    def self.first_parent_node_with_attribute(node, attribute)
+      return if !node.instance_of? Hpricot::Elem || @node
+      @node = node if node.attributes[attribute]
+      first_parent_node_with_attribute(node.parent, attribute)
+    end
+
+    def self.to_general_XPath(xpath)
+      xpath.gsub(/\[.+?\]/) {""}
+    end #End of method to_general_XPath
+  end #End of class XPathUtils
+end #End of module Scrubyt

data/lib/scrubyt.rb

@@ -0,0 +1,53 @@
+if RUBY_VERSION < '1.9'
+  $KCODE = "u"
+  require "jcode"
+end
+
+#ruby core
+require "open-uri"
+require "erb"
+
+#gems
+require "rexml/text"
+require "rubygems"
+require "mechanize"
+require "hpricot"
+
+#scrubyt
+require "#{File.dirname(__FILE__)}/scrubyt/logging"
+require "#{File.dirname(__FILE__)}/scrubyt/utils/ruby_extensions.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/utils/xpathutils.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/utils/shared_utils.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/utils/simple_example_lookup.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/utils/compound_example_lookup.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/core/scraping/constraint_adder.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/core/scraping/constraint.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/core/scraping/result_indexer.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/core/scraping/pre_filter_document.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/core/scraping/compound_example.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/output/result_node.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/output/scrubyt_result.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/core/navigation/agents/mechanize.rb"
+
+# -- Making Firewatir optional --
+begin
+  require "#{File.dirname(__FILE__)}/scrubyt/core/navigation/agents/firewatir.rb"
+rescue LoadError
+  puts "The gem firewatir is not installed, you'll be able to use Mechanize as the agent only"
+end
+# --
+
+require "#{File.dirname(__FILE__)}/scrubyt/core/navigation/navigation_actions.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/core/navigation/fetch_action.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/core/shared/extractor.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/core/scraping/filters/base_filter.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/core/scraping/filters/attribute_filter.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/core/scraping/filters/constant_filter.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/core/scraping/filters/script_filter.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/core/scraping/filters/text_filter.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/core/scraping/filters/detail_page_filter.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/core/scraping/filters/download_filter.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/core/scraping/filters/html_subtree_filter.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/core/scraping/filters/regexp_filter.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/core/scraping/filters/tree_filter.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/core/scraping/pattern.rb"