pstuteville-scrubyt 0.4.31

data/lib/scrubyt/core/shared/extractor.rb

@@ -0,0 +1,171 @@
+module Scrubyt
+  ##
+  #=<tt>Driving the whole extraction process</tt>
+  #
+  #Extractor is a performer class - it gets an extractor definition and carries
+  #out the actions and evaluates the wrappers sequentially. 
+  #
+  #Originally also the navigation actions were here, but since the class got too
+  #big, they were factored out to an own class, NavigationAction.
+  class Extractor
+    include FetchAction
+    
+    attr_accessor :result, :evaluating_extractor_definition, :mode, :root_patterns, :next_page_pattern#, :hpricot_doc, :current_doc_url
+    
+    #The definition of the extractor is passed through this method
+    def self.define(mode=nil, &extractor_definition)
+      if mode.is_a?(Hash)
+        if mode[:agent] == :firefox
+          FetchAction.class_eval do
+            include Navigation::Firewatir
+          end
+        else
+          FetchAction.class_eval do
+            include Navigation::Mechanize
+          end
+        end
+      else
+        FetchAction.class_eval do
+          include Navigation::Mechanize
+        end
+      end
+      extractor = self.new(mode, extractor_definition)
+      extractor.result
+    end
+    
+    def self.load(filename)
+      define(&eval(IO.read(filename)))
+    end
+    
+    def initialize(mode, extractor_definition)
+      @mode = mode
+      @root_patterns = []
+      @next_page_pattern = nil
+      #      @hpricot_doc = nil
+      #      @hpricot_doc_url = nil
+      @evaluating_extractor_definition = false
+      @next_page_list = []
+      @processed_pages = []
+      
+      backtrace = SharedUtils.get_backtrace
+      parts = backtrace[1].split(':')
+      source_file = parts[0]
+      
+      Scrubyt.log :MODE, mode == :production ? 'Production' : 'Learning'
+      
+      @evaluating_extractor_definition = true
+      context = Object.new
+      context.extend NavigationActions
+      context.instance_eval do
+        def extractor=(value)
+          @extractor = value
+        end
+        
+        def next_page(*args)
+          @extractor.next_page_pattern = Scrubyt::Pattern.new('next_page', args, @extractor)
+        end
+        
+        def method_missing(method_name, *args, &block)
+          root_pattern = Scrubyt::Pattern.new(method_name.to_s, args, @extractor, nil, &block)
+          @extractor.root_patterns << root_pattern
+          root_pattern
+        end
+      end
+      FetchAction.extractor = self
+      context.extractor = self
+      context.instance_eval(&extractor_definition)
+      @evaluating_extractor_definition = false
+      
+      if @root_patterns.empty?
+        # TODO: this should be an exception
+        Scrubyt.log :ERROR, 'No extractor defined, exiting...'
+        exit
+      end
+      
+      #Once all is set up, evaluate the extractor from the root pattern!
+      root_results = evaluate_extractor
+      FetchAction.close_firefox if @mode.is_a?(Hash) && @mode[:close]
+
+      
+      @result = ScrubytResult.new('root')
+      @result.push(*@root_results)
+      @result.root_patterns = @root_patterns
+      @result.source_file = source_file
+      @result.source_proc = extractor_definition
+      
+      #Return the root pattern
+      Scrubyt.log :INFO, 'Extraction finished succesfully!'
+    end
+    
+    def get_hpricot_doc
+      FetchAction.get_hpricot_doc
+    end
+    
+    def get_current_doc_url
+      FetchAction.get_current_doc_url
+    end
+    
+    def get_detail_pattern_relations
+      @detail_pattern_relations
+    end
+    
+    def get_mode
+      @mode
+    end
+    
+    def get_original_host_name
+      @original_host_name
+    end
+    
+    def add_to_next_page_list(result_node)
+      if result_node.result.is_a? Hpricot::Elem
+        node = XPathUtils.find_nearest_node_with_attribute(result_node.result, 'href')
+        return if node == nil || node.attributes['href'] == nil
+        href = node.attributes['href'].gsub('&amp;') {'&'}
+      elsif result_node.result.is_a? String
+        href = result_node.result
+      end
+      url = href #TODO need absolute address here 1/4
+      @next_page_list << url
+    end
+    
+    def evaluate_extractor
+      @root_results ||= []
+      current_page_count = 1
+      catch :quit_next_page_loop do
+        loop do
+          url = get_current_doc_url #TODO need absolute address here 2/4
+          @processed_pages << url
+          @root_patterns.each do |root_pattern|
+            @root_results.push(*root_pattern.evaluate(get_hpricot_doc, nil))
+          end
+          
+          while @processed_pages.include? url #TODO need absolute address here 3/4
+            if !@next_page_pattern.nil?
+              throw :quit_next_page_loop if @next_page_pattern.options[:limit] == current_page_count
+              throw :quit_next_page_loop unless @next_page_pattern.filters[0].generate_XPath_for_example(true)
+              xpath = @next_page_pattern.filters[0].xpath
+              node = (get_hpricot_doc/xpath).map.last
+              node = XPathUtils.find_nearest_node_with_attribute(node, 'href')
+              throw :quit_next_page_loop if node == nil || node.attributes['href'] == nil
+              href = node.attributes['href'].gsub('&amp;') {'&'}
+              throw :quit_next_page_loop if href == nil
+              url = href #TODO need absolute address here 4/4
+            else
+              throw :quit_next_page_loop if @next_page_list.empty?
+              url = @next_page_list.pop
+            end
+          end
+
+          restore_host_name
+          FetchAction.fetch(url)
+          
+          current_page_count += 1
+        end
+      end
+      @root_patterns = []
+      @root_results
+    end
+    
+  end
+end

data/lib/scrubyt/logging.rb

@@ -0,0 +1,154 @@
+#
+# TODO: if multiline messages aren't needed, then remove them.
+#
+# TODO: switch to the conventional Ruby logger interface,
+# or create an adapter to it. If the former, then decided what to
+# do with the unit tests.
+#
+
+module Scrubyt
+  # Logging is disabled by default. It can be enabled as follows:
+  #
+  #   Scrubyt.logger = Scrubyt::Logger.new  # logs *all* messages to STDERR
+  #
+  def self.logger=(logger)
+    @logger = logger
+  end
+
+  # Simple logger implementation, based on Scrubyt's original logging style.
+  # Messages will be sent to STDERR. Logging can be limited to certain message
+  # levels by specifying them on initialization, e.g.
+  #
+  #   Scrubyt::Logger.new(:ACTION, :ERROR)  # will only log action/error messages
+  #
+  class Logger
+    class Message
+      def initialize(level, text)
+        @level, @text = level.to_s, text.to_s
+      end
+
+      def to_s
+        prefix + @text
+      end
+
+      protected
+
+      def prefix
+        @prefix ||= "[#{@level}] "
+      end
+    end
+
+    class MultiLineMessage < Message
+      def initialize(level, lines)
+        super level, lines.shift
+
+        @lines = lines
+      end
+
+      def to_s
+        [ super, indented_lines ] * "\n"
+      end
+
+      private
+
+      def indented_lines
+        @lines.inject([]) { |lines, line| lines << indented(line) } * "\n"
+      end
+
+      def indented(line)
+        ' ' * prefix.length + line
+      end
+    end
+
+    def initialize(*levels)
+      @levels = levels
+    end
+
+    def log(level, message)
+      return unless logging?(level)
+
+      message_class = message.is_a?(Array) ? MultiLineMessage : Message
+
+      output_stream.puts message_class.new(level, message)
+    end
+
+    def output_stream
+      @output_stream || STDERR
+    end
+
+    attr_writer :output_stream
+
+    private
+
+    def logging?(level)
+      @levels.empty? || @levels.include?(level)
+    end
+  end
+
+  def self.log(level, message)
+    return if logger.nil?
+
+    logger.log(level, message)
+  end
+
+  private
+
+  def self.logger
+    @logger
+  end
+end
+
+
+if __FILE__ == $0 then
+
+  require 'test/unit'
+
+  class ScrubytLoggingTestCase < Test::Unit::TestCase
+    class FauxOutputStream < Array
+      def puts(object)
+        self << object.to_s
+      end
+    end
+
+    def setup_logger_with_faux_output_stream!(*logger_args)
+      @stream = FauxOutputStream.new
+      logger = Scrubyt::Logger.new(*logger_args)
+      logger.output_stream = @stream
+      Scrubyt.logger = logger
+    end
+
+    def test_that_logging_works_with_nil_logger
+      Scrubyt.logger = nil
+      assert_nothing_raised { Scrubyt.log(:ERROR, 'message') }
+    end
+
+    def test_simple_messages_are_output_correctly
+      setup_logger_with_faux_output_stream!
+
+      Scrubyt.log :ACTION, 'i just did something'
+
+      assert_equal 1, @stream.size
+      assert_equal '[ACTION] i just did something', @stream.first
+    end
+
+    def test_that_multiline_messages_are_output_correctly
+      setup_logger_with_faux_output_stream!
+
+      Scrubyt.log :ERROR, ['something bad happened', 'dear oh dear']
+
+      assert_equal 1, @stream.size
+      assert_equal "[ERROR] something bad happened\n        dear oh dear", @stream.first
+    end
+
+    def test_that_loggers_can_be_limited_to_specfied_message_levels
+      setup_logger_with_faux_output_stream! :ERROR
+
+      Scrubyt.log :ACTION, 'i just did something'
+      Scrubyt.log :ERROR, 'something bad happened'
+
+      assert_equal 1, @stream.size
+      assert_equal '[ERROR] something bad happened', @stream.first
+    end
+  end
+
+end

data/lib/scrubyt/output/post_processor.rb

@@ -0,0 +1,139 @@
+module Scrubyt
+
+########################################## NOT USED ANY MORE ##########################################
+require 'set'
+##
+#=<tt>Post processing results after the extraction</tt>
+#Some things can not be carried out during evaluation - for example 
+#the ensure_presence_of_pattern constraint (since the evaluation is top
+#to bottom, at a given point we don't know yet whether the currently
+#evaluated pattern will have a child pattern or not) or removing unneeded
+#results caused by evaluating multiple filters.
+#
+#The sole purpose of this class is to execute these post-processing tasks.
+  class PostProcessor
+    ##
+    #This is just a convenience method do call all the postprocessing 
+    #functionality and checks
+    def self.apply_post_processing(root_pattern)
+      ensure_presence_of_pattern_full(root_pattern)      
+      remove_multiple_filter_duplicates(root_pattern) if root_pattern.children[0].filters.size > 1
+      report_if_no_results(root_pattern) if root_pattern.evaluation_context.extractor.get_mode != :production
+    end
+  
+    ##
+    #Apply the ensure_presence_of_pattern constraint on 
+    #the full extractor
+    def self.ensure_presence_of_pattern_full(pattern)
+      ensure_presence_of_pattern(pattern)
+      pattern.children.each {|child| ensure_presence_of_pattern_full(child)}
+    end  
+  
+    ##
+    #Remove unneeded results of a pattern (caused by evaluating multiple filters)
+    #See for example the B&N scenario - the book titles are extracted two times
+    #for every pattern (since both examples generate the same XPath for them)
+    #but since always only one of the results has a price, the other is discarded
+    def self.remove_multiple_filter_duplicates(pattern)
+      remove_multiple_filter_duplicates_intern(pattern) if pattern.parent_of_leaf
+      pattern.children.each {|child| remove_multiple_filter_duplicates(child)}
+    end
+    
+    ##
+    #Issue an error report if the document did not extract anything.
+    #Probably this is because the structure of the page changed or 
+    #because of some rather nasty bug - in any case, something wrong 
+    #is going on, and we need to inform the user about this!
+    def self.report_if_no_results(root_pattern)
+      results_found = false
+      root_pattern.children.each {|child| return if (child.result.childmap.size > 0)}
+      
+      Scrubyt.log :WARNING, [
+        "The extractor did not find any result instances. Most probably this is wrong.",
+        "Check your extractor and if you are sure it should work, report a bug!"
+      ]
+    end
+  
+private
+    def self.ensure_presence_of_pattern(pattern)
+      #holds the name of those child patterns which have to be present as children of the input parameter  
+      epop_names = pattern.constraints.select {|c| c.type == Scrubyt::Constraint::CONSTRAINT_TYPE_ENSURE_PRESENCE_OF_PATTERN}.map {|c| c.target}
+      return if epop_names.empty?
+      #all_parent_values holds instances extracted by pattern    
+      all_parent_values = []
+      pattern.result.childmap.each { |h| all_parent_values << h.values }
+      all_parent_values.flatten!
+      #indices of result instances (of pattern) we are going to remove
+      results_to_remove = Set.new
+      pattern.children.each do |child_pattern|
+        #all_child_values holds instances extracted by child_pattern
+        all_child_values = []
+        child_pattern.result.childmap.each { |h| all_child_values << h.values }
+        all_child_values.flatten!
+      
+        #populate results_to_remove
+        i = 0      
+        all_parent_values.each do |parent_value|
+          #Hey! Not just the direct children but all the ancestors
+          @found_ancestor = false
+          check_ancestors(parent_value, all_child_values)
+          
+          results_to_remove << i if (!@found_ancestor && (epop_names.include? child_pattern.name))
+          i += 1
+        end
+      end   
+      #based on results_to_remove, populate the array 'rejected' which holds the actual instances
+      #(and not indices, as in the case of results_to_remove!). In other words, we are mapping 
+      #results_to_remove indices to their actual instances
+      rejected = []    
+      i = -1 
+      pattern.result.childmap.each do |h|
+        h.each { |k,v| rejected = v.reject {|e| i += 1; !results_to_remove.include? i } }
+      end    
+      
+      #Finally, do the actual delete!
+      pattern.result.childmap.each { |h| h.each { |k,v| rejected.each  { |r| v.delete(r)} } }    
+    end
+    
+    def self.check_ancestors(parent_value, all_child_values)
+      parent_value.children.each { |child| @found_ancestor = true if all_child_values.include? child } if
+parent_value.is_a? Hpricot::Elem
+      parent_value.children.each { |child| check_ancestors(child, all_child_values) if child.is_a? Hpricot::Elem } if parent_value.is_a? Hpricot::Elem    
+    end
+  
+    def self.remove_multiple_filter_duplicates_intern(pattern)
+      possible_duplicates = {}
+      longest_result = 0
+      pattern.result.childmap.each { |r|
+        r.each do |k,v|
+          v.each do |x|
+            all_child_results = []
+            pattern.children.each { |child| 
+              temp_res = child.result.lookup(x) 
+              all_child_results << temp_res if temp_res != nil
+            }
+            next if all_child_results.size <= 1
+            longest_result = all_child_results.map {|e| e.size}.max
+            all_child_results.each { |r| (r.size+1).upto(longest_result) { r << nil } }
+            possible_duplicates[x] = all_child_results.transpose
+          end            
+        end
+      }
+      #Determine the 'real' duplicates
+      real_duplicates = {}
+      possible_duplicates.each { |k,v|
+        next if v.size == 1
+        v.each { |r| real_duplicates[k] = r }
+      }
+      
+      #Finally, remove them!
+      pattern.children.each { |child| 
+        child.result.childmap.each { |r|
+          r.each { |k,v|     
+           real_duplicates[k].each {|e| v.delete e} if real_duplicates.keys.include? k
+          }
+        }
+      }
+    end #end of function
+  end #end of class PostProcessor
+end #end of module Scrubyt

data/lib/scrubyt/output/result.rb

@@ -0,0 +1,44 @@
+########################################## NOT USED ANY MORE ##########################################
+module Scrubyt
+  ##
+  #=<tt>Represents the results of a pattern</tt>
+  class Result
+    attr_reader :childmap, :instances
+
+    def initialize
+      @childmap ||= []
+    end
+
+    def add_result(source, result)
+      @childmap.each do |hash|
+        if hash.keys[0] == source
+          hash[source] << result if !hash[source].include? result
+          return
+        end
+      end
+      @childmap << {source => [result]}
+    end
+
+    def lookup(last_result)
+      @childmap.each do |hashes|
+        hashes.each { |key, value| return value if (key == last_result) }
+      end
+      nil
+    end#end of method lookup
+  end#end of class Result
+end#end of module Scrubyt
+
+  #It roughly works like this:
+  #
+  # root
+  # source:         nil
+  # childmap:       [ {doc1 => [doc1]}, {doc2 => [doc2]} ]
+
+  #table
+  #  source:         doc1
+  #  childmap        [ {doc1 => [table[1]s1, table[2]s1, table[3]s1]}, {doc2 => [table[1]s2, table[2]s2, table[3]s2]} ]
+
+  #row
+  #  source:         table1s1, table2s1, table3s1
+  #  childmap:       [ {table[1]s1 => [row1s1, row2s1]}, {table[2]s1 => [row3s1, row3s1, row5s1]},
+  #                    {table[1]s2 => [row1s2, row2s2]}, {table[2]s2 => [row3s2, row3s2, row5s2]}]

data/lib/scrubyt/output/result_dumper.rb

@@ -0,0 +1,154 @@
+require 'rexml/document'
+require 'rexml/xpath'
+
+########################################## NOT USED ANY MORE ##########################################
+module Scrubyt
+  ##
+  #=<tt>Dumping the result in various formats and providing statistics on the results</tt>
+  class ResultDumper
+    ##
+    #Output the results as XML
+    def self.to_xml(pattern)
+      doc = REXML::Document.new
+      root = REXML::Element.new('root')
+      doc.add_element(root)
+      all_extracted_docs = pattern.last_result
+      [all_extracted_docs].flatten.each do |lr|
+        pattern.last_result = lr
+        to_xml_recursive(pattern, root)
+      end
+      remove_empty_leaves(doc)
+      @@last_doc = doc
+    end
+
+    def self.remove_empty_leaves(node)
+      node.remove if  node.elements.empty? && node.text == nil
+      node.elements.each {|child| remove_empty_leaves child }
+    end
+
+    ##
+    #Output the text of the pattern; If this pattern is a tree, collect the text from its
+    #result instance node; otherwise rely on the last_result
+    #TODO: throw this away!!!
+    def self.to_text(pattern)
+      last_result = pattern.last_result
+      result = ""
+      if pattern.type == :tree
+        last_result.traverse_text { |t| result += t.to_s }
+      else
+        result = last_result
+      end
+      result
+    end
+
+    def self.to_csv(pattern)
+      result = []
+      flat_csv_inner = lambda {|e, parts|
+        content = e.text || ''
+        parts << content if ((e.is_a? REXML::Element) && content != '')
+        e.children.each {|c| flat_csv_inner.call(c, parts) if c.is_a? REXML::Element }
+        parts
+      }
+      to_xml(pattern).root.elements['/root'].each {|e| result << flat_csv_inner.call(e, []) }
+      (result.map! {|a| a.join(',')}).join("\n")
+    end
+
+    def self.to_hash(pattern)
+      result = []
+      flat_hash_inner = lambda {|e, parts|
+        content = e.text ? REXML::Text.unnormalize(e.text) : ''
+        if ((e.is_a? REXML::Element) && content != '')
+          if parts[e.local_name]
+            parts[e.local_name] = parts[e.local_name] + "," + content
+          else
+            parts[e.local_name] = content
+          end
+        end
+        e.children.each {|c| flat_hash_inner.call(c, parts) if c.is_a? REXML::Element }
+        parts
+      }
+      to_xml(pattern).root.elements['/root'].each {|e| result << flat_hash_inner.call(e, {}) }
+      result
+    end
+
+
+
+    ##
+    #Print some simple statistics on the extracted results, like the count of extracted
+    #instances by each pattern
+    def self.print_statistics(pattern)
+      puts "\n" * 2
+      print_statistics_recursive(pattern,0)
+      puts
+    end
+
+private
+    def self.to_xml_recursive(pattern, element)
+      pattern.children.each do |child|
+        childresults = child.result.lookup(child.parent.last_result)
+        #Output text for leaf nodes only; Maybe add possibility to customize this later
+        if (childresults == nil)
+##TODO: is this needed for anything? I guess not! Drop it!!!!!!
+#Update: it seems the blackbox tests are not passing because of this (?) so temporarily adding it back
+##=begin
+          res = ""
+          if child.parent.last_result.is_a? String
+            res = child.parent.last_result
+          else
+            child.parent.last_result.traverse_text { |t| res += t.to_s }
+          end
+          if (child.parent.respond_to?(:size) && child.parent.size == 0) #TODO: respond_to should not be used here, it's just a quick workaround
+            element.text = SharedUtils.unescape_entities(res).strip unless element.parent.is_a? REXML::Document
+          end
+          next
+##=end
+        end
+
+        generate_children(child, childresults, element)
+      end
+    end
+
+    def self.generate_children(child, childresults, element)
+      if childresults == nil
+        child_node = REXML::Element.new(child.name)
+        child_node.text = child.default
+        element.add_element(child_node)
+      else
+        childresults.size.times do |num|
+          child.last_result = childresults[num]
+          res = ""
+          if child.last_result.instance_of? String
+            res = child.last_result
+          else
+            if child.last_result.respond_to? 'traverse_text'
+              child.last_result.traverse_text { |t| res += t.to_s } if child.last_result != nil
+            else
+              child.last_result.children.each { |c| element.add_element c }
+            end
+          end
+          child_node = REXML::Element.new(child.name)
+          child_node.text = SharedUtils.unescape_entities(res).strip if child.write_text
+          element.add_element(child_node) if (child.type != :detail_page && child_node.text != '')
+          to_xml_recursive(child, child_node)
+        end
+      end
+    end
+
+    def self.print_statistics_recursive(pattern, depth)
+      if pattern.name != 'root'
+        if pattern.type == :detail_page
+          pattern.evaluation_context.extractor.get_detail_pattern_relations[pattern].parent.children.each do |child|
+            print_statistics_recursive(child, depth)
+          end
+        else
+          count = REXML::XPath.match(@@last_doc, "//#{pattern.name}").size
+          Scrubyt.log :INFO, (' ' * depth.to_i) + "#{pattern.name} extracted #{count} instances."
+        end
+      end
+
+      pattern.children.each do |child|
+        print_statistics_recursive(child, depth + 4)
+      end
+      end#end of method print_statistics_recursive
+    end #end of class ResultDumper
+  end #end of module Scrubyt