scrubber-scrubyt 0.4.11

data/lib/scrubyt/logging.rb

@@ -0,0 +1,154 @@
+#
+# TODO: if multiline messages aren't needed, then remove them.
+#
+# TODO: switch to the conventional Ruby logger interface,
+# or create an adapter to it. If the former, then decided what to
+# do with the unit tests.
+#
+
+module Scrubyt
+  # Logging is disabled by default. It can be enabled as follows:
+  #
+  #   Scrubyt.logger = Scrubyt::Logger.new  # logs *all* messages to STDERR
+  #
+  def self.logger=(logger)
+    @logger = logger
+  end
+
+  # Simple logger implementation, based on Scrubyt's original logging style.
+  # Messages will be sent to STDERR. Logging can be limited to certain message
+  # levels by specifying them on initialization, e.g.
+  #
+  #   Scrubyt::Logger.new(:ACTION, :ERROR)  # will only log action/error messages
+  #
+  class Logger
+    class Message
+      def initialize(level, text)
+        @level, @text = level.to_s, text.to_s
+      end
+
+      def to_s
+        prefix + @text
+      end
+
+      protected
+
+      def prefix
+        @prefix ||= "[#{@level}] "
+      end
+    end
+
+    class MultiLineMessage < Message
+      def initialize(level, lines)
+        super level, lines.shift
+
+        @lines = lines
+      end
+
+      def to_s
+        [ super, indented_lines ] * "\n"
+      end
+
+      private
+
+      def indented_lines
+        @lines.inject([]) { |lines, line| lines << indented(line) } * "\n"
+      end
+
+      def indented(line)
+        ' ' * prefix.length + line
+      end
+    end
+
+    def initialize(*levels)
+      @levels = levels
+    end
+
+    def log(level, message)
+      return unless logging?(level)
+
+      message_class = message.is_a?(Array) ? MultiLineMessage : Message
+
+      output_stream.puts message_class.new(level, message)
+    end
+
+    def output_stream
+      @output_stream || STDERR
+    end
+
+    attr_writer :output_stream
+
+    private
+
+    def logging?(level)
+      @levels.empty? || @levels.include?(level)
+    end
+  end
+
+  def self.log(level, message)
+    return if logger.nil?
+
+    logger.log(level, message)
+  end
+
+  private
+
+  def self.logger
+    @logger
+  end
+end
+
+
+if __FILE__ == $0 then
+
+  require 'test/unit'
+
+  class ScrubytLoggingTestCase < Test::Unit::TestCase
+    class FauxOutputStream < Array
+      def puts(object)
+        self << object.to_s
+      end
+    end
+
+    def setup_logger_with_faux_output_stream!(*logger_args)
+      @stream = FauxOutputStream.new
+      logger = Scrubyt::Logger.new(*logger_args)
+      logger.output_stream = @stream
+      Scrubyt.logger = logger
+    end
+
+    def test_that_logging_works_with_nil_logger
+      Scrubyt.logger = nil
+      assert_nothing_raised { Scrubyt.log(:ERROR, 'message') }
+    end
+
+    def test_simple_messages_are_output_correctly
+      setup_logger_with_faux_output_stream!
+
+      Scrubyt.log :ACTION, 'i just did something'
+
+      assert_equal 1, @stream.size
+      assert_equal '[ACTION] i just did something', @stream.first
+    end
+
+    def test_that_multiline_messages_are_output_correctly
+      setup_logger_with_faux_output_stream!
+
+      Scrubyt.log :ERROR, ['something bad happened', 'dear oh dear']
+
+      assert_equal 1, @stream.size
+      assert_equal "[ERROR] something bad happened\n        dear oh dear", @stream.first
+    end
+
+    def test_that_loggers_can_be_limited_to_specfied_message_levels
+      setup_logger_with_faux_output_stream! :ERROR
+
+      Scrubyt.log :ACTION, 'i just did something'
+      Scrubyt.log :ERROR, 'something bad happened'
+
+      assert_equal 1, @stream.size
+      assert_equal '[ERROR] something bad happened', @stream.first
+    end
+  end
+
+end

data/lib/scrubyt/output/post_processor.rb

@@ -0,0 +1,139 @@
+module Scrubyt
+
+########################################## NOT USED ANY MORE ##########################################
+require 'set'
+##
+#=<tt>Post processing results after the extraction</tt>
+#Some things can not be carried out during evaluation - for example 
+#the ensure_presence_of_pattern constraint (since the evaluation is top
+#to bottom, at a given point we don't know yet whether the currently
+#evaluated pattern will have a child pattern or not) or removing unneeded
+#results caused by evaluating multiple filters.
+#
+#The sole purpose of this class is to execute these post-processing tasks.
+  class PostProcessor
+    ##
+    #This is just a convenience method do call all the postprocessing 
+    #functionality and checks
+    def self.apply_post_processing(root_pattern)
+      ensure_presence_of_pattern_full(root_pattern)      
+      remove_multiple_filter_duplicates(root_pattern) if root_pattern.children[0].filters.size > 1
+      report_if_no_results(root_pattern) if root_pattern.evaluation_context.extractor.get_mode != :production
+    end
+  
+    ##
+    #Apply the ensure_presence_of_pattern constraint on 
+    #the full extractor
+    def self.ensure_presence_of_pattern_full(pattern)
+      ensure_presence_of_pattern(pattern)
+      pattern.children.each {|child| ensure_presence_of_pattern_full(child)}
+    end  
+  
+    ##
+    #Remove unneeded results of a pattern (caused by evaluating multiple filters)
+    #See for example the B&N scenario - the book titles are extracted two times
+    #for every pattern (since both examples generate the same XPath for them)
+    #but since always only one of the results has a price, the other is discarded
+    def self.remove_multiple_filter_duplicates(pattern)
+      remove_multiple_filter_duplicates_intern(pattern) if pattern.parent_of_leaf
+      pattern.children.each {|child| remove_multiple_filter_duplicates(child)}
+    end
+    
+    ##
+    #Issue an error report if the document did not extract anything.
+    #Probably this is because the structure of the page changed or 
+    #because of some rather nasty bug - in any case, something wrong 
+    #is going on, and we need to inform the user about this!
+    def self.report_if_no_results(root_pattern)
+      results_found = false
+      root_pattern.children.each {|child| return if (child.result.childmap.size > 0)}
+      
+      Scrubyt.log :WARNING, [
+        "The extractor did not find any result instances. Most probably this is wrong.",
+        "Check your extractor and if you are sure it should work, report a bug!"
+      ]
+    end
+  
+private
+    def self.ensure_presence_of_pattern(pattern)
+      #holds the name of those child patterns which have to be present as children of the input parameter  
+      epop_names = pattern.constraints.select {|c| c.type == Scrubyt::Constraint::CONSTRAINT_TYPE_ENSURE_PRESENCE_OF_PATTERN}.map {|c| c.target}
+      return if epop_names.empty?
+      #all_parent_values holds instances extracted by pattern    
+      all_parent_values = []
+      pattern.result.childmap.each { |h| all_parent_values << h.values }
+      all_parent_values.flatten!
+      #indices of result instances (of pattern) we are going to remove
+      results_to_remove = Set.new
+      pattern.children.each do |child_pattern|
+        #all_child_values holds instances extracted by child_pattern
+        all_child_values = []
+        child_pattern.result.childmap.each { |h| all_child_values << h.values }
+        all_child_values.flatten!
+      
+        #populate results_to_remove
+        i = 0      
+        all_parent_values.each do |parent_value|
+          #Hey! Not just the direct children but all the ancestors
+          @found_ancestor = false
+          check_ancestors(parent_value, all_child_values)
+          
+          results_to_remove << i if (!@found_ancestor && (epop_names.include? child_pattern.name))
+          i += 1
+        end
+      end   
+      #based on results_to_remove, populate the array 'rejected' which holds the actual instances
+      #(and not indices, as in the case of results_to_remove!). In other words, we are mapping 
+      #results_to_remove indices to their actual instances
+      rejected = []    
+      i = -1 
+      pattern.result.childmap.each do |h|
+        h.each { |k,v| rejected = v.reject {|e| i += 1; !results_to_remove.include? i } }
+      end    
+      
+      #Finally, do the actual delete!
+      pattern.result.childmap.each { |h| h.each { |k,v| rejected.each  { |r| v.delete(r)} } }    
+    end
+    
+    def self.check_ancestors(parent_value, all_child_values)
+      parent_value.children.each { |child| @found_ancestor = true if all_child_values.include? child } if
+parent_value.is_a? Hpricot::Elem
+      parent_value.children.each { |child| check_ancestors(child, all_child_values) if child.is_a? Hpricot::Elem } if parent_value.is_a? Hpricot::Elem    
+    end
+  
+    def self.remove_multiple_filter_duplicates_intern(pattern)
+      possible_duplicates = {}
+      longest_result = 0
+      pattern.result.childmap.each { |r|
+        r.each do |k,v|
+          v.each do |x|
+            all_child_results = []
+            pattern.children.each { |child| 
+              temp_res = child.result.lookup(x) 
+              all_child_results << temp_res if temp_res != nil
+            }
+            next if all_child_results.size <= 1
+            longest_result = all_child_results.map {|e| e.size}.max
+            all_child_results.each { |r| (r.size+1).upto(longest_result) { r << nil } }
+            possible_duplicates[x] = all_child_results.transpose
+          end            
+        end
+      }
+      #Determine the 'real' duplicates
+      real_duplicates = {}
+      possible_duplicates.each { |k,v|
+        next if v.size == 1
+        v.each { |r| real_duplicates[k] = r }
+      }
+      
+      #Finally, remove them!
+      pattern.children.each { |child| 
+        child.result.childmap.each { |r|
+          r.each { |k,v|     
+           real_duplicates[k].each {|e| v.delete e} if real_duplicates.keys.include? k
+          }
+        }
+      }
+    end #end of function
+  end #end of class PostProcessor
+end #end of module Scrubyt

data/lib/scrubyt/output/result.rb

@@ -0,0 +1,44 @@
+########################################## NOT USED ANY MORE ##########################################
+module Scrubyt
+  ##
+  #=<tt>Represents the results of a pattern</tt>
+  class Result
+    attr_reader :childmap, :instances
+
+    def initialize
+      @childmap ||= []
+    end
+
+    def add_result(source, result)
+      @childmap.each do |hash|
+        if hash.keys[0] == source
+          hash[source] << result if !hash[source].include? result
+          return
+        end
+      end
+      @childmap << {source => [result]}
+    end
+
+    def lookup(last_result)
+      @childmap.each do |hashes|
+        hashes.each { |key, value| return value if (key == last_result) }
+      end
+      nil
+    end#end of method lookup
+  end#end of class Result
+end#end of module Scrubyt
+
+  #It roughly works like this:
+  #
+  # root
+  # source:         nil
+  # childmap:       [ {doc1 => [doc1]}, {doc2 => [doc2]} ]
+
+  #table
+  #  source:         doc1
+  #  childmap        [ {doc1 => [table[1]s1, table[2]s1, table[3]s1]}, {doc2 => [table[1]s2, table[2]s2, table[3]s2]} ]
+
+  #row
+  #  source:         table1s1, table2s1, table3s1
+  #  childmap:       [ {table[1]s1 => [row1s1, row2s1]}, {table[2]s1 => [row3s1, row3s1, row5s1]},
+  #                    {table[1]s2 => [row1s2, row2s2]}, {table[2]s2 => [row3s2, row3s2, row5s2]}]

data/lib/scrubyt/output/result_dumper.rb

@@ -0,0 +1,154 @@
+require 'rexml/document'
+require 'rexml/xpath'
+
+########################################## NOT USED ANY MORE ##########################################
+module Scrubyt
+  ##
+  #=<tt>Dumping the result in various formats and providing statistics on the results</tt>
+  class ResultDumper
+    ##
+    #Output the results as XML
+    def self.to_xml(pattern)
+      doc = REXML::Document.new
+      root = REXML::Element.new('root')
+      doc.add_element(root)
+      all_extracted_docs = pattern.last_result
+      [all_extracted_docs].flatten.each do |lr|
+        pattern.last_result = lr
+        to_xml_recursive(pattern, root)
+      end
+      remove_empty_leaves(doc)
+      @@last_doc = doc
+    end
+
+    def self.remove_empty_leaves(node)
+      node.remove if  node.elements.empty? && node.text == nil
+      node.elements.each {|child| remove_empty_leaves child }
+    end
+
+    ##
+    #Output the text of the pattern; If this pattern is a tree, collect the text from its
+    #result instance node; otherwise rely on the last_result
+    #TODO: throw this away!!!
+    def self.to_text(pattern)
+      last_result = pattern.last_result
+      result = ""
+      if pattern.type == :tree
+        last_result.traverse_text { |t| result += t.to_s }
+      else
+        result = last_result
+      end
+      result
+    end
+
+    def self.to_csv(pattern)
+      result = []
+      flat_csv_inner = lambda {|e, parts|
+        content = e.text || ''
+        parts << content if ((e.is_a? REXML::Element) && content != '')
+        e.children.each {|c| flat_csv_inner.call(c, parts) if c.is_a? REXML::Element }
+        parts
+      }
+      to_xml(pattern).root.elements['/root'].each {|e| result << flat_csv_inner.call(e, []) }
+      (result.map! {|a| a.join(',')}).join("\n")
+    end
+
+    def self.to_hash(pattern)
+      result = []
+      flat_hash_inner = lambda {|e, parts|
+        content = e.text ? REXML::Text.unnormalize(e.text) : ''
+        if ((e.is_a? REXML::Element) && content != '')
+          if parts[e.local_name]
+            parts[e.local_name] = parts[e.local_name] + "," + content
+          else
+            parts[e.local_name] = content
+          end
+        end
+        e.children.each {|c| flat_hash_inner.call(c, parts) if c.is_a? REXML::Element }
+        parts
+      }
+      to_xml(pattern).root.elements['/root'].each {|e| result << flat_hash_inner.call(e, {}) }
+      result
+    end
+
+
+
+    ##
+    #Print some simple statistics on the extracted results, like the count of extracted
+    #instances by each pattern
+    def self.print_statistics(pattern)
+      puts "\n" * 2
+      print_statistics_recursive(pattern,0)
+      puts
+    end
+
+private
+    def self.to_xml_recursive(pattern, element)
+      pattern.children.each do |child|
+        childresults = child.result.lookup(child.parent.last_result)
+        #Output text for leaf nodes only; Maybe add possibility to customize this later
+        if (childresults == nil)
+##TODO: is this needed for anything? I guess not! Drop it!!!!!!
+#Update: it seems the blackbox tests are not passing because of this (?) so temporarily adding it back
+##=begin
+          res = ""
+          if child.parent.last_result.is_a? String
+            res = child.parent.last_result
+          else
+            child.parent.last_result.traverse_text { |t| res += t.to_s }
+          end
+          if (child.parent.respond_to?(:size) && child.parent.size == 0) #TODO: respond_to should not be used here, it's just a quick workaround
+            element.text = SharedUtils.unescape_entities(res).strip unless element.parent.is_a? REXML::Document
+          end
+          next
+##=end
+        end
+
+        generate_children(child, childresults, element)
+      end
+    end
+
+    def self.generate_children(child, childresults, element)
+      if childresults == nil
+        child_node = REXML::Element.new(child.name)
+        child_node.text = child.default
+        element.add_element(child_node)
+      else
+        childresults.size.times do |num|
+          child.last_result = childresults[num]
+          res = ""
+          if child.last_result.instance_of? String
+            res = child.last_result
+          else
+            if child.last_result.respond_to? 'traverse_text'
+              child.last_result.traverse_text { |t| res += t.to_s } if child.last_result != nil
+            else
+              child.last_result.children.each { |c| element.add_element c }
+            end
+          end
+          child_node = REXML::Element.new(child.name)
+          child_node.text = SharedUtils.unescape_entities(res).strip if child.write_text
+          element.add_element(child_node) if (child.type != :detail_page && child_node.text != '')
+          to_xml_recursive(child, child_node)
+        end
+      end
+    end
+
+    def self.print_statistics_recursive(pattern, depth)
+      if pattern.name != 'root'
+        if pattern.type == :detail_page
+          pattern.evaluation_context.extractor.get_detail_pattern_relations[pattern].parent.children.each do |child|
+            print_statistics_recursive(child, depth)
+          end
+        else
+          count = REXML::XPath.match(@@last_doc, "//#{pattern.name}").size
+          Scrubyt.log :INFO, (' ' * depth.to_i) + "#{pattern.name} extracted #{count} instances."
+        end
+      end
+
+      pattern.children.each do |child|
+        print_statistics_recursive(child, depth + 4)
+      end
+      end#end of method print_statistics_recursive
+    end #end of class ResultDumper
+  end #end of module Scrubyt

data/lib/scrubyt/output/result_node.rb

@@ -0,0 +1,140 @@
+module Scrubyt
+  class ResultNode < Array
+    OUTPUT_OPTIONS = [:write_text]
+
+    attr_accessor :name, :result, :options, :generated_by_leaf
+
+    def initialize(name, result=nil, options={})
+      @name = name
+      @result = result
+      @options = options
+    end
+
+    def write_text
+      @options[:write_text].nil? ? @generated_by_leaf : @options[:write_text]
+    end
+
+    def has_content?
+      return true if result.is_a? String
+      write_text || (inject(false) { |one_child_has_content, child| one_child_has_content || child.has_content? })
+    end
+
+    def to_s
+      text = (@result.is_a? String) ? @result : @result.inner_html.gsub(/<.*?>/, '')
+      text = SharedUtils.unescape_entities(text)
+      text.strip!
+      if (@options[:default] && ((text == '') || (text == @options[:default])))
+        @options[:default]
+      else
+        text
+      end
+    end
+
+    def to_libxml
+      libxml_node = XML::Node.new(name)
+      self.each { |child| libxml_node << child.to_libxml if child.has_content? }
+      libxml_node << to_s if write_text
+      libxml_node
+    end
+
+    #note: see ruby_extensions.rb for String#write
+    def to_xml
+      to_xml_lines.join("\n")
+    end
+
+    def to_hash(delimiter=',')
+      result = []
+      flat_hash_inner = lambda {|e, hash|
+        hash[e.name.to_sym] = hash[e.name.to_sym] ? hash[e.name.to_sym] + delimiter + e.to_s : e.to_s  if ((e.write_text && !e.to_s.empty?) || e.options[:default])
+        e.each {|c| flat_hash_inner.call(c, hash)  }
+        hash
+      }
+      self.each {|e| result << flat_hash_inner.call(e, {}) }
+      result
+    end
+    
+    def to_flat_hash()
+      hash_result = self.to_hash('@@@@@@')
+      merged_hash = hash_result.delete_at 0
+      hash_result.each do |hash|
+        merged_hash.keys.each do |key|
+          merged_hash[key] += "@@@@@@#{hash[key]}"
+        end
+      end
+      result_sets = merged_hash.values.map!{|x| x.split('@@@@@@')}.transpose
+      final_result = []
+
+      result_sets.each do |rs|
+        temp_result = {}
+        merged_hash.keys.each do |k|
+          temp_result[k] = rs[merged_hash.keys.index(k)]
+        end
+        final_result << temp_result
+      end
+      final_result
+    end
+
+    def to_flat_xml(delimiter=nil)
+      lines = []
+      hash_result = delimiter ? self.to_hash(delimiter) : self.to_hash
+      merged_hash = hash_result.delete_at 0
+
+      hash_result.each do |hash|
+        merged_hash.keys.each do |key|
+          merged_hash[key] += "#{delimiter}#{hash[key]}"
+        end
+      end
+
+      if delimiter
+        result_sets = merged_hash.values.map!{|x| x.split(delimiter)}.transpose
+        final_result = []
+
+        result_sets.each do |rs|
+          temp_result = {}
+          merged_hash.keys.each do |k|
+            temp_result[k] = rs[merged_hash.keys.index(k)]
+          end
+          final_result << temp_result
+        end
+        hash_result = final_result
+      end
+
+      hash_result.each do |hash|
+        lines << "<item>"
+        hash.each do |key, value|
+          xml_tag = key.to_s
+          value = '' if value == '#empty#'
+          lines << "  <#{xml_tag}>#{REXML::Text.normalize(value)}</#{xml_tag}>"
+        end
+        lines << "</item>"
+      end
+      return lines.join("\n")
+
+    end
+
+    def to_xml_lines
+      lines = []
+      children = self.select{ |child| child.has_content? }
+      if children.empty?
+        if result.is_a? String
+          lines << "<#{name}>#{result}</#{name}>"
+        elsif write_text && !to_s.empty?
+          lines << "<#{name}>#{ERB::Util.html_escape(to_s)}</#{name}>"
+        else
+          if @options[:default]
+            lines << "<#{name}>#{@options[:default]}</#{name}>"
+          else
+            lines << "<#{name}/>"
+          end
+        end
+      else
+        lines << "<#{name}>"
+        lines << "  #{ERB::Util.html_escape(to_s)}" if write_text && !to_s.empty?
+        children.each do |child|
+          lines.push(*child.to_xml_lines.map{ |line| "  #{line}" })
+        end
+        lines << "</#{name}>"
+      end
+    end
+  end
+end

data/lib/scrubyt/output/scrubyt_result.rb

@@ -0,0 +1,42 @@
+module Scrubyt
+  class ScrubytResult < ResultNode
+    attr_accessor :root_patterns, :source_file, :source_proc
+
+    def export
+      #Temporary solution; the real one will be back later - or not     
+     result = <<-EXPLANATION
+     
+     === Extractor tree ===
+     
+     export() is not working at the moment, due to the removal or ParseTree, ruby2ruby and RubyInline.
+     For now, in case you are using examples, you can replace them by hand based on the output below.
+     So if your pattern in the learning extractor looks like
+     
+     book "Ruby Cookbook" 
+     
+     and you see the following below:
+     
+     [book] /table[1]/tr/td[2]
+     
+     then replace "Ruby Cookbook" with "/table[1]/tr/td[2]" (and all the other XPaths) and you are ready!
+     
+     EXPLANATION
+     
+     tree_builder = lambda do |node, level|        
+       result += current_level = ("   " * (level == 0 ? 0 : level-1) + 
+                                  "|\n" * (level == 0 ? 0 : 1) +
+                                  "   " * (level == 0 ? 0 : level-1) + 
+                                 "+-- " * (level == 0 ? 0 : 1) + 
+                                 "[#{node.name}]")
+       result += " #{node.filters[0].xpath}" if node.type == :tree
+       result += "\n"                                 
+                                 
+       node.children.each {|c| tree_builder[c, level+1]}
+     end
+     
+     tree_builder[root_patterns[0],0]
+          
+     result += "\n"
+    end
+  end
+end