scrubyt 0.3.0 → 0.3.4

data/lib/scrubyt/logging.rb

@@ -1,16 +1,154 @@
+#
+# TODO: if multiline messages aren't needed, then remove them.
+#
+# TODO: switch to the conventional Ruby logger interface,
+# or create an adapter to it. If the former, then decided what to
+# do with the unit tests.
+#
+
 module Scrubyt
-  def self.log(message_type, message)
+  # Logging is disabled by default. It can be enabled as follows:
+  #
+  #   Scrubyt.logger = Scrubyt::Logger.new  # logs *all* messages to STDERR
+  #
+  def self.logger=(logger)
+    @logger = logger
+  end
+
+  # Simple logger implementation, based on Scrubyt's original logging style.
+  # Messages will be sent to STDERR. Logging can be limited to certain message
+  # levels by specifying them on initialization, e.g.
+  #
+  #   Scrubyt::Logger.new(:ACTION, :ERROR)  # will only log action/error messages
+  #
+  class Logger
+    class Message
+      def initialize(level, text)
+        @level, @text = level.to_s, text.to_s
+      end
+
+      def to_s
+        prefix + @text
+      end
+
+      protected
+
+      def prefix
+        @prefix ||= "[#{@level}] "
+      end
+    end
+
+    class MultiLineMessage < Message
+      def initialize(level, lines)
+        super level, lines.shift
+
+        @lines = lines
+      end
+
+      def to_s
+        [ super, indented_lines ] * "\n"
+      end
+
+      private
+
+      def indented_lines
+        @lines.inject([]) { |lines, line| lines << indented(line) } * "\n"
+      end
+
+      def indented(line)
+        ' ' * prefix.length + line
+      end
+    end
+
+    def initialize(*levels)
+      @levels = levels
+    end
+
+    def log(level, message)
+      return unless logging?(level)
+
+      message_class = message.is_a?(Array) ? MultiLineMessage : Message
+
+      output_stream.puts message_class.new(level, message)
+    end
+
+    def output_stream
+      @output_stream || STDERR
+    end
 
-    pre = "[#{message_type}] "
+    attr_writer :output_stream
 
-    if message.is_a? Array
-      puts pre + message.first
-      message[1..-1].each do |line|
-        puts ' ' * pre.length + line
+    private
+
+    def logging?(level)
+      @levels.empty? || @levels.include?(level)
+    end
+  end
+
+  def self.log(level, message)
+    return if logger.nil?
+
+    logger.log(level, message)
+  end
+
+  private
+
+  def self.logger
+    @logger
+  end
+end
+
+
+if __FILE__ == $0 then
+
+  require 'test/unit'
+
+  class ScrubytLoggingTestCase < Test::Unit::TestCase
+    class FauxOutputStream < Array
+      def puts(object)
+        self << object.to_s
       end
-    else
-      puts pre + message.to_s
     end
 
+    def setup_logger_with_faux_output_stream!(*logger_args)
+      @stream = FauxOutputStream.new
+      logger = Scrubyt::Logger.new(*logger_args)
+      logger.output_stream = @stream
+      Scrubyt.logger = logger
+    end
+
+    def test_that_logging_works_with_nil_logger
+      Scrubyt.logger = nil
+      assert_nothing_raised { Scrubyt.log(:ERROR, 'message') }
+    end
+
+    def test_simple_messages_are_output_correctly
+      setup_logger_with_faux_output_stream!
+
+      Scrubyt.log :ACTION, 'i just did something'
+
+      assert_equal 1, @stream.size
+      assert_equal '[ACTION] i just did something', @stream.first
+    end
+
+    def test_that_multiline_messages_are_output_correctly
+      setup_logger_with_faux_output_stream!
+
+      Scrubyt.log :ERROR, ['something bad happened', 'dear oh dear']
+
+      assert_equal 1, @stream.size
+      assert_equal "[ERROR] something bad happened\n        dear oh dear", @stream.first
+    end
+
+    def test_that_loggers_can_be_limited_to_specfied_message_levels
+      setup_logger_with_faux_output_stream! :ERROR
+
+      Scrubyt.log :ACTION, 'i just did something'
+      Scrubyt.log :ERROR, 'something bad happened'
+
+      assert_equal 1, @stream.size
+      assert_equal '[ERROR] something bad happened', @stream.first
+    end
   end
+
 end

data/lib/scrubyt/output/export.rb

@@ -61,47 +61,70 @@ module Scrubyt
     #This snippet will export the extractor to a file named 'my_super_camera_extractor.rb'.
     #After running 'my_super_camera_extractor.rb', the result will be dumped to the file
     #'/home/peter/stuff/result.xml'.
-    def self.export(root_pattern, wrapper_name, output_file_name=nil, extractor_result_file_name=nil)
-      sexp = [:block]
-      sexp << export_header(wrapper_name)
-      sexp << export_extractor(root_pattern, wrapper_name)
-      sexp << export_footer(wrapper_name, extractor_result_file_name)
+    def self.export(root_patterns, data)
+      wrapper_name = data[:wrapper_name]
+      template = data[:template] || 'default'
+      output_file_name = data[:output_file_name]
+      extractor_result_file_name = data[:extractor_result_file_name]
       
-      result = RubyToRuby.new.process(sexp)
-      result.gsub! '"' + root_pattern.source_file + '"', '__FILE__'
-
-      output_file_name ||= "#{wrapper_name}_extractor_export.rb"
-      output_file = open(output_file_name, 'w')
-      output_file.write(result)      
-      output_file.close
-      result
-    end
+      case template
+      when 'default'
+        header = "require 'rubygems'\n"
+        header += "require 'scrubyt'\n\n"
+        header += "#{wrapper_name} = Scrubyt::Extractor.define do\n"
+        if extractor_result_file_name
+          footer = "\nend\n\n#{wrapper_name}.to_xml.write(open('#{extractor_result_file_name}', 'w'), 1)"
+        else
+          footer = "\nend\n\n#{wrapper_name}.to_xml.write($stdout, 1)"
+        end
+        indent = 1
+      when 'lambda'
+        header = "lambda do\n"
+        footer = "\nend"
+        indent = 1
+      when 'none'
+        header = ''
+        footer = ''
+        indent = 0
+      else
+        puts "Unknown template type: #{template}"
+        return
+      end
+      
+      header = data[:header] if data[:header]
+      footer = data[:footer] if data[:footer]
+      indent = data[:indent] if data[:indent]
 
-private
-    def self.create_sexp(code)
-      (ParseTree.new.parse_tree_for_string(code))[0]
-    end
+      sexp = [:block]
+      sexp << export_extractor(root_patterns, data[:source_proc], wrapper_name)
+      extractor = RubyToRuby.new.process(sexp).gsub('"' + data[:source_file] + '"', '__FILE__')
+      extractor = extractor.strip
+      extractor = extractor.split("\n").map{ |l| ('  ' * indent) + l }.join("\n")
 
-    def self.export_header(wrapper_name)
-      create_sexp "require 'rubygems'; require 'scrubyt'"
+      result = header + extractor + footer
+      
+      if output_file_name
+        open(output_file_name, 'w') do |file|
+          file.write(result)
+        end
+      end
+      
+      result
     end
     
-    def self.export_footer(wrapper_name, extractor_result_file_name)
-      if extractor_result_file_name
-        create_sexp "#{wrapper_name}.to_xml.write(open('result_of_exported_extractor.xml', 'w'), 1)"
-      else
-        create_sexp "#{wrapper_name}.to_xml.write($stdout, 1)"
-      end
+    private
+    def self.create_sexp(code)
+     (ParseTree.new.parse_tree_for_string(code))[0]
     end
     
-    def self.export_extractor(root_pattern, wrapper_name)
+    def self.export_extractor(root_patterns, source_proc, wrapper_name)
       # filter actions before and after pattern
       pre_pattern_sexp = []
       post_pattern_sexp = []
-      pattern_skipped = false
-      actions = ['next_page', *NavigationActions::KEYWORDS]
+      patterns_passed = false
+      actions = ['next_page', *NavigationActions.instance_methods]
       
-      root_pattern.source_proc.to_sexp[3][1..-1].each do |sexp|
+      source_proc.to_sexp[3][1..-1].each do |sexp|
         get_call = lambda { |sexp|
           if sexp[0] == :fcall
             return sexp[1].to_s
@@ -113,29 +136,22 @@ private
         }
         call = get_call.call(sexp)
         if(call.nil? || actions.index(call) != nil)
-          if !pattern_skipped
+          if !patterns_passed
             pre_pattern_sexp.push(sexp)
           else
             post_pattern_sexp.push(sexp)
           end
         else
-          raise "Second pattern tree found while exporting." if pattern_skipped
-          pattern_skipped = true
+          patterns_passed = true
         end
       end
-
-      # build extractor content
-      inner_block = [:block]
-      inner_block.push([:block, *pre_pattern_sexp])
-      inner_block.push([:block, export_pattern(root_pattern)])
-      inner_block.push([:block, *post_pattern_sexp])
       
       # build extractor
-      [:block, [:lasgn, wrapper_name, [:iter, [:call, [:colon2, [:const, :Scrubyt], :Extractor], :define], nil, inner_block]]]
-    end    
-    
-    def self.export_pattern(root_pattern)
-      root_pattern.to_sexp
+      sexp = [:block]
+      sexp.push([:block, *pre_pattern_sexp])
+      sexp.push([:block, *root_patterns.to_sexp_array])
+      sexp.push([:block, *post_pattern_sexp])
+      sexp
     end
   end
 end

data/lib/scrubyt/output/result_node.rb

@@ -20,7 +20,7 @@ module Scrubyt
     end
 
     def to_s
-      text = (@result.is_a? String) ? @result : @result.inner_text
+      text = (@result.is_a? String) ? @result : @result.inner_html.gsub(/<.*?>/, '')
       text = SharedUtils.unescape_entities(text)
       text.strip!
       text
@@ -38,10 +38,10 @@ module Scrubyt
       to_xml_lines.join("\n")
     end
 
-    def to_hash
+    def to_hash(delimiter=',')
       result = []
       flat_hash_inner = lambda {|e, hash|
-        hash[e.name.to_sym] = hash[e.name.to_sym] ? hash[e.name.to_sym] + "," + e.to_s : e.to_s  if e.write_text && !e.to_s.empty?
+        hash[e.name.to_sym] = hash[e.name.to_sym] ? hash[e.name.to_sym] + delimiter + e.to_s : e.to_s  if e.write_text && !e.to_s.empty?
         e.each {|c| flat_hash_inner.call(c, hash)  }
         hash
       }
@@ -49,6 +49,37 @@ module Scrubyt
       result
     end
 
+    def to_flat_xml(delimiter=nil)
+      lines = []
+      hash_result = delimiter ? self.to_hash(delimiter) : self.to_hash
+
+      if delimiter
+        result_sets = hash_result[0].values.map!{|x| x.split(delimiter)}.transpose
+        final_result = []
+
+        result_sets.each do |rs|
+          temp_result = {}
+          hash_result[0].keys.each do |k|
+            temp_result[k] = rs[hash_result[0].keys.index(k)]
+          end
+          final_result << temp_result
+        end
+        hash_result = final_result
+      end
+
+      hash_result.each do |hash|
+        lines << "<item>"
+        hash.each do |key, value|
+          xml_tag = key.to_s
+          value = '' if value == '#empty#'
+          lines << "  <#{xml_tag}>#{REXML::Text.normalize(value)}</#{xml_tag}>"
+        end
+        lines << "</item>"
+      end
+      return lines.join("\n")
+
+    end
+
     def to_xml_lines
       lines = []
       children = self.select{ |child| child.has_content? }

data/lib/scrubyt/output/scrubyt_result.rb

@@ -1,13 +1,16 @@
 module Scrubyt
   class ScrubytResult < ResultNode
-    attr_accessor :root_pattern
+    attr_accessor :root_patterns, :source_file, :source_proc
 
     def export(arg1, output_file_name=nil, extractor_result_file_name=nil)
-      #      require 'scrubyt/output/export_old'; Scrubyt::ExportOld.export(arg1, self, output_file_name, extractor_result_file_name) ; return
-      if File.exists? arg1
-        old_export(arg1, output_file_name, extractor_result_file_name)
+      if arg1.is_a? String
+        if File.exists? arg1
+          export_old1(arg1, output_file_name, extractor_result_file_name)
+        else
+          export_old2(arg1, output_file_name, extractor_result_file_name)
+        end
       else
-        new_export(arg1, output_file_name, extractor_result_file_name)
+        export_new(arg1)
       end
     end
 
@@ -15,14 +18,20 @@ module Scrubyt
       #Implement me...
     end
 
-    def old_export(input_file, output_file_name=nil, extractor_result_file_name=nil)
+    def export_old1(input_file, output_file_name=nil, extractor_result_file_name=nil)
       contents = open(input_file).read
       wrapper_name = contents.scan(/\s+(.+)\s+=.*Extractor\.define.*/)[0][0]
-      Scrubyt::Export.export(@root_pattern, wrapper_name, output_file_name, extractor_result_file_name)
+      export_old2(wrapper_name, output_file_name, extractor_result_file_name)
     end
 
-    def new_export(wrapper_name, output_file_name=nil, extractor_result_file_name=nil)
-      Scrubyt::Export.export(@root_pattern, wrapper_name, output_file_name, extractor_result_file_name)
+    def export_old2(wrapper_name, output_file_name=nil, extractor_result_file_name=nil)
+      export_new({ :wrapper_name => wrapper_name, :output_file_name => output_file_name || "#{wrapper_name}_extractor_export.rb", :extractor_result_file_name => extractor_result_file_name })
+    end
+    
+    def export_new(data)
+      data[:source_file] = @source_file
+      data[:source_proc] = @source_proc
+      Scrubyt::Export.export(@root_patterns, data)
     end
   end
 end

data/lib/scrubyt/utils/compound_example_lookup.rb

@@ -43,7 +43,7 @@ private
     end
 
     def self.refine_partial_results(regexp)
-      @partial_results = @partial_results.select {|pr| pr.inner_text =~ regexp}
+      @partial_results = @partial_results.select {|pr| pr.inner_html.gsub(/<.*?>/, '') =~ regexp}
     end
 
   end #End of class CompoundExampleLookup

data/lib/scrubyt/utils/shared_utils.rb

@@ -28,7 +28,7 @@ module Scrubyt
     def self.traverse_for_match(node, regexp)
       results = []
       traverse_for_match_inner = lambda { |node, regexp|
-        ft = prepare_text_for_comparison(node.inner_text)
+        ft = prepare_text_for_comparison(node.inner_html.gsub(/<.*?>/, ''))
         if ft =~ regexp
           node.instance_eval do
             @match_data = $~

data/lib/scrubyt/utils/simple_example_lookup.rb

@@ -16,7 +16,7 @@ module Scrubyt
     #In this case, <a>'s text is considered to be "Bon nuit, monsieur"
     def self.find_node_from_text(doc, text, next_link=false, index = 0)
       text.gsub!('»', '&#187;')
-      #Process immediate attribute extraction (like "go to google.com[@href]")
+      #Process immediate attribute extraction (like "go to google.com/@href")
       if text =~ /.+\/@.+$/
         text = text.scan(/^(.+?)\/@.+$/)[0][0]
       elsif text =~ /.+\[\d+\]$/
@@ -27,10 +27,14 @@ module Scrubyt
         final_element_name = text.scan(/^(.+?)\[/)[0][0]
         text = text.scan(/\[(.+?)\]/)[0][0]
       end
-      text = Regexp.escape(text) if text.is_a? String
-      result = SharedUtils.traverse_for_match(doc,/^#{text}$/)[index]
-      result = XPathUtils.traverse_up_until_name(result,final_element_name) if final_element_name
-      result
+      if final_element_name
+        text = Regexp.escape(text) if text.is_a? String
+        result = SharedUtils.traverse_for_match(doc,/#{text}/)[index]
+        result = XPathUtils.traverse_up_until_name(result,final_element_name)
+      else
+        text = Regexp.escape(text) if text.is_a? String
+        result = SharedUtils.traverse_for_match(doc,/^#{text}$/)[index]
+      end
     end
   end #End of class SimpleExampleLookup
 end #End of module Scrubyt