scrubyt 0.2.8 → 0.3.0

data/lib/scrubyt/core/scraping/filters/base_filter.rb

@@ -43,13 +43,14 @@ module Scrubyt
     EXAMPLE_TYPE_IMAGE = 2
     #No example - the actual XPath is determined from the children XPaths (their LCA)
     EXAMPLE_TYPE_CHILDREN = 3
+
     #Regexp example, like /\d+@*\d+[a-z]/
     EXAMPLE_TYPE_REGEXP = 4
     #Compound example, like :contains => 'goodies'
     EXAMPLE_TYPE_COMPOUND = 5
 
     attr_accessor(:example_type, :parent_pattern, :temp_sink,
-                  :constraints, :xpath, :regexp, :example, :source, :sink)
+                  :constraints, :xpath, :regexp, :example, :final_result)
 
     def self.create(parent_pattern, example=nil)
 
@@ -63,6 +64,7 @@ module Scrubyt
 
     #Dispatcher method to add constraints; of course, as with any method_missing, this method
     #should not be called directly
+
     #TODO still used?
     def method_missing(method_name, *args, &block)
       case method_name.to_s
@@ -82,8 +84,6 @@ module Scrubyt
     def initialize(parent_pattern, example)
       @example_type = BaseFilter.determine_example_type(example)
       @parent_pattern = parent_pattern
-      @sink = []                  #output of a filter
-      @source = []                #input of a filter
       @example = example
       @xpath = nil                #The xpath to evaluate this filter
       @constraints = [] #list of constraints
@@ -98,9 +98,9 @@ module Scrubyt
         case example
         when nil
           EXAMPLE_TYPE_CHILDREN
-        when /\.(jpg|png|gif|jpeg)$/
+        when /\.(jpg|png|gif|jpeg)(\[\d+\])?$/
           EXAMPLE_TYPE_IMAGE
-        when /^\/{1,2}[a-z]+[0-9]?(\[[0-9]+\])?(\/{1,2}[a-z()]+[0-9]?(\[[0-9]+\])?)*$/
+        when /^\/{1,2}[a-z]+[0-9]?(\[[0-9]+\])?(\/{1,2}[a-z()]+[0-9]?(\[[0-9]+\])?)*(\[@.+=.+\])?(\/@.+)?$/
          (example.include? '/' || example.include?('[')) ? EXAMPLE_TYPE_XPATH : EXAMPLE_TYPE_STRING
         else
           EXAMPLE_TYPE_STRING

data/lib/scrubyt/core/scraping/filters/detail_page_filter.rb

@@ -3,9 +3,9 @@ module Scrubyt
 
     def evaluate(source)
       if source.is_a? String
-        result = @parent_pattern.evaluation_context.extractor.evaluate_subextractor(source, @parent_pattern, @parent_pattern.resolve)
+        @parent_pattern.evaluation_context.extractor.evaluate_subextractor(source, @parent_pattern, @parent_pattern.resolve)
       else
-        result = @parent_pattern.evaluation_context.extractor.evaluate_subextractor(
+        @parent_pattern.evaluation_context.extractor.evaluate_subextractor(
           XPathUtils.find_nearest_node_with_attribute(source, 'href').attributes['href'],
           @parent_pattern, @parent_pattern.resolve)
       end

data/lib/scrubyt/core/scraping/filters/download_filter.rb

@@ -20,7 +20,8 @@ private
       return '' if source.size < 4
       file_name = source.scan(/.+\/(.*)/)[0][0]
       Net::HTTP.start(base_url) { |http|
-        resp = http.get(source)
+        puts "downloading: #{source.scan(/\s*(.+)/)[0][0]}"
+        resp = http.get(source.scan(/\s*(.+)/)[0][0])
         outfile = DownloadFilter.find_nonexisting_file_name(File.join(@example, file_name))
         FileUtils.mkdir_p @example
         open(outfile, 'wb') {|f| f.write(resp.body) }

data/lib/scrubyt/core/scraping/filters/html_subtree_filter.rb

@@ -1,7 +1,12 @@
 module Scrubyt
-  class HTMLSubTreeFilter < BaseFilter
- 
+  class HtmlSubtreeFilter < BaseFilter
+
     def evaluate(source)
+      source.inner_html
+    end
+
+    def to_sexp
+      nil
     end #end of method
   end #End of class TreeFilter
 end #End of module Scrubyt

data/lib/scrubyt/core/scraping/filters/tree_filter.rb

@@ -2,9 +2,22 @@ module Scrubyt
   class TreeFilter < BaseFilter
 
     def evaluate(source)
+      return [@final_result] if @final_result
+      #Crude hack! Drop it after it will be supported in Hpricot
+      if @xpath =~ /.+\/@.+$/
+        @example = @xpath
+        @xpath = @xpath.scan(/^(.+?)\/@/)[0][0]
+      end
       result = source/@xpath
-      #puts "Evaluating #{@parent_pattern.name} with #{@xpath}"
-      xpath_results = if result.class == Hpricot::Elements then result.map else [result] end
+
+      Scrubyt.log :ACTION, "Evaluating #{@parent_pattern.name} with #{@xpath}"
+
+      xpath_results = Hpricot::Elements === result ? result : [result]
+
+      if @example =~ /.+\/@.+$/
+        result_attribute = @example.scan(/.+\/@(.+?)$/)[0][0]
+        xpath_results.map! {|r| r.attributes[result_attribute] }
+      end
       if @regexp == nil
         xpath_results
       else
@@ -22,6 +35,8 @@ module Scrubyt
     def generate_regexp_for_example
       return if @example_type != EXAMPLE_TYPE_STRING
       return if @temp_sink.nil?
+      return if @temp_sink.is_a? String
+      return if @example =~ /.+\[.+\]$/
 
       text = SharedUtils.prepare_text_for_comparison(@temp_sink.inner_text)
       match_range = @temp_sink.match_data.begin(0)..@temp_sink.match_data.end(0)
@@ -49,10 +64,14 @@ module Scrubyt
       when EXAMPLE_TYPE_XPATH
         @xpath = @example
       when EXAMPLE_TYPE_STRING
-        @temp_sink = SimpleExampleLookup.find_node_from_text(@parent_pattern.evaluation_context.root_pattern.filters[0].source[0],
+        @temp_sink = SimpleExampleLookup.find_node_from_text(@parent_pattern.evaluation_context.extractor.get_hpricot_doc,
                                                              @example,
                                                              next_page_example)
         return if @temp_sink == nil
+        if @temp_sink.is_a? String
+          @final_result = @temp_sink
+          return
+        end
 
         mark_changing_ranges = lambda { |element, range|
           element.instance_eval do
@@ -63,14 +82,14 @@ module Scrubyt
           end
         }
         mark_changing_ranges.call(@temp_sink, @temp_sink.match_data.begin(0)..@temp_sink.match_data.end(0))
-
-        @xpath = XPathUtils.generate_XPath(@temp_sink, nil, !@parent_pattern.generalize)
+        write_indices = next_page_example ? true : !@parent_pattern.generalize
+        @xpath = XPathUtils.generate_XPath(@temp_sink, nil, write_indices)
       when EXAMPLE_TYPE_CHILDREN
         current_example_index = 0
         loop do
           all_child_temp_sinks = []
           @parent_pattern.children.each do |child_pattern|
-            all_child_temp_sinks << child_pattern.filters[current_example_index].temp_sink
+            all_child_temp_sinks << child_pattern.filters[current_example_index].temp_sink if child_pattern.filters[current_example_index].temp_sink
           end
           result = all_child_temp_sinks.pop
           if all_child_temp_sinks.empty?
@@ -81,7 +100,7 @@ module Scrubyt
             end
           end
           xpath = @parent_pattern.generalize ? XPathUtils.generate_XPath(result, nil, false) :
-          XPathUtils.generate_XPath(result, nil, true)
+                                               XPathUtils.generate_XPath(result, nil, true)
           if @parent_pattern.filters.size < current_example_index + 1
             @parent_pattern.filters << Scrubyt::BaseFilter.create(@parent_pattern)
           end
@@ -97,24 +116,30 @@ module Scrubyt
           current_example_index += 1
         end
       when EXAMPLE_TYPE_IMAGE
-        #@temp_sink = XPathUtils.find_image(@parent_pattern.evaluation_context.root_pattern.filters[0].source[0], @example)\
         @temp_sink = XPathUtils.find_image(@parent_pattern.evaluation_context.extractor.get_hpricot_doc, @example)
-        @xpath = XPathUtils.generate_XPath(@temp_sink, nil, false)
+        @xpath = XPathUtils.generate_XPath(@temp_sink, nil, true)
       when EXAMPLE_TYPE_COMPOUND
-        @temp_sink = CompoundExampleLookup.find_node_from_compund_example(@parent_pattern.evaluation_context.root_pattern.filters[0].source[0],
+        @temp_sink = CompoundExampleLookup.find_node_from_compund_example(@parent_pattern.evaluation_context.extractor.get_hpricot_doc,
                                                                           @example,
                                                                           next_page_example)
         @xpath = @parent_pattern.generalize ? XPathUtils.generate_XPath(@temp_sink, nil, false) :
-        XPathUtils.generate_XPath(@temp_sink, nil, true)
+                                              XPathUtils.generate_XPath(@temp_sink, nil, true)
       end
     end
 
     def generate_relative_XPath(parent_xpath)
+      parent_xpath = XPathUtils.to_full_XPath(@parent_pattern.evaluation_context.extractor.get_hpricot_doc,
+                                              parent_xpath,
+                                              @parent_pattern.parent.generalize) if parent_xpath =~ /(\[@.+=.+\])$/
       @xpath = XPathUtils.generate_relative_XPath_from_XPaths(parent_xpath, @xpath) if (@xpath =~ /^\/html/) #TODO: should not rely on <html> being the root node
     end
 
     def to_sexp
-      [:str, @xpath]
+      if @example =~ /.+\[@.+\]$/
+        [:str, "#{@xpath}/@#{@example.scan(/\[@(.+?)\]/)[0][0]}"]
+      else
+        [:str, @xpath]
+      end
     end
 
   end #End of class TreeFilter

data/lib/scrubyt/core/scraping/pattern.rb

@@ -33,7 +33,7 @@ module Scrubyt
     #    # write out the HTML subtree beginning at the matched element
     #    PATTERN_TYPE_HTML_SUBTREE = :PATTERN_TYPE_HTML_SUBTREE
 
-    VALID_PATTERN_TYPES = [:root, :tree, :attribute, :regexp, :detail_page, :download, :html_subtree]
+    VALID_PATTERN_TYPES = [:tree, :attribute, :regexp, :detail_page, :download, :html_subtree]
 
     #The pattern can be either a model pattern (in this case it is
     #written to the output) or a temp pattern (in this case it is skipped)
@@ -49,10 +49,11 @@ module Scrubyt
     VALID_OUTPUT_TYPES = [:model, :temp]
 
     #These options can be set upon wrapper creation
-    VALID_OPTIONS = [:generalize, :type, :output_type, :write_text, :references, :limit, :default, :resolve]  + Scrubyt::CompoundExample::DESCRIPTORS
+    PATTERN_OPTIONS = [:generalize, :type, :output_type, :references, :limit, :default, :resolve]
+    VALID_OPTIONS = PATTERN_OPTIONS + Scrubyt::CompoundExample::DESCRIPTORS + Scrubyt::ResultNode::OUTPUT_OPTIONS
 
     attr_accessor(:name, :options, :children, :constraints, :filters, :parent,
-                  :last_result, :result, :evaluation_context,
+                  :last_result, :evaluation_context,
                   :indices_to_extract, :referenced_extractor, :referenced_pattern,
                   :source_file, :source_proc, :modifier_calls)
 
@@ -60,7 +61,7 @@ module Scrubyt
 
     option_reader(:type => :tree, :output_type => :model, :generalize => false,
                   :write_text => lambda { @children.size == 0 }, :limit => nil,
-                  :default => nil, :resolve => :full)
+    :default => nil, :resolve => :full)
 
     def initialize(name, args=[], evaluation_context=nil, parent=nil, &block)
       #init attributes
@@ -71,7 +72,6 @@ module Scrubyt
       @children = []
       @filters = []
       @constraints = []
-      @result = Result.new
       @modifier_calls = []
 
       #grab any examples that are defined
@@ -93,9 +93,12 @@ module Scrubyt
         end
       end
 
-      #by default, generalize direct children of the root pattern, but only in the case if
+      #by default, generalize the root pattern, but only in the case if
       #@generalize was not set up explicitly
-      @options[:generalize] = true if parent && parent.type == :root && @options[:generalize].nil?
+      if @options[:generalize].nil?
+        @options[:generalize] = true if parent.nil?
+        @options[:generalize] = false if filters[0].example =~ /.+\[[a-zA-Z].+\]$/
+      end
 
       #parse child patterns if available
       parse_child_patterns(&block) if ( !block.nil? && type != :detail_page )
@@ -104,7 +107,7 @@ module Scrubyt
       if type == :tree
         #generate xpaths and regexps
         @filters.each do |filter|
-          filter.generate_XPath_for_example(false)
+          filter.generate_XPath_for_example(false) unless @name == 'next_page'
           filter.generate_regexp_for_example
         end
         #when the xpaths of this pattern have been created, its children can make their xpaths relative
@@ -154,9 +157,6 @@ module Scrubyt
     #an ancestor of a detail pattern , and store this in a hash if yes (to be able to
     #traverse the pattern structure on detail pages as well).
     def check_if_detail_page(block)
-      #return if !@options[:references]
-      #@options[:type] = :detail_page
-      #@referenced_extractor = @options[:references]
       if @name =~ /.+_detail/
         @options[:type] = :detail_page
         @referenced_extractor = block
@@ -168,6 +168,10 @@ module Scrubyt
       @children.inject(false) { |is_parent_of_leaf, child| is_parent_of_leaf || child.children.empty? }
     end
 
+    def filter_count
+      @filters.size
+    end
+
     def parse_child_patterns(&block)
       context = Object.new
       context.instance_eval do
@@ -177,7 +181,8 @@ module Scrubyt
         def method_missing(method_name, *args, &block)
           if method_name.to_s[0..0] == '_'
             #add hash option
-            key = :"#{method_name.to_s[1..-1]}"
+            key = method_name.to_s[1..-1].to_sym
+            check_option(key)
             args.each do |arg|
               current_value = @current.options[key]
               if current_value.nil?
@@ -216,8 +221,6 @@ module Scrubyt
       when 'select_indices'
         @result_indexer = Scrubyt::ResultIndexer.new(*args)
         return self
-      when /^to_/
-        return Scrubyt::ResultDumper.send(method_name.to_s, self)
       when /^ensure_/
         @constraints << Scrubyt::ConstraintAdder.send(method_name, *args)
         return self #To make chaining possible
@@ -228,80 +231,72 @@ module Scrubyt
       raise NoMethodError.new(method_name.to_s, method_name.to_s, args)
     end
 
-    #Companion function to the previous one (Pattern::method_missing). It makes
-    #inspecting results, like
-    #
-    #    camera_data.item[1].item_name[0]
-    #
-    #possible. The method Pattern::method missing handles the 'item', 'item_name' etc.
-    #parts, while the indexing ([1], [0]) is handled by this function.
-    #If you would like to select a different document than the first one (which is
-    #the default), you should use the form:
-    #
-    #    camera_data[1].item[1].item_name[0]
-    def [](index)
-      if @name == 'root'
-        @evaluation_context.document_index = index
-      else
-        @parent.last_result = @parent.last_result[@evaluation_context.document_index] if @parent.last_result.is_a? Array
-        return nil if (@result.lookup(@parent.last_result)) == nil
-        @last_result = @result.lookup(@parent.last_result)[index]
+    def evaluate(source, filter_indices)
+      if type == :detail_page # DIRTY!
+        return @filters[0].evaluate(source)
       end
-      self
-    end
 
-    ##
-    #If export is called on the root pattern, it exports the whole extractor wher it is
-    #defined; See export.rb for further details on the parameters
-    def export(arg1, output_file_name=nil, extractor_result_file_name=nil)
-      #      require 'scrubyt/output/export_old'; Scrubyt::ExportOld.export(arg1, self, output_file_name, extractor_result_file_name) ; return
-      if File.exists? arg1
-        old_export(arg1, output_file_name, extractor_result_file_name)
-      else
-        new_export(arg1, output_file_name, extractor_result_file_name)
+      #we apply all filters if filter_indices is nil
+      indices_to_evaluate = filter_indices.nil? ? 0...@filters.size : filter_indices
+      #stores the results of all filters
+      all_filter_results = []
+      #remembers which filters have retured a certain result
+      indices_mapping = {}
+      #evaluate filters and collect filter results
+      indices_to_evaluate.each do |filter_index|
+        filter = @filters[filter_index]
+        filter_results = filter.evaluate(source)
+        filter_results.each do |result|
+          #add result to list if not already there
+          all_filter_results << result if all_filter_results.index(result).nil?
+          #add the current filter's index to the mapping
+           (indices_mapping[result] ||= []) << filter_index
+        end
       end
-    end
-
-    def old_export(input_file, output_file_name=nil, extractor_result_file_name=nil)
-      contents = open(input_file).read
-      wrapper_name = contents.scan(/\s+(.+)\s+=.*Extractor\.define.*/)[0][0]
-      Scrubyt::Export.export(self, wrapper_name, output_file_name, extractor_result_file_name)
-    end
-
-    def new_export(wrapper_name, output_file_name=nil, extractor_result_file_name=nil)
-      Scrubyt::Export.export(self, wrapper_name, output_file_name, extractor_result_file_name)
-    end
 
-    ##
-    #Evaluate the pattern. This means evaluating all the filters and adding
-    #their extracted instances to the array of results of this pattern
-    def evaluate(parent_filters)
-      if type != :root #TODO: should be removed, but there is more refactoring of filter handling needed to do so
-        all_filter_results = []
-        @filters.each do |filter|
-          filter_index = @filters.index(filter)
-          filter_index = 0 if parent_filters.size <= filter_index
-          filter.source = parent_filters[filter_index].sink
-          filter.source.each do |source|
-            results = filter.evaluate(source)
-            next if results == nil
-            #apply constraints
-            if @constraints.size > 0
-              results = results.select do |result|
-                @constraints.inject(true) { |accepted, constraint| accepted && constraint.check(result) }
-              end
-            end
-            #apply indexer
-            results = @result_indexer.select_indices_to_extract(results) if !@result_indexer.nil?
-            add_result(filter, source, results)
+      #apply constraints
+      if @constraints.size > 0
+        all_filter_results = all_filter_results.select do |result|
+          @constraints.inject(true) { |accepted, constraint| accepted && constraint.check(result) }
+        end
+      end
+      #apply indexer
+      all_filter_results = @result_indexer.select_indices_to_extract(all_filter_results) if !@result_indexer.nil?
+
+      #create result nodes and evaluate children
+      result_nodes = []
+      all_filter_results.each do |result|
+        #create result node
+        node = ResultNode.new(@name, result, @options)
+        node.generated_by_leaf = (@children.size == 0)
+        #evaluate children
+        @children.each do |child|
+          raise if self.filter_count != 1 && child.filter_count != self.filter_count
+          if self.filter_count == 1
+            #evaluate all child filters
+            node.push(*child.evaluate(result, nil))
+          else
+            #evaluate appropriate child filters
+            node.push(*child.evaluate(result, indices_mapping[result]))
           end
         end
+        #apply child constraints (ensure_presence_of_pattern)
+        required_child_names = @constraints.select {|c| c.type == Scrubyt::Constraint::CONSTRAINT_TYPE_ENSURE_PRESENCE_OF_PATTERN }.map {|c| c.target}
+        unless required_child_names.empty?
+          check = lambda { |node_to_check|
+            required_child_names.delete node_to_check.name
+            node_to_check.each { |child| check.call child }
+          }
+          check.call node
+        end
+        next unless required_child_names.empty?
+        #add the current result node to the list
+        result_nodes << node
       end
-
-      #evaluate children
-      @children.each { |child| child.evaluate(@filters) }
-
-      #do postprocessing
+      if result_nodes.empty?
+        result_nodes << ResultNode.new(@name,@options[:default],@options) if @options[:default]
+      end
+      result_nodes
     end
 
     def to_sexp
@@ -316,7 +311,7 @@ module Scrubyt
       if type == :detail_page
         #add detail page extractor
         detail_root = @evaluation_context.extractor.get_detail_extractor(self)
-        sexp = [:iter, sexp, nil, [:block, *detail_root.children.to_sexp_array ]]
+        sexp = [:iter, sexp, nil, [:block, detail_root.to_sexp]]
       else
         #add child block if the pattern has children
         sexp = [:iter, sexp, nil, [:block, *@children.to_sexp_array ]] if !@children.empty?
@@ -336,11 +331,15 @@ module Scrubyt
       #merge provided hash
       @options.merge!(hash)
       #check if valid
-      hash.each { |key, value| raise "Unknown pattern option: #{key.to_s}" if VALID_OPTIONS.index(key.to_sym).nil? }
+      hash.each { |key, value| check_option(key.to_sym) }
       raise "Invalid pattern type: #{type.to_s}" if VALID_PATTERN_TYPES.index(type.to_sym).nil?
       raise "Invalid output type: #{output_type.to_s}" if VALID_OUTPUT_TYPES.index(output_type.to_sym).nil?
     end
 
+    def check_option(option)
+      raise "Unknown pattern option: #{option.to_s}" if VALID_OPTIONS.index(option).nil?
+    end
+
     def look_for_examples(args)
       if (args[0].is_a? String)
         examples = args.select {|e| e.is_a? String}
@@ -370,12 +369,5 @@ module Scrubyt
       examples
     end
 
-    def add_result(filter, source, results)
-      results.each do |res|
-        filter.sink << res
-        @result.add_result(source, res)
-      end
-    end
-
   end #end of class Pattern
 end #end of module Scrubyt