scrubyt 0.2.0 → 0.2.3

data/lib/scrubyt/{filter.rb → core/scraping/filter.rb}

@@ -69,12 +69,18 @@ module Scrubyt
     #Evaluate this filter. This method shoulf not be called directly - as the pattern hierarchy 
     #is evaluated, every pattern evaluates its filters and then they are calling this method
     def evaluate(source)
-      case @parent_pattern.type
+      case @parent_pattern.type        
         when Scrubyt::Pattern::PATTERN_TYPE_TREE      
           result = source/@xpath
+          #puts "Evaluating #{@parent_pattern.name} with #{@xpath}"
           result.class == Hpricot::Elements ? result.map : [result]
         when Scrubyt::Pattern::PATTERN_TYPE_ATTRIBUTE  
-          [source.attributes[@example]]
+          attribute_value = [source.attributes[@example]] 
+          return attribute_value if attribute_value[0]
+          @@attribute_in_parent = nil
+          Filter.traverse_up_until_attribute_found(source.parent, @example)
+          @@attribute_in_parent
+                  
         when Scrubyt::Pattern::PATTERN_TYPE_REGEXP
           source.inner_text.scan(@example).flatten
       end      
@@ -88,7 +94,9 @@ module Scrubyt
         when EXAMPLE_TYPE_XPATH
           @xpath = @example
         when EXAMPLE_TYPE_STRING
-          @temp_sink = XPathUtils.find_node_from_text( @parent_pattern.root_pattern.filters[0].source[0], @example, false )
+          @temp_sink = XPathUtils.find_node_from_text( @parent_pattern.evaluation_context.root_pattern.filters[0].source[0], 
+	                                                   @example, 
+						                               false )
           @xpath = @parent_pattern.generalize ? XPathUtils.generate_XPath(@temp_sink, nil, false) :
                                                  XPathUtils.generate_XPath(@temp_sink, nil, true)
         when EXAMPLE_TYPE_CHILDREN          
@@ -127,7 +135,7 @@ module Scrubyt
             current_example_index += 1
           end
         when EXAMPLE_TYPE_IMAGE
-          @temp_sink = XPathUtils.find_image(@parent_pattern.root_pattern.filters[0].source[0], @example)
+          @temp_sink = XPathUtils.find_image(@parent_pattern.evaluation_context.root_pattern.filters[0].source[0], @example)
           @xpath = XPathUtils.generate_XPath(@temp_sink, nil, false)
       end
     end
@@ -139,6 +147,16 @@ module Scrubyt
     end    
     
 private    
+    def self.traverse_up_until_attribute_found(source, attribute)
+      if (!source.parent.is_a? Hpricot::Doc)
+        #p source.attributes
+        #p attribute
+        #p source.attributes[attribute]
+        @@attribute_in_parent = source.attributes[attribute] if source.attributes[attribute]        
+        traverse_up_until_attribute_found(source.parent, attribute) if !@attribute_in_parent
+      end
+    end
+        
     def self.determine_example_type(example)
       if example.instance_of? Regexp
         EXAMPLE_TYPE_REGEXP

data/lib/scrubyt/{pattern.rb → core/scraping/pattern.rb}

@@ -41,23 +41,18 @@ module Scrubyt
     SETTABLE_FIELDS = ['generalize', 'type', 'output_type', 'example']
     
     attr_accessor :name, :output_type, :generalize, :children, :filters, :parent, 
-                  :last_result, :result, :root_pattern, :example,  :block_count, 
-                  :next_page, :limit, :extractor, :extracted_docs,
-                  :examples, :parent_of_leaf, :document_index
-    attr_reader :type, :generalize_set, :next_page_url
+                  :last_result, :result, :example, :limit, 
+                  :examples, :parent_of_leaf, :evaluation_context,
+                  :indices_to_extract, :evaluation_context
+    attr_reader :type, :generalize_set, :next_page_url, :result_indexer
 
     def initialize (name, *args)
       @name = name                #name of the pattern
       parse_args(args)            #parse the rest of the arguments
-      @root_pattern = nil         #root pattern of the wrapper
       @children = []              #child patterns
       @filters = []               #filters of the wrapper
       @result = Result.new        #hierarchical results of the pattern
-      @@instance_count = Hash.new(0)
-      @evaluated_examples = []
-      @next_page = nil
-      @document_index = 0
-      if @examples == nil       
+      if @examples == nil
         filters << Scrubyt::Filter.new(self) #create a default filter
       else
         @examples.each do |example|
@@ -112,6 +107,9 @@ module Scrubyt
     # camera_data.item[1].item_name[0]
     def method_missing(method_name, *args, &block)
       case method_name.to_s
+      when 'select_indices'
+        @result_indexer = Scrubyt::ResultIndexer.new(*args)
+        self
       when /^to_/
         Scrubyt::ResultDumper.send(method_name.to_s, self)
       when /^ensure_/
@@ -135,9 +133,9 @@ module Scrubyt
     #    camera_data[1].item[1].item_name[0]
     def [](index)
       if @name == 'root'
-        @root_pattern.document_index = index
+        @evaluation_context.document_index = index
       else
-        @parent.last_result = @parent.last_result[@root_pattern.document_index] if @parent.last_result.is_a? Array
+        @parent.last_result = @parent.last_result[@evaluation_context.document_index] if @parent.last_result.is_a? Array
         return nil if (@result.lookup(@parent.last_result)) == nil
         @last_result = @result.lookup(@parent.last_result)[index]
       end
@@ -150,7 +148,7 @@ module Scrubyt
     def export(file, output_file_name=nil, extractor_result_file_name=nil)
       Scrubyt::Export.export(file, self, output_file_name, extractor_result_file_name)
     end
-
+    
     ##
     #Add a filter to this pattern    
     def add_filter(filter)
@@ -167,39 +165,7 @@ module Scrubyt
       child.generalize = true if (!child.generalize_set && child.parent != nil && child.parent.parent == nil)
       @children << child
     end
-    
-    ##
-    #Crawl to a new page. This function should not be called from the outside - it is automatically called
-    #if the next_page is defined
-    def crawl_to_new_page
-      temp_document = generate_next_page_link(@next_page)
-      return nil if temp_document == nil
-      clear_sources_and_sinks(@root_pattern)
-      @root_pattern.extractor.fetch(temp_document)
-      attach_current_document
-    end
-    
-    ##
-    #Attach document to the root pattern; This is happening automatically as the root pattern is defined or
-    #crawling to a new page
-    def attach_current_document
-      doc = @root_pattern.extractor.get_hpricot_doc
-      filters[0].source << doc
-      filters[0].sink << doc
-      @last_result ||= []
-      @last_result << doc          
-      @result.add_result(filters[0].source, filters[0].sink)
-    end
-
-    ##
-    #Based on the given examples, calculate the XPaths for the tree patterns    
-    def setup_examples      
-      get_root_pattern(self)
-      mark_leaf_parents(self)
-      set_root_pattern_whole_wrapper(@root_pattern, @root_pattern)
-      generate_examples(@root_pattern)
-    end
-
+        
     ##
     #Evaluate the pattern. This means evaluating all the filters and adding 
     #their extracted instances to the array of results of this pattern
@@ -225,23 +191,21 @@ module Scrubyt
             end
             result = result_hash.reject {|k,v| k if !v}
             sorted_result = r.reject {|e| !result.keys.include? e}
-            add_result(filter, source, sorted_result)
+            indexer = @result_indexer == nil ? sorted_result : @result_indexer.select_indices_to_extract(sorted_result)
+            add_result(filter, source, indexer)
           else
-            add_result(filter, source, r)
+            indexer = @result_indexer == nil ? r : @result_indexer.select_indices_to_extract(r)
+            add_result(filter, source, indexer)
           end#end of constraint check
         end#end of source iteration
       end#end of filter iteration    
-    end    
-       
-    def get_instance_count
-      @@instance_count
-    end
+    end   
     
     def get_constraints
       filters[0].constraints
-    end
-            
-private
+    end     
+       
+private        
     def look_for_examples(args)
       if (args[0].is_a? String)
         @examples = args.select {|e| e.is_a? String}
@@ -269,48 +233,7 @@ private
       results.each do |res|
         filter.sink << res
         @result.add_result(source, res)
-        @@instance_count[@name] += 1
       end      
-    end
-
-    def get_root_pattern(pattern)
-      if @root_pattern == nil
-        while (pattern.parent != nil)
-          get_root_pattern(pattern.parent)
-        end
-        @root_pattern = pattern
-      end
-    end
-    
-    def mark_leaf_parents(pattern)
-      pattern.children.each { |child|
-        pattern.parent_of_leaf = true if child.children.size == 0
-      }
-      pattern.children.each { |child| mark_leaf_parents(child) }      
-    end
-    
-    def set_root_pattern_whole_wrapper(pattern, root_pattern)
-      pattern.children.each {|child| set_root_pattern_whole_wrapper(child, root_pattern)}
-      pattern.root_pattern = root_pattern
-    end
-       
-    def generate_examples(pattern)
-      pattern.children.each {|child_pattern| generate_examples(child_pattern) }
-      pattern.filters.each { |filter| filter.generate_XPath_for_example } if pattern.type == PATTERN_TYPE_TREE
-    end
-    
-    def clear_sources_and_sinks(pattern)
-      pattern.filters.each do |filter|
-        filter.source = []
-        filter.sink = []
-      end
-      pattern.children.each {|child| clear_sources_and_sinks child}
-    end
-    
-    def generate_next_page_link(example)
-      node = XPathUtils.find_node_from_text(@root_pattern.filters[0].source[0], example, true)
-      return nil if node == nil
-      node.attributes['href'].gsub('&amp;') {'&'}
-    end # end of method generate_next_page_link    
+    end # end of method generate_examples
   end #end of class Pattern
 end #end of module Scrubyt

data/lib/scrubyt/core/scraping/pre_filter_document.rb

@@ -0,0 +1,13 @@
+module Scrubyt
+  ##
+  #=<tt>Apply different functions on the input document</tt>
+  #Before the document is passed to Hpricot for parsing, we may need
+  #to do different stuff with it which are clumsy/not appropriate/impossible 
+  #to do once the document is loaded.
+  class PreFilterDocument
+     #Replace <br/> tags with newlines
+     def self.br_to_newline(doc)
+       doc.gsub(/<br[ \/]*>/i, "\r\n")
+     end #end of function  br_to_newline
+  end #end of class PreFilterDocument
+end #end of module Scrubyt

data/lib/scrubyt/core/scraping/result_indexer.rb

@@ -0,0 +1,88 @@
+module Scrubyt
+   ##
+   #=<tt>Selecting results based on indices</tt>
+   #
+   #If the results is list-like (as opposed to a 'hard' result, like a _price_ or a _title_),
+   #probably with a variable count of results (like tags, authors etc.), you may need just 
+   #specific elements - like the last one, every third one, or at specific indices.
+   #In this case you should use the select_indices syntax.
+  class ResultIndexer    
+    attr_reader :indices_to_extract
+  
+    def initialize(*args)
+      select_indices(*args)
+    end
+
+    ##
+    #Perform selection of the desires result instances, based on their indices
+    def select_indices_to_extract(ary)
+      return ary if @indices_to_extract == nil
+      to_keep = []
+      @indices_to_extract.each {|e|
+        if e.is_a? Symbol
+          case e
+            when :first
+              to_keep << 0
+            when :last
+              to_keep << ary.size-1
+            when :all_but_last
+              (0..ary.size-2).each {|i| to_keep << i} 
+            when :all_but_first
+              (1..ary.size-1).each {|i| to_keep << i} 
+            when :every_even
+              (0..ary.size).each {|i| to_keep << i if (i % 2 == 1)}
+            when :every_odd
+              (0..ary.size).each {|i| to_keep << i if (i % 2 == 0)}
+            when :every_second
+              (0..ary.size).each {|i| to_keep << i if (i % 2 == 0)}
+            when :every_third
+              (0..ary.size).each {|i| to_keep << i if (i % 3 == 0)}
+          end
+        end
+      }
+      @indices_to_extract.each {|i| to_keep << i if !i.is_a? Symbol}
+      to_keep.sort!
+      ary.reject! {|e| !to_keep.include? ary.index(e)}
+      ary
+    end
+    
+private
+    ##
+    #Do not return the whole result set, just specified indices - like
+    #first,last, every odd index, indices from [1..3] etc.
+    #
+    #This method can accept:
+    #- a range, like (2..3)
+    #- an array of indices, like [1,2,3]
+    #- specified set of keywords:
+    #  - :first
+    #  - :last
+    #  - :every_even
+    #  - :every_odd
+    #  (there can be more of these keywords in one select_indices call)
+    def select_indices(*args)
+      indices_to_grab = args[0]
+      case indices_to_grab.class.to_s
+        when "Range"
+          @indices_to_extract = indices_to_grab.to_a
+        when "Array"
+          nested_arrays = []
+          indices_to_grab.each {|e|
+            if e.is_a? Array
+              nested_arrays << e
+            elsif e.is_a? Range
+              nested_arrays << e.to_a
+            end
+          }
+          @indices_to_extract = indices_to_grab
+          nested_arrays.each {|a| a.each {|e| @indices_to_extract << e if !@indices_to_extract.include? e }}
+          @indices_to_extract.reject! {|e| ((e.is_a? Range) || (e.is_a? Array)) }
+        when "Symbol"
+          #parse this when  we already have the results
+          @indices_to_extract = [indices_to_grab]
+        else
+          puts "Invalid index specification"
+      end
+    end #end of function select_indices
+  end #end of class ResultIndexer
+end #end of module Scrubyt

data/lib/scrubyt/core/shared/evaluation_context.rb

@@ -0,0 +1,97 @@
+module Scrubyt
+  ##
+  #=<tt>Holding the evaluation context of the extraction process</tt>
+  #
+  #Every kind of data that is shared among patterns during the extraction process
+  #is held in this class, so it can be looked up anytime.
+  #
+  #This class provides also some high-level basic functionality in navigation, like 
+  #crawling to new pages, attaching doucment to the root pattern once arrived at the
+  #desired page etc.
+  #
+  #It can be viewed as a glue between Extractor and NavigationActions as well - these
+  #two classes need to communicate frequently as well as share different information
+  #and this is accomplished through EvaluationContext.
+  class EvaluationContext
+    attr_accessor :root_pattern, :next_page, :document_index, :block_count,
+                  :extractor, :limit
+    
+    def initialize
+      @root_pattern = nil
+      @next_page = nil
+      @block_count = 0
+      @document_index = 0
+      @extractor = nil
+    end
+    
+    ##
+    #Crawl to a new page. This function should not be called from the outside - it is automatically called
+    #if the next_page is defined
+    def crawl_to_new_page
+      temp_document = generate_next_page_link(@next_page)
+      return nil if temp_document == nil
+      clear_sources_and_sinks(@root_pattern)
+      @extractor.fetch(temp_document)
+      attach_current_document
+    end
+
+    ##
+    #Attach document to the root pattern; This is happening automatically as the root pattern is defined or
+    #crawling to a new page
+    def attach_current_document
+      doc = @extractor.get_hpricot_doc
+      @root_pattern.filters[0].source << doc
+      @root_pattern.filters[0].sink << doc
+      @root_pattern.last_result ||= []
+      @root_pattern.last_result << doc          
+      @root_pattern.result.add_result(@root_pattern.filters[0].source, 
+                                      @root_pattern.filters[0].sink)
+    end
+
+    ##
+    #Based on the given examples, calculate the XPaths for the tree patterns    
+    def setup_examples      
+      get_root_pattern(nil)
+      mark_leaf_parents(@root_pattern)
+      generate_examples(@root_pattern)
+    end
+    
+    ##
+    #After crawling to the new page, the sources and sinks need to be cleaned
+    #since they are no more valid 
+    def clear_sources_and_sinks(pattern)
+      pattern.filters.each do |filter|
+        filter.source = []
+        filter.sink = []
+      end
+      pattern.children.each {|child| clear_sources_and_sinks child}
+    end
+    
+    def generate_next_page_link(example)
+      node = XPathUtils.find_node_from_text(@root_pattern.filters[0].source[0], example, true)
+      return nil if node == nil
+      node.attributes['href'].gsub('&amp;') {'&'}
+    end         
+        
+    def mark_leaf_parents(pattern)
+      pattern.children.each { |child|
+        pattern.parent_of_leaf = true if child.children.size == 0
+      }
+      pattern.children.each { |child| mark_leaf_parents(child) }      
+    end
+    
+    def generate_examples(pattern)
+      pattern.children.each {|child_pattern| generate_examples(child_pattern) }
+      pattern.filters.each { |filter| filter.generate_XPath_for_example } if pattern.type == Pattern::PATTERN_TYPE_TREE
+    end    
+    
+    def get_root_pattern(pattern)
+      if @root_pattern == nil
+        while (pattern.parent != nil)
+          get_root_pattern(pattern.parent)
+        end
+        @root_pattern = pattern
+      end
+    end #end of function
+  end #end of class EvaluationContext
+end #end of module Scrubyt

data/lib/scrubyt/core/shared/extractor.rb

@@ -0,0 +1,116 @@
+require 'logger'
+require 'open-uri'
+require 'rubygems'
+require 'mechanize'
+require 'hpricot'
+
+module Scrubyt
+##
+#=<tt>Driving the whole extraction process</tt>
+#
+#Extractor is a performer class - it gets an extractor definition and carries
+#out the actions and evaluates the wrappers sequentially. 
+#
+#Originally also the navigation actions were here, but since the class got too
+#big, they were factored out to an own class, NavigationAction.
+  class Extractor 
+    #The definition of the extractor is passed through this method
+    def self.define(mode=nil, &extractor_definition)
+      @@mode = mode
+      mode_name = (mode == :production ? 'Production' : 'Learning')
+      puts "[MODE] #{mode_name}"
+      NavigationActions.new
+      @@evaluation_context = EvaluationContext.new
+      #Hack up an artificial root pattern (i.e. do not return the pattern which 
+      #is the root one in the user's definition, but rather the real (invisible)
+      #root pattern
+      evaluated_extractor = (class_eval(&extractor_definition))
+      if evaluated_extractor == nil
+        puts "No extractor defined, exiting..."
+        exit
+      end
+      root_pattern = evaluated_extractor.parent
+      #Recursively match data based on examples
+      @@evaluation_context.setup_examples
+      #Once all is set up, evaluate the extractor from the root pattern!
+      evaluate_extractor(root_pattern)
+      #Apply all postprocess steps
+      PostProcessor.apply_post_processing(root_pattern)
+      #Return the root pattern
+      puts "Extraction finished succesfully!"
+      root_pattern
+    end
+
+  #build the current wrapper    
+  def self.method_missing(method_name, *args, &block)
+    if NavigationActions::KEYWORDS.include? method_name.to_s
+      NavigationActions.send(method_name, *args)
+      return
+    end
+    pattern = Scrubyt::Pattern.new(method_name.to_s, *args)
+    pattern.evaluation_context = @@evaluation_context
+    if @parent == nil
+      if method_name.to_s == 'next_page'
+        @@evaluation_context.next_page = args[0]
+        @@evaluation_context.limit = 
+          args[1][:limit] if args.size > 1
+        return @@last_pattern
+      else
+        #Create a root pattern
+        root_pattern = Scrubyt::Pattern.new('root', :type => :root) 
+        root_pattern.evaluation_context = @@evaluation_context
+        @@evaluation_context.root_pattern = root_pattern
+        @@evaluation_context.extractor = self
+        #add the currently active document to the root pattern 
+        @@evaluation_context.attach_current_document
+        @@evaluation_context.root_pattern.add_child_pattern(pattern)        
+        @@evaluation_context.block_count = 0
+      end
+    else
+      @parent.add_child_pattern(pattern) if @parent != nil
+    end
+    if block_given?
+      @@evaluation_context.block_count = @@evaluation_context.block_count + 1
+      @stack ||=[]
+      @parent = pattern
+      @stack.push @parent
+      class_eval(&block)
+      @stack.pop
+      @parent = @stack.last
+    end    
+    @@last_pattern = pattern
+  end
+
+  #Used in lord of the hacks vol 1. Check out export.rb if you are still interested
+  #(You should not be :)    
+  def self.get_block_count
+    @@root_pattern.block_count
+  end
+  
+  def self.get_hpricot_doc
+    NavigationActions.get_hpricot_doc
+  end
+  
+  def self.get_mode
+    @@mode
+  end  
+private
+    def self.evaluate_extractor(root_pattern)  
+      if @@evaluation_context.next_page
+        current_page_count = 1
+        loop do
+          really_evaluate_extractor(root_pattern)
+          break if (@@evaluation_context.limit == current_page_count || @@evaluation_context.crawl_to_new_page == nil)
+          current_page_count += 1 if @@evaluation_context.limit != nil
+        end
+      else 
+        really_evaluate_extractor(root_pattern)
+      end
+    end  
+    
+    def self.really_evaluate_extractor(pattern)
+      pattern.evaluate
+      pattern.children.each { |child| really_evaluate_extractor child } 
+    end #end of method evaluate_wrapper    
+  end #end of class Extractor
+end #end of module Scrubyt