scrubyt 0.2.6 → 0.2.8

data/lib/scrubyt/core/scraping/result_indexer.rb

@@ -1,18 +1,18 @@
 module Scrubyt
-   ##
-   #=<tt>Selecting results based on indices</tt>
-   #
-   #If the results is list-like (as opposed to a 'hard' result, like a _price_ or a _title_),
-   #probably with a variable count of results (like tags, authors etc.), you may need just 
-   #specific elements - like the last one, every third one, or at specific indices.
-   #In this case you should use the select_indices syntax.
+  ##
+  #=<tt>Selecting results based on indices</tt>
+  #
+  #If the results is list-like (as opposed to a 'hard' result, like a _price_ or a _title_),
+  #probably with a variable count of results (like tags, authors etc.), you may need just 
+  #specific elements - like the last one, every third one, or at specific indices.
+  #In this case you should use the select_indices syntax.
   class ResultIndexer    
     attr_reader :indices_to_extract
-  
+    
     def initialize(*args)
       select_indices(*args)
     end
-
+    
     ##
     #Perform selection of the desires result instances, based on their indices
     def select_indices_to_extract(ary)
@@ -21,24 +21,24 @@ module Scrubyt
       @indices_to_extract.each {|e|
         if e.is_a? Symbol
           case e
-            when :first
-              to_keep << 0
-            when :last
-              to_keep << ary.size-1
-            when :all_but_last
-              (0..ary.size-2).each {|i| to_keep << i} 
-            when :all_but_first
-              (1..ary.size-1).each {|i| to_keep << i} 
-            when :every_even
-              (0..ary.size).each {|i| to_keep << i if (i % 2 == 1)}
-            when :every_odd
-              (0..ary.size).each {|i| to_keep << i if (i % 2 == 0)}
-            when :every_second
-              (0..ary.size).each {|i| to_keep << i if (i % 2 == 0)}
-            when :every_third
-              (0..ary.size).each {|i| to_keep << i if (i % 3 == 0)}
-            when :every_fourth
-              (0..ary.size).each {|i| to_keep << i if (i % 4 == 0)}
+          when :first
+            to_keep << 0
+          when :last
+            to_keep << ary.size-1
+          when :all_but_last
+           (0..ary.size-2).each {|i| to_keep << i} 
+          when :all_but_first
+           (1..ary.size-1).each {|i| to_keep << i} 
+          when :every_even
+           (0..ary.size).each {|i| to_keep << i if (i % 2 == 1)}
+          when :every_odd
+           (0..ary.size).each {|i| to_keep << i if (i % 2 == 0)}
+          when :every_second
+           (0..ary.size).each {|i| to_keep << i if (i % 2 == 0)}
+          when :every_third
+           (0..ary.size).each {|i| to_keep << i if (i % 3 == 0)}
+          when :every_fourth
+           (0..ary.size).each {|i| to_keep << i if (i % 4 == 0)}
           end
         end
       }
@@ -48,7 +48,11 @@ module Scrubyt
       ary
     end
     
-private
+    #    def to_sexp
+    #      [:array, *@indices_to_extract.collect { |index| [:lit, index] }]
+    #    end
+    
+    private
     ##
     #Do not return the whole result set, just specified indices - like
     #first,last, every odd index, indices from [1..3] etc.
@@ -65,25 +69,25 @@ private
     def select_indices(*args)
       indices_to_grab = args[0]
       case indices_to_grab.class.to_s
-        when "Range"
-          @indices_to_extract = indices_to_grab.to_a
-        when "Array"
-          nested_arrays = []
-          indices_to_grab.each {|e|
-            if e.is_a? Array
-              nested_arrays << e
-            elsif e.is_a? Range
-              nested_arrays << e.to_a
-            end
-          }
-          @indices_to_extract = indices_to_grab
-          nested_arrays.each {|a| a.each {|e| @indices_to_extract << e if !@indices_to_extract.include? e }}
-          @indices_to_extract.reject! {|e| ((e.is_a? Range) || (e.is_a? Array)) }
-        when "Symbol"
-          #parse this when  we already have the results
-          @indices_to_extract = [indices_to_grab]
-        else
-          puts "Invalid index specification"
+      when "Range"
+        @indices_to_extract = indices_to_grab.to_a
+      when "Array"
+        nested_arrays = []
+        indices_to_grab.each {|e|
+          if e.is_a? Array
+            nested_arrays << e
+          elsif e.is_a? Range
+            nested_arrays << e.to_a
+          end
+        }
+        @indices_to_extract = indices_to_grab
+        nested_arrays.each {|a| a.each {|e| @indices_to_extract << e if !@indices_to_extract.include? e }}
+        @indices_to_extract.reject! {|e| ((e.is_a? Range) || (e.is_a? Array)) }
+      when "Symbol"
+        #parse this when  we already have the results
+        @indices_to_extract = [indices_to_grab]
+      else
+        puts "Invalid index specification"
       end
     end #end of function select_indices
   end #end of class ResultIndexer

data/lib/scrubyt/core/shared/evaluation_context.rb

@@ -13,15 +13,14 @@ module Scrubyt
   #two classes need to communicate frequently as well as share different information
   #and this is accomplished through EvaluationContext.
   class EvaluationContext
-    attr_accessor :root_pattern, :document_index, :block_count,
-                  :extractor, :uri_builder
+    attr_accessor :root_pattern, :document_index, :extractor, :uri_builder, :evaluating_extractor_definition
     
     def initialize
       @root_pattern = nil
       @next_page = nil
-      @block_count = 0
       @document_index = 0
       @extractor = nil
+      @evaluating_extractor_definition = false
     end
     
     ##
@@ -33,6 +32,7 @@ module Scrubyt
                         uri_builder.generate_next_uri
       return nil if temp_document == nil
       clear_sources_and_sinks(@root_pattern)
+      FetchAction.restore_host_name
       @extractor.fetch(temp_document)
       attach_current_document
     end
@@ -49,15 +49,6 @@ module Scrubyt
       @root_pattern.result.add_result(@root_pattern.filters[0].source, 
                                       @root_pattern.filters[0].sink)
     end
-
-    ##
-    #Based on the given examples, calculate the XPaths for the tree patterns    
-    def setup_examples      
-      get_root_pattern(nil)
-      mark_leaf_parents(@root_pattern)
-      generate_examples(@root_pattern)
-      check_for_multipe_examples(@root_pattern)
-    end
     
     ##
     #After crawling to the new page, the sources and sinks need to be cleaned
@@ -85,35 +76,5 @@ module Scrubyt
       end
       @uri_builder = URIBuilder.new(pattern,args)
     end
-    
-    def get_root_pattern(pattern)
-      if @root_pattern == nil
-        while (pattern.parent != nil)
-          get_root_pattern(pattern.parent)
-        end
-        @root_pattern = pattern
-      end
-    end 
-
-private
-    def mark_leaf_parents(pattern)
-      pattern.children.each { |child|
-        pattern.parent_of_leaf = true if child.children.size == 0
-      }
-      pattern.children.each { |child| mark_leaf_parents(child) }      
-    end
-    
-    ##
-    #Check the tree and turn all the XPaths for the examples (but the topmost one)
-    #into relative ones
-    def check_for_multipe_examples(pattern)
-      pattern.children.each {|child_pattern| check_for_multipe_examples(child_pattern) }
-      pattern.filters.each { |filter| filter.setup_relative_XPaths } if pattern.type == Pattern::PATTERN_TYPE_TREE    
-    end
-    
-    def generate_examples(pattern)
-      pattern.children.each {|child_pattern| generate_examples(child_pattern) }
-      pattern.filters.each { |filter| filter.generate_XPath_for_example(false) } if pattern.type == Pattern::PATTERN_TYPE_TREE
-    end #end of function generate_examples
   end #end of class EvaluationContext
 end #end of module Scrubyt

data/lib/scrubyt/core/shared/extractor.rb

@@ -1,20 +1,19 @@
-require 'open-uri'
-require 'rubygems'
-require 'mechanize'
-require 'hpricot'
-
 module Scrubyt
-##
-#=<tt>Driving the whole extraction process</tt>
-#
-#Extractor is a performer class - it gets an extractor definition and carries
-#out the actions and evaluates the wrappers sequentially. 
-#
-#Originally also the navigation actions were here, but since the class got too
-#big, they were factored out to an own class, NavigationAction.
-  class Extractor 
+  ##
+  #=<tt>Driving the whole extraction process</tt>
+  #
+  #Extractor is a performer class - it gets an extractor definition and carries
+  #out the actions and evaluates the wrappers sequentially.
+  #
+  #Originally also the navigation actions were here, but since the class got too
+  #big, they were factored out to an own class, NavigationAction.
+  class Extractor
     #The definition of the extractor is passed through this method
     def self.define(mode=nil, &extractor_definition)
+      backtrace = SharedUtils.get_backtrace
+      parts = backtrace[1].split(':')
+      source_file = parts[0]
+
       @@mode = mode
       #We are keeping the relations between the detail patterns and their root patterns
       @@detail_extractor_to_pattern_name = {}
@@ -25,17 +24,19 @@ module Scrubyt
       puts "[MODE] #{mode_name}"
       NavigationActions.new
       @@evaluation_context = EvaluationContext.new
-      #Hack up an artificial root pattern (i.e. do not return the pattern which 
+      #Hack up an artificial root pattern (i.e. do not return the pattern which
       #is the root one in the user's definition, but rather the real (invisible)
       #root pattern
-      evaluated_extractor = (class_eval(&extractor_definition))
-      if evaluated_extractor == nil
+      @@evaluation_context.evaluating_extractor_definition = true
+      class_eval(&extractor_definition)
+      @@evaluation_context.evaluating_extractor_definition = false
+      root_pattern = @@evaluation_context.root_pattern
+      if root_pattern.nil?
         puts "No extractor defined, exiting..."
         exit
       end
-      root_pattern = evaluated_extractor.parent
-      #Recursively match data based on examples
-      @@evaluation_context.setup_examples
+      root_pattern.source_file = source_file
+      root_pattern.source_proc = extractor_definition
       #Once all is set up, evaluate the extractor from the root pattern!
       evaluate_extractor(root_pattern)
       #Apply all postprocess steps
@@ -45,168 +46,126 @@ module Scrubyt
       root_pattern
     end
 
-  #Evaluate a subexttractor (i.e. an extractor on a detail page).
-  #The url passed to this function is automatically loaded.
-  #The definition of the subextractor is passed as a block
-  #
-  #!!!! THIS CODE IS A MESS, IT needs to be refactored ASAP....
-  def self.evaluate_subextractor(url, parent_pattern)
-    if @@detail_pattern_relations.keys.include? @@detail_extractor_to_pattern_name[parent_pattern.referenced_extractor]
-      detail_root = @@detail_pattern_relations[@@detail_extractor_to_pattern_name[parent_pattern.referenced_extractor]].parent
-      detail_root.result = Result.new
-      detail_root.last_result = nil
-      @@original_evaluation_context.push @@evaluation_context
-      @@evaluation_context = EvaluationContext.new
-      @@evaluation_context.clear_sources_and_sinks detail_root
-      FetchAction.restore_host_name
-      fetch url
-      @@evaluation_context.extractor = self
-      @@evaluation_context.root_pattern = detail_root      
-      @@evaluation_context.attach_current_document     
-      evaluate_extractor detail_root      
-      @@evaluation_context = @@original_evaluation_context.pop
-      detail_root.to_xml
-    else      
-      @@original_evaluation_context ||= []
-      FetchAction.restore_host_name
-      @@original_evaluation_context.push @@evaluation_context
-      @@evaluation_context = EvaluationContext.new
-      fetch url
-      evaluated_extractor = (class_eval(&parent_pattern.referenced_extractor))
-      root_pattern = evaluated_extractor.parent
-      @@detail_pattern_relations[@@detail_extractor_to_pattern_name[parent_pattern.referenced_extractor]] = root_pattern.children[0]
-      @@evaluation_context.setup_examples
-      evaluate_extractor(root_pattern)
-      #Apply all postprocess steps
-      PostProcessor.apply_post_processing(root_pattern)
-      #Return the root pattern
-      #puts "Extracted detail page"
-      @@evaluation_context = @@original_evaluation_context.pop
-      root_pattern.to_xml
+    #Evaluate a subexttractor (i.e. an extractor on a detail page).
+    #The url passed to this function is automatically loaded.
+    #The definition of the subextractor is passed as a block
+    #
+    #!!!! THIS CODE IS A MESS, IT needs to be refactored ASAP....
+    def self.evaluate_subextractor(url, parent_pattern, resolve)
+      if @@detail_pattern_relations.keys.include? @@detail_extractor_to_pattern_name[parent_pattern.referenced_extractor]
+        detail_root = @@detail_pattern_relations[@@detail_extractor_to_pattern_name[parent_pattern.referenced_extractor]].parent
+        detail_root.result = Result.new
+        detail_root.last_result = nil
+        FetchAction.store_page
+        @@original_evaluation_context.push @@evaluation_context
+        @@host_stack.push FetchAction.get_host_name
+        @@evaluation_context = EvaluationContext.new
+        @@evaluation_context.clear_sources_and_sinks detail_root
+        FetchAction.restore_host_name
+        fetch url, :resolve => resolve
+        @@evaluation_context.extractor = self
+        @@evaluation_context.root_pattern = detail_root
+        @@evaluation_context.attach_current_document
+        evaluate_extractor detail_root
+        @@evaluation_context = @@original_evaluation_context.pop
+        FetchAction.restore_page
+        FetchAction.store_host_name(@@host_stack.pop)
+        detail_root.to_xml
+      else
+        @@original_evaluation_context ||= []
+        @@host_stack ||= []
+        FetchAction.store_page
+        @@original_evaluation_context.push @@evaluation_context
+        @@host_stack.push FetchAction.get_host_name
+        @@evaluation_context = EvaluationContext.new
+        FetchAction.restore_host_name
+        fetch url, :resolve => resolve
+        evaluated_extractor = (class_eval(&parent_pattern.referenced_extractor))
+        root_pattern = evaluated_extractor.parent
+        @@detail_pattern_relations[@@detail_extractor_to_pattern_name[parent_pattern.referenced_extractor]] = root_pattern.children[0]
+        evaluate_extractor(root_pattern)
+        #Apply all postprocess steps
+        PostProcessor.apply_post_processing(root_pattern)
+        @@evaluation_context = @@original_evaluation_context.pop
+        FetchAction.restore_page
+        FetchAction.store_host_name(@@host_stack.pop)
+        root_pattern.to_xml
+      end
     end
-  end
 
-  #build the current wrapper    
-  def self.method_missing(method_name, *args, &block)
-    if NavigationActions::KEYWORDS.include? method_name.to_s
-      NavigationActions.send(method_name, *args)
-      return
-    end
-    pattern = Scrubyt::Pattern.new(method_name.to_s, *args)
-    check_if_shortcut_pattern(pattern)
-    check_if_detail_page(pattern, args)
-    pattern.evaluation_context = @@evaluation_context
-    if @parent == nil
+    #build the current wrapper
+    def self.method_missing(method_name, *args, &block)
+      if NavigationActions::KEYWORDS.include? method_name.to_s
+        NavigationActions.send(method_name, *args)
+        return
+      end
       if method_name.to_s == 'next_page'
+        pattern = Scrubyt::Pattern.new(method_name.to_s, args, @@evaluation_context)
+        pattern.evaluation_context = @@evaluation_context
+
         @@evaluation_context.setup_uri_builder(pattern, args)
         @@next_patterns[@@last_root_pattern] = @@evaluation_context.uri_builder
-        p @@last_root_pattern.children[0].name
-        return @@last_pattern
       else
+        raise "Only one root pattern allowed" if !@@evaluation_context.root_pattern.nil?
         #Create a root pattern
-        root_pattern = Scrubyt::Pattern.new('root', :type => :root) 
+        root_pattern = Scrubyt::Pattern.new('root', [:type => :root], @@evaluation_context)
         @@last_root_pattern = root_pattern
-        root_pattern.evaluation_context = @@evaluation_context
         @@evaluation_context.root_pattern = root_pattern
         @@evaluation_context.extractor = self
-        #add the currently active document to the root pattern 
+        #add the currently active document to the root pattern
         @@evaluation_context.attach_current_document
-        @@evaluation_context.root_pattern.add_child_pattern(pattern)
-        @@evaluation_context.block_count = 0
+        pattern = Scrubyt::Pattern.new(method_name.to_s, args, @@evaluation_context, root_pattern, &block)
+        root_pattern.children << pattern
+        pattern
       end
-    else
-      @parent.add_child_pattern(pattern) if @parent != nil
     end
-    if block_given?
-      @@evaluation_context.block_count = @@evaluation_context.block_count + 1
-      @stack ||=[]
-      @parent = pattern
-      @stack.push @parent
-      class_eval(&block)
-      @stack.pop
-      @parent = @stack.last
-    end    
-    @@last_pattern = pattern
-  end
-  
-  #Shortcut patterns, as their name says, are a shortcut for creating patterns
-  #from predefined rules; for example:
-  #
-  #  detail_url 
-  #  
-  #  is equivalent to
-  #  
-  #  detail_url 'href', type => :attribute
-  #  
-  #i.e. the system figures out on it's own that because of the postfix, the
-  #example should be looked up (but it should never override the user input!)
-  #another example (will be available later):
-  #
-  # every_img
-  # 
-  # is equivivalent to
-  # 
-  # every_img '//img'
-  # 
-  def self.check_if_shortcut_pattern(pattern)
-    case pattern.name
-      when /.+_url/
-        #make sure that we are not overriding the user's settings
-        if !pattern.examples
-          pattern.filters[0].example = 'href'
-          pattern.type = Scrubyt::Pattern::PATTERN_TYPE_ATTRIBUTE
-        end
+
+    def self.add_detail_extractor_to_pattern_name(referenced_extractor, pattern)
+      @@detail_extractor_to_pattern_name[referenced_extractor] ||= [] << pattern
+    end
+
+    def self.get_detail_extractor(parent_pattern)
+      @@detail_pattern_relations[@@detail_extractor_to_pattern_name[parent_pattern.referenced_extractor]].parent
+    end
+
+    def self.get_hpricot_doc
+      NavigationActions.get_hpricot_doc
+    end
+
+    def self.get_current_doc_url
+      NavigationActions.get_current_doc_url
+    end
+
+    def self.get_detail_pattern_relations
+      @@detail_pattern_relations
+    end
+
+    def self.get_host_name
+      NavigationActions.get_host_name
+    end
+
+    def self.get_mode
+      @@mode
+    end
+
+    def self.get_original_host_name
+      @@original_host_name
     end
-  end
-  
-  #Check whether the currently created pattern is a detail pattern (i.e. it refrences
-  #a subextractor). Also check if the currently created pattern is 
-  #an ancestor of a detail pattern , and store this in a hash if yes (to be able to 
-  #traverse the pattern structure on detail pages as well).
-  def self.check_if_detail_page(pattern, args)
-    return if args.size == 0
-    return if !args[0].is_a? Hash 
-    return if !args[0][:references]
-    referenced_extractor = args[0][:references]
-    pattern.type = Scrubyt::Pattern::PATTERN_TYPE_DETAIL
-    pattern.referenced_extractor = referenced_extractor
-    @@detail_extractor_to_pattern_name[referenced_extractor] ||= []
-    @@detail_extractor_to_pattern_name[referenced_extractor] = @@detail_extractor_to_pattern_name[referenced_extractor] << pattern    
-  end
-    
-  def self.get_hpricot_doc
-    NavigationActions.get_hpricot_doc
-  end
-  
-  def self.get_current_doc_url
-    NavigationActions.get_current_doc_url
-  end
-  
-  def self.get_detail_pattern_relations
-    @@detail_pattern_relations
-  end
-  
-  def self.get_mode
-    @@mode
-  end
-  
-private
+
+    private
+
     def self.evaluate_extractor(root_pattern)
       if @@next_patterns[root_pattern]
         current_page_count = 1
         loop do
-          really_evaluate_extractor(root_pattern)
-          break if (@@next_patterns[root_pattern].limit == current_page_count || @@evaluation_context.crawl_to_new_page(root_pattern, @@next_patterns[root_pattern]) == nil)
+          root_pattern.evaluate(nil)
+          break if (@@next_patterns[root_pattern].limit == current_page_count || !@@evaluation_context.crawl_to_new_page(root_pattern, @@next_patterns[root_pattern]))
           current_page_count += 1 if @@next_patterns[root_pattern].limit != nil
         end
-      else 
-        really_evaluate_extractor(root_pattern)
+      else
+        root_pattern.evaluate(nil)
       end
-    end  
-    
-    def self.really_evaluate_extractor(pattern)
-      pattern.evaluate
-      pattern.children.each { |child| really_evaluate_extractor child } 
-    end #end of method evaluate_wrapper    
+    end
+
   end #end of class Extractor
-end #end of module Scrubyt
+end #end of module Scrubyt