scrubyt 0.1.0 → 0.1.9

data/lib/scrubyt.rb

@@ -7,3 +7,4 @@ require 'scrubyt/pattern.rb'
 require 'scrubyt/result_dumper.rb'
 require 'scrubyt/result.rb'
 require 'scrubyt/xpathutils.rb'
+require 'scrubyt/post_processor.rb'

data/lib/scrubyt/constraint.rb

@@ -36,12 +36,11 @@ module Scrubyt
     #2b) Do it on the XML level - most probably this solution will be implemented
     
     # Different constraint types
-    CONSTRAINT_TYPE_ENSURE_PRESENCE_OF_ANCESTOR_PATTERN = 0
-    CONSTRAINT_TYPE_ENSURE_ABSENCE_OF_ANCESTOR_PATTERN = 1
-    CONSTRAINT_TYPE_ENSURE_PRESENCE_OF_ATTRIBUTE = 2
-    CONSTRAINT_TYPE_ENSURE_ABSENCE_OF_ATTRIBUTE = 3
-    CONSTRAINT_TYPE_ENSURE_PRESENCE_OF_ANCESTOR_NODE = 4
-    CONSTRAINT_TYPE_ENSURE_ABSENCE_OF_ANCESTOR_NODE = 5
+    CONSTRAINT_TYPE_ENSURE_PRESENCE_OF_PATTERN = 0
+    CONSTRAINT_TYPE_ENSURE_PRESENCE_OF_ATTRIBUTE = 1
+    CONSTRAINT_TYPE_ENSURE_ABSENCE_OF_ATTRIBUTE = 2
+    CONSTRAINT_TYPE_ENSURE_PRESENCE_OF_ANCESTOR_NODE = 3
+    CONSTRAINT_TYPE_ENSURE_ABSENCE_OF_ANCESTOR_NODE = 4
     
     
     attr_reader :type, :target, :parent_filter
@@ -52,22 +51,12 @@ module Scrubyt
     #(child pattern, or child pattern of a child pattern, etc.) denoted by "ancestor"
     #'Has an ancestor pattern' means that the ancestor pattern actually extracts something
     #(just by looking at the wrapper model, the ancestor pattern is always present)
-    #ON result level!!!
-    def self.add_ensure_presence_of_ancestor_pattern(parent_filter, ancestor)
-      Constraint.new(parent_filter, ancestor, CONSTRAINT_TYPE_ENSURE_PRESENCE_OF_ANCESTOR_PATTERN)
+    #Note that from this type of constraint there is no 'ensure_absence' version, since
+    #I could not think about an use case for that 
+    def self.add_ensure_presence_of_pattern(parent_filter, ancestor)
+      Constraint.new(parent_filter, ancestor, CONSTRAINT_TYPE_ENSURE_PRESENCE_OF_PATTERN)
     end
     
-    #Add 'ensure presence of ancestor pattern' constraint
-    
-    #If this type of constraint is added to a pattern, it must NOT have an ancestor pattern
-    #(child pattern, or child pattern of a child pattern, etc.) denoted by "ancestor"
-    #'Has an ancestor pattern' means that the ancestor pattern actually extracts something
-    #(just by looking at the wrapper model, the ancestor pattern is always present)
-    #ON result level!!!
-    def self.add_ensure_absence_of_ancestor_pattern(parent_filter, ancestor)
-      Constraint.new(parent_filter, ancestor, CONSTRAINT_TYPE_ENSURE_ABSENCE_OF_ANCESTOR_PATTERN)
-    end  
-    
     #Add 'ensure absence of attribute' constraint
   
     #If this type of constraint is added to a pattern, the HTML node it targets
@@ -127,10 +116,9 @@ module Scrubyt
     #content of the pattern
     def check(result)
       case @type
-        when CONSTRAINT_TYPE_ENSURE_PRESENCE_OF_ANCESTOR_PATTERN
-          puts "CONSTRAINT_TYPE_ENSURE_PRESENCE_OF_ANCESTOR_PATTERN"
-        when CONSTRAINT_TYPE_ENSURE_ABSENCE_OF_ANCESTOR_PATTERN
-          puts "CONSTRAINT_TYPE_ENSURE_ABSENCE_OF_ANCESTOR_PATTERN"
+        #checked after evaluation, so here always return true
+        when CONSTRAINT_TYPE_ENSURE_PRESENCE_OF_PATTERN
+          return true
         when CONSTRAINT_TYPE_ENSURE_PRESENCE_OF_ATTRIBUTE
           attribute_present(result)
         when CONSTRAINT_TYPE_ENSURE_ABSENCE_OF_ATTRIBUTE

data/lib/scrubyt/constraint_adder.rb

@@ -11,15 +11,8 @@ module Scrubyt
   #functions with their documentation in Scrubyt::Constraint.rb
   class ConstraintAdder
   
-    def self.ensure_presence_of_ancestor_pattern(pattern, ancestor_node_name)    
-      data = self.prepare_ensure_ancestor_pattern(pattern, sym_root, sym_ancestor)
-      pattern.filters[0].ensure_presence_of_ancestor_pattern(ancestor_node_name)
-      pattern #To make chaining possible
-    end
-    
-    def self.ensure_absence_of_ancestor_pattern(pattern, ancestor_node_name)
-      data = self.prepare_ensure_ancestor_pattern(pattern, sym_root, sym_ancestor)
-      pattern.filters[0].ensure_absence_of_ancestor_pattern(ancestor_node_name)
+    def self.ensure_presence_of_pattern(pattern, ancestor_node_name)    
+      pattern.filters[0].ensure_presence_of_pattern(ancestor_node_name)
       pattern #To make chaining possible
     end
     
@@ -74,13 +67,6 @@ private
         end
       end
       return attribute_pairs
-    end    
-    
-    def self.prepare_ensure_ancestor_pattern(pattern, root, ancestor)
-      context_pattern = find_by_name(pattern.root_pattern, root)
-      target_pattern = find_by_name(pattern.root_pattern, ancestor)
-      return [context_pattern, target_pattern]    
-    end
-  
+    end #end of method prepare_attributes      
   end #end of class ConstraintAddere
 end #end of module Scrubyt

data/lib/scrubyt/export.rb

@@ -80,7 +80,7 @@ module Scrubyt
 
 private
     def self.export_header(output_file)
-      @result += "require 'lib/extractor.rb'\n\n"
+      @result += "require 'rubygems'\nrequire 'scrubyt'\n\n"
     end
     
     def self.cleanup_result
@@ -142,19 +142,21 @@ private
       @name_to_xpath_map = {}
       create_name_to_xpath_map(pattern)
       #Replace the examples which are quoted with " and '
-      @name_to_xpath_map.each do |name, xpath| 
-        replace_example_with_xpath(name, xpath, %q{"})
-        replace_example_with_xpath(name, xpath, %q{'})
+      @name_to_xpath_map.each do |name, xpaths| 
+        replace_example_with_xpath(name, xpaths, %q{"})
+        replace_example_with_xpath(name, xpaths, %q{'})
       end
       #Finally, add XPaths to pattern which had no example at the beginning (the XPath was
       #generated from the child patterns
-      @name_to_xpath_map.each do |name, xpath|  
-        comma = @full_definition.scan(Regexp.new("P.#{name}(.+)$"))[0][0].sub('do'){}.strip == '' ? '' : ','
-        if (@full_definition.scan(Regexp.new("P.#{name}(.+)$"))[0][0]).include?('{')
-          @full_definition.sub!("P.#{name}") {"P.#{name}('#{xpath}')"}
-        else
-          @full_definition.sub!("P.#{name}") {"P.#{name} \"#{xpath}\"#{comma}"}
-        end      
+      @name_to_xpath_map.each do |name, xpaths| 
+        xpaths.each do |xpath|
+          comma = @full_definition.scan(Regexp.new("P.#{name}(.+)$"))[0][0].sub('do'){}.strip == '' ? '' : ','
+          if (@full_definition.scan(Regexp.new("P.#{name}(.+)$"))[0][0]).include?('{')
+            @full_definition.sub!("P.#{name}") {"P.#{name}('#{xpath}')"}
+          else
+            @full_definition.sub!("P.#{name}") {"P.#{name} \"#{xpath}\"#{comma}"}
+          end
+        end
       end
       @result += @full_definition
     end    
@@ -169,18 +171,32 @@ private
 
     
     def self.create_name_to_xpath_map(pattern)
-      @name_to_xpath_map[pattern.name] = pattern.filters[0].xpath if pattern.filters[0].xpath != nil
+      @name_to_xpath_map[pattern.name] = []
+      pattern.filters.each do |filter|
+        @name_to_xpath_map[pattern.name] << filter.xpath if pattern.filters[0].xpath != nil
+      end
       pattern.children.each {|child| create_name_to_xpath_map child}      
     end
     
-    def self.replace_example_with_xpath(name, xpath, left_delimiter, right_delimiter=left_delimiter)
-      replacing_xpath = (@full_definition.scan(Regexp.new("P.#{name}(.+)$"))[0][0]).include?('{') ? 
-              "P.#{name}('\"#{xpath}\"')" :
-              "P.#{name} \"#{xpath}\""
+    def self.replace_example_with_xpath(name, xpaths, left_delimiter, right_delimiter=left_delimiter)
+      return if name=='root'
+      full_line = @full_definition.scan(Regexp.new("P.#{name}(.+)$"))[0][0]
+      examples = full_line.split(",")
+      examples.reject! {|exa| exa.strip!;  exa[0..0] != %q{"} && exa[0..0] != %q{'} }
+      all_xpaths = ""
+      examples.each do |e|
+        index = examples.index(e)
+        xpath = xpaths[index]
+        return if xpath == nil
+        all_xpaths += ", " if index > 0
+        all_xpaths += '"' + xpath + '"'
+      end
+      replacing_xpath = full_line.include?('{') ? "P.#{name}('#{all_xpaths}')" :
+                                                  "P.#{name} #{all_xpaths}"
       @full_definition.sub!(/P\.#{name}\s+#{left_delimiter}(.*)#{right_delimiter}/) do
         @name_to_xpath_map.delete("#{name}")
         replacing_xpath
-      end
+      end                                                  
     end
     
   end

data/lib/scrubyt/extractor.rb

@@ -4,6 +4,7 @@ require 'rubygems'
 require 'mechanize'
 require 'hpricot'
 require 'pp'
+require 'set'
 
 module Scrubyt
 ##
@@ -43,6 +44,8 @@ module Scrubyt
       else 
         evaluate_wrapper(root_pattern)
       end
+      ensure_all_postconditions(root_pattern)
+      PostProcessor.remove_multiple_filter_duplicates(root_pattern)
       #Return the root pattern
       root_pattern
     end
@@ -104,39 +107,35 @@ module Scrubyt
   
   ##
   #Action to fetch a document (either a file or a http address)
-  #
+  # 
   #*parameters*
   #
   #_doc_url_ - the url or file name to fetch
   def self.fetch(doc_url, mechanize_doc=nil)
-    puts "fetching: #{doc_url}"
     if (mechanize_doc == nil)
       @@current_doc_url = doc_url    
       @@current_doc_protocol = ((doc_url =~ /^http/ || doc_url =~ /^www/) ? :http : :file)
       if @@base_dir == nil
         @@base_dir = doc_url.scan(/.+\//)[0] if @@current_doc_protocol == :file
-      else
+      else        
         @@current_doc_url = ((@@base_dir + doc_url) if doc_url !~ /#{@@base_dir}/)
       end
       
-      if @@host_name == nil
-        if @@current_doc_protocol == :http
-          @@host_name = doc_url.scan(/http:\/\/.+?\//)[0] 
-          @@host_name = doc_url if @@host_name == nil
-        end
-      else
-        @@current_doc_url = (@@host_name + doc_url) if doc_url !~ /#{@@host_name}/
+      if @@host_name != nil
+        if doc_url !~ /#{@@host_name}/
+          @@current_doc_url = (@@host_name + doc_url) 
+          @@current_doc_url.gsub!(/([^:])\/\//) {"#{$1}/"}
+        end  
       end
-      
+      puts "[ACTION] fetching document: #{@@current_doc_url}"
       @@mechanize_doc = @@agent.get(@@current_doc_url) if @@current_doc_protocol == :http
     else
       @@current_doc_url = doc_url
       @@mechanize_doc = mechanize_doc
+      @@host_name = 'http://' + @@mechanize_doc.uri.to_s.scan(/http:\/\/(.+\/)+/).flatten[0]
+      @@host_name = doc_url if @@host_name == nil    
     end
-    @@hpricot_doc = mechanize_doc != nil ? Hpricot(@@mechanize_doc.body) : Hpricot(open(@@current_doc_url))
-    out = open('kamaty.html', 'w')
-    out.write @@hpricot_doc.to_s
-    out.close
+    @@hpricot_doc = Hpricot(open(@@current_doc_url))#.to_original_html
   end
   
   ##
@@ -149,7 +148,7 @@ module Scrubyt
   #
   #_query_string_ - the string that should be entered into the textfield
   def self.fill_textfield(textfield_name, query_string)
-    puts 'fill textfield'
+    puts "[ACTION] typing #{query_string} into the textfield named '#{textfield_name}'"
     textfield = (@@hpricot_doc/"input[@name=#{textfield_name}]").map()[0]
     formname = Scrubyt::XPathUtils.traverse_up_until_name(textfield, 'form').attributes['name']
     @@current_form = @@mechanize_doc.forms.with.name(formname).first
@@ -158,16 +157,16 @@ module Scrubyt
   
   #Submit the last form; 
   def self.submit    
-    puts 'submit'
+    puts '[ACTION] submitting form...'
     result_page = @@agent.submit(@@current_form)#, @@current_form.buttons.first)
     @@current_doc_url = result_page.uri.to_s
     fetch(@@current_doc_url, result_page)
   end
   
-  def self.click_link(link_text)    
-    puts 'click link'
-    puts /^#{Regexp.escape(link_text)}$/
-    p /^#{Regexp.escape(link_text)}$/
+  def self.click_link(link_text)
+    puts "[ACTION] clicking link: #{link_text}"
+    #puts /^#{Regexp.escape(link_text)}$/
+    #p /^#{Regexp.escape(link_text)}$/
     link = @@mechanize_doc.links.text(/^#{Regexp.escape(link_text)}$/)
     result_page = @@agent.click(link)
     @@current_doc_url = result_page.uri.to_s
@@ -178,10 +177,62 @@ module Scrubyt
 #############  
         
 private
+  def self.ensure_all_postconditions(pattern)
+    ensure_postconditions(pattern)
+    pattern.children.each {|child| ensure_all_postconditions(child)}
+  end
+  
+  def self.ensure_postconditions(pattern)
+    #holds the name of those child patterns which have to be present as children of the input parameter  
+    epop_names = pattern.get_constraints.select {|c| c.type == Scrubyt::Constraint::CONSTRAINT_TYPE_ENSURE_PRESENCE_OF_PATTERN}.map {|c| c.target}
+    return if epop_names.empty?
+    #all_parent_values holds instances extracted by pattern    
+    all_parent_values = []
+    pattern.result.childmap.each { |h| all_parent_values << h.values }
+    all_parent_values.flatten!
+    #indices of result instances (of pattern) we are going to remove
+    results_to_remove = Set.new
+    pattern.children.each do |child_pattern|
+      #all_child_values holds instances extracted by child_pattern
+      all_child_values = []
+      child_pattern.result.childmap.each { |h| all_child_values << h.values }
+      all_child_values.flatten!
+    
+      #populate results_to_remove
+      i = 0      
+      all_parent_values.each do |parent_value|
+        #Hey! Not just the direct children but all the ancestors
+        @found_ancestor = false
+        check_ancestors(parent_value, all_child_values)
+        
+        results_to_remove << i if (!@found_ancestor && (epop_names.include? child_pattern.name))
+        i += 1
+      end
+    end   
+    #based on results_to_remove, populate the array 'rejected' which holds the actual instances
+    #(and not indices, as in the case of results_to_remove!). In other words, we are mapping 
+    #results_to_remove indices to their actual instances
+    rejected = []    
+    i = -1 
+    pattern.result.childmap.each do |h|
+      h.each { |k,v| rejected = v.reject {|e| i += 1; !results_to_remove.include? i } }
+    end    
+    
+    #Correct the statistics
+    pattern.get_instance_count[pattern.name] -= rejected.size
+    
+    #Finally, do the actual delete!
+    pattern.result.childmap.each { |h| h.each { |k,v| rejected.each  { |r| v.delete(r)} } }    
+  end
+  
+  def self.check_ancestors(parent_value, all_child_values)
+    parent_value.children.each { |child| @found_ancestor = true if all_child_values.include? child }
+    parent_value.children.each { |child| check_ancestors(child, all_child_values) if child.is_a? Hpricot::Elem }
+  end
+  
     def self.evaluate_wrapper(pattern)
       pattern.evaluate
       pattern.children.each { |child| evaluate_wrapper child } 
-    end
-    
+    end #end of method evaluate_wrapper    
   end #end of class Extractor
 end #end of module Scrubyt

data/lib/scrubyt/filter.rb

@@ -2,7 +2,7 @@ module Scrubyt
   ##
   #=<tt>Filter out relevant pieces from the parent pattern</tt>
   #
-  #A Scrubyt wrapper is almost like a waterfall: water is pouring from the top until
+  #A Scrubyt extractor is almost like a waterfall: water is pouring from the top until
   #it reaches the bottom. The biggest difference is that instead of water, a HTML 
   #document travels through the space.
   #
@@ -15,12 +15,12 @@ module Scrubyt
   #The working of a filter will be explained most easily by the help of an example.
   #Let's consider that we would like to extract information from a webshop; Concretely
   #we are interested in the name of the items and the URL pointing to the image of the 
-  #item
+  #item.
   #
-  #To accomplish this. first we select the items with the pattern item (a pattern is 
+  #To accomplish this, first we select the items with the pattern item (a pattern is 
   #a logical grouping of fillters; see Pattern documentation) Then our new 
-  #context is the result extracted by the item pattern; For every pattern, further
-  #extract the name and the image of the item; and finally, extractr the href attribute
+  #context is the result extracted by the 'item' pattern; For every 'item' pattern, further
+  #extract the name and the image of the item; and finally, extract the href attribute
   #of the image. Let's see an illustration:
   #
   #   root             --> This pattern is called a 'root pattern', It is invisible to you
@@ -46,15 +46,18 @@ module Scrubyt
     #Regexp example, like /\d+@*\d+[a-z]/
     EXAMPLE_TYPE_REGEXP = 4
     
-    attr_accessor :example_type, :parent_pattern, :temp_sink, :constraints, :xpath, :regexp
+    attr_accessor :example_type, :parent_pattern, :temp_sink, 
+                  :constraints, :xpath, :regexp, :example, :source, :sink
 
-    def initialize(parent_pattern, *args)
+    def initialize(parent_pattern, example=nil, *args)
       @parent_pattern = parent_pattern
       #If the example type is not explicitly defined in the pattern definition,
       #try to determine it automatically from the example
-      @example_type = (args[0] == nil ? Filter.determine_example_type(parent_pattern.example) :
+      @example_type = (args[0] == nil ? Filter.determine_example_type(example) :
                                         args[0][:example_type])
-      @regexp = parent_pattern.example if @example_type == EXAMPLE_TYPE_REGEXP
+      @sink = []                  #output of a filter
+      @source = []                #input of a filter                                        
+      @example = example
       @xpath = nil #The xpath to evaluate this filter
       #temp sinks are used for the initial run when determining the XPaths for examples;
       @temp_sink = nil 
@@ -64,14 +67,15 @@ module Scrubyt
     #Evaluate this filter. This method shoulf not be called directly - as the pattern hierarchy 
     #is evaluated, every pattern evaluates its filters and then they are calling this method
     def evaluate(source)
+      @parent_pattern.root_pattern.already_evaluated_sources ||= {}
       case @parent_pattern.type
-        when Scrubyt::Pattern::PATTERN_TYPE_TREE
+        when Scrubyt::Pattern::PATTERN_TYPE_TREE      
           result = source/@xpath
           result.class == Hpricot::Elements ? result.map : [result]
         when Scrubyt::Pattern::PATTERN_TYPE_ATTRIBUTE  
-          [source.attributes[@parent_pattern.example]]
+          [source.attributes[@example]]
         when Scrubyt::Pattern::PATTERN_TYPE_REGEXP 
-          source.inner_text.scan(@regexp).flatten
+          source.inner_text.scan(@example).flatten
       end      
     end
              
@@ -81,38 +85,49 @@ module Scrubyt
     def generate_XPath_for_example
       case @example_type
         when EXAMPLE_TYPE_XPATH
-          @xpath = @parent_pattern.example
+          @xpath = @example
         when EXAMPLE_TYPE_STRING
-          @temp_sink = XPathUtils.find_node_from_text( @parent_pattern.root_pattern.source[0], @parent_pattern.example )
+          @temp_sink = XPathUtils.find_node_from_text( @parent_pattern.root_pattern.filters[0].source[0], @example )
           @xpath = @parent_pattern.generalize ? XPathUtils.generate_XPath(@temp_sink, nil, false) :
                                                  XPathUtils.generate_XPath(@temp_sink, nil, true)
-        when EXAMPLE_TYPE_CHILDREN
-          all_child_temp_sinks = []
-          @parent_pattern.children.each do |child_pattern|
-            child_pattern.filters.each do |filter|
-              all_child_temp_sinks << filter.temp_sink
+          puts @xpath                                                 
+        when EXAMPLE_TYPE_CHILDREN          
+          current_example_index = 0
+          loop do
+            all_child_temp_sinks = []
+            @parent_pattern.children.each do |child_pattern|
+              all_child_temp_sinks << child_pattern.filters[current_example_index].temp_sink
             end
-          end    
-          result = all_child_temp_sinks.pop
-          if all_child_temp_sinks.empty?
-            result = result.parent
-          else
-            all_child_temp_sinks.each do |child_sink|              
-              result = XPathUtils.lowest_common_ancestor(result, child_sink)
+            
+            result = all_child_temp_sinks.pop
+            if all_child_temp_sinks.empty?
+              result = result.parent
+            else
+              all_child_temp_sinks.each do |child_sink|              
+                result = XPathUtils.lowest_common_ancestor(result, child_sink)
+              end
             end
+            xpath = @parent_pattern.generalize ? XPathUtils.generate_XPath(result, nil, false) :
+                                                   XPathUtils.generate_XPath(result, nil, true)
+            if @parent_pattern.filters.size < current_example_index + 1
+              @parent_pattern.filters << Scrubyt::Filter.new(@parent_pattern)
+            end  
+            @parent_pattern.filters[current_example_index].xpath = xpath
+            @parent_pattern.filters[current_example_index].temp_sink = result
+            @parent_pattern.children.each do |child_pattern|
+                  child_pattern.filters[current_example_index].xpath = 
+                    child_pattern.generalize ? XPathUtils.generate_generalized_relative_XPath(child_pattern.filters[current_example_index].temp_sink, result) :
+                                               XPathUtils.generate_relative_XPath(child_pattern.filters[current_example_index].temp_sink, result)                                            
+            end                                                
+            if @parent_pattern.children[0].examples == nil
+              break if @parent_pattern.children[0].filters.size == current_example_index+1
+            else
+              break if @parent_pattern.children[0].examples.size == current_example_index+1
+            end             
+            current_example_index += 1
           end
-          @temp_sink = result
-          @xpath = @parent_pattern.generalize ? XPathUtils.generate_XPath(@temp_sink, nil, false) :
-                                                 XPathUtils.generate_XPath(@temp_sink, nil, true)
-          @parent_pattern.children.each do |child_pattern|
-            child_pattern.filters.each do |filter|
-                filter.xpath = 
-                  child_pattern.generalize ? XPathUtils.generate_generalized_relative_XPath(filter.temp_sink, result) :              
-                                             XPathUtils.generate_relative_XPath(filter.temp_sink, result)                                            
-            end
-          end          
         when EXAMPLE_TYPE_IMAGE
-          @temp_sink = XPathUtils.find_image(@parent_pattern.root_pattern.source[0], @parent_pattern.example)
+          @temp_sink = XPathUtils.find_image(@parent_pattern.root_pattern.filters[0].source[0], @example)
           @xpath = XPathUtils.generate_XPath(@temp_sink, nil, false)
       end
     end