scrubyt 0.2.0 → 0.2.3

data/lib/scrubyt/post_processor.rb

@@ -1,73 +0,0 @@
-module Scrubyt
-##
-#=<tt>Post processing results after the extraction</tt>
-#Some things can not be carried out during evaluation - for example 
-#the ensure_presence_of_pattern constraint (since the evaluation is top
-#to bottom, at a given point we don't know yet whether the currently
-#evaluated pattern will have a child pattern or not) or removing unneeded
-#results caused by evaluating multiple filters.
-#
-#The sole purpose of this class is to execute these post-processing tasks.
-  class PostProcessor
-    ##
-    #Remove unneeded results of a pattern (caused by evaluating multiple filters)
-    #See for example the B&N scenario - the book titles are extracted two times
-    #for every pattern (since both examples generate the same XPath for them)
-    #but since always only one of the results has a price, the other is discarded
-    def self.remove_multiple_filter_duplicates(pattern)
-      remove_multiple_filter_duplicates_intern(pattern) if pattern.parent_of_leaf
-      pattern.children.each {|child| remove_multiple_filter_duplicates(child)}
-    end
-    
-    ##
-    #Issue an error report if the document did not extract anything.
-    #Probably this is because the structure of the page changed or 
-    #because of some rather nasty bug - in any case, something wrong 
-    #is going on, and we need to inform the user about this!
-    def self.report_if_no_results(root_pattern)
-      results_found = false
-      root_pattern.children.each {|child| return if (child.result.childmap.size > 0)}
-      puts
-      puts "!!!!!! WARNING: The extractor did not find any result instances"
-      puts "Most probably this is wrong. Check your extractor and if you are"
-      puts "sure it should work, report a bug!"
-      puts
-    end
-  
-private  
-    def self.remove_multiple_filter_duplicates_intern(pattern)
-      possible_duplicates = {}
-      longest_result = 0
-      pattern.result.childmap.each { |r|
-        r.each do |k,v|
-          v.each do |x|
-            all_child_results = []
-            pattern.children.each { |child| 
-              temp_res = child.result.lookup(x) 
-              all_child_results << temp_res if temp_res != nil
-            }
-            next if all_child_results.size <= 1
-            longest_result = all_child_results.map {|e| e.size}.max
-            all_child_results.each { |r| (r.size+1).upto(longest_result) { r << nil } }
-            possible_duplicates[x] = all_child_results.transpose
-          end            
-        end
-      }
-      #Determine the 'real' duplicates
-      real_duplicates = {}
-      possible_duplicates.each { |k,v|
-        next if v.size == 1
-        v.each { |r| real_duplicates[k] = r }
-      }
-      
-      #Finally, remove them!
-      pattern.children.each { |child| 
-        child.result.childmap.each { |r|
-          r.each { |k,v|     
-           real_duplicates[k].each {|e| v.delete e} if real_duplicates.keys.include? k
-          }
-        }
-      }
-    end    
-  end
-end