scrubyt 0.2.6 → 0.2.8

data/lib/scrubyt/core/scraping/filters/html_subtree_filter.rb

@@ -0,0 +1,7 @@
+module Scrubyt
+  class HTMLSubTreeFilter < BaseFilter
+ 
+    def evaluate(source)
+    end #end of method
+  end #End of class TreeFilter
+end #End of module Scrubyt

data/lib/scrubyt/core/scraping/filters/regexp_filter.rb

@@ -0,0 +1,17 @@
+module Scrubyt
+  class RegexpFilter < BaseFilter
+    
+    def evaluate(source)
+      if source.is_a? String
+        source.scan(@example).flatten
+      else
+        source.inner_text.scan(@example).flatten
+      end    
+    end
+    
+    def to_sexp
+      [:lit, @example]
+    end
+    
+  end #End of class TreeFilter
+end #End of module Scrubyt

data/lib/scrubyt/core/scraping/filters/tree_filter.rb

@@ -0,0 +1,121 @@
+module Scrubyt
+  class TreeFilter < BaseFilter
+
+    def evaluate(source)
+      result = source/@xpath
+      #puts "Evaluating #{@parent_pattern.name} with #{@xpath}"
+      xpath_results = if result.class == Hpricot::Elements then result.map else [result] end
+      if @regexp == nil
+        xpath_results
+      else
+        regexp_results = []
+        xpath_results.each do |entry|
+          text = SharedUtils.prepare_text_for_comparison(result.inner_html)
+          if text =~ @regexp
+            regexp_results << $1
+          end
+        end
+        regexp_results
+      end
+    end
+
+    def generate_regexp_for_example
+      return if @example_type != EXAMPLE_TYPE_STRING
+      return if @temp_sink.nil?
+
+      text = SharedUtils.prepare_text_for_comparison(@temp_sink.inner_text)
+      match_range = @temp_sink.match_data.begin(0)..@temp_sink.match_data.end(0)
+      return if match_range == (0..text.length)
+
+      @regexp = text
+      @temp_sink.changing_ranges.sort.reverse.each do |range|
+        @regexp[range] = if range == match_range then '<<<regexp_selection>>>' else '<<<regexp_changing>>>' end
+      end
+      @regexp = Regexp.escape(@regexp)
+      @regexp = @regexp.gsub('<<<regexp_changing>>>', '.*?')
+      @regexp = @regexp.gsub('<<<regexp_selection>>>', '(.*?)')
+      @regexp = '^' + @regexp + '$'
+      @regexp = /#{@regexp}/
+    end
+
+
+    #For all the tree patterns, generate an XPath based on the given example
+    #Also this method should not be called directly; It is automatically called for every tree
+    #pattern directly after wrapper definition
+    def generate_XPath_for_example(next_page_example=false)
+      #puts "generating example for: #{@parent_pattern.name}"
+      #puts @example_type
+      case @example_type
+      when EXAMPLE_TYPE_XPATH
+        @xpath = @example
+      when EXAMPLE_TYPE_STRING
+        @temp_sink = SimpleExampleLookup.find_node_from_text(@parent_pattern.evaluation_context.root_pattern.filters[0].source[0],
+                                                             @example,
+                                                             next_page_example)
+        return if @temp_sink == nil
+
+        mark_changing_ranges = lambda { |element, range|
+          element.instance_eval do
+            @changing_ranges ||= [] << range
+            def changing_ranges
+              @changing_ranges
+            end
+          end
+        }
+        mark_changing_ranges.call(@temp_sink, @temp_sink.match_data.begin(0)..@temp_sink.match_data.end(0))
+
+        @xpath = XPathUtils.generate_XPath(@temp_sink, nil, !@parent_pattern.generalize)
+      when EXAMPLE_TYPE_CHILDREN
+        current_example_index = 0
+        loop do
+          all_child_temp_sinks = []
+          @parent_pattern.children.each do |child_pattern|
+            all_child_temp_sinks << child_pattern.filters[current_example_index].temp_sink
+          end
+          result = all_child_temp_sinks.pop
+          if all_child_temp_sinks.empty?
+            result = result.parent
+          else
+            all_child_temp_sinks.each do |child_sink|
+              result = XPathUtils.lowest_common_ancestor(result, child_sink)
+            end
+          end
+          xpath = @parent_pattern.generalize ? XPathUtils.generate_XPath(result, nil, false) :
+          XPathUtils.generate_XPath(result, nil, true)
+          if @parent_pattern.filters.size < current_example_index + 1
+            @parent_pattern.filters << Scrubyt::BaseFilter.create(@parent_pattern)
+          end
+          @parent_pattern.filters[current_example_index].xpath = xpath
+          @parent_pattern.filters[current_example_index].temp_sink = result
+          @parent_pattern.children.each do |child_pattern|
+          next if child_pattern.type == :detail_page
+            child_pattern.filters[current_example_index].xpath =
+            child_pattern.generalize ? XPathUtils.generate_generalized_relative_XPath(child_pattern.filters[current_example_index].temp_sink, result) :
+            XPathUtils.generate_relative_XPath(child_pattern.filters[current_example_index].temp_sink, result)
+          end
+          break if @parent_pattern.children[0].filters.size == current_example_index + 1
+          current_example_index += 1
+        end
+      when EXAMPLE_TYPE_IMAGE
+        #@temp_sink = XPathUtils.find_image(@parent_pattern.evaluation_context.root_pattern.filters[0].source[0], @example)\
+        @temp_sink = XPathUtils.find_image(@parent_pattern.evaluation_context.extractor.get_hpricot_doc, @example)
+        @xpath = XPathUtils.generate_XPath(@temp_sink, nil, false)
+      when EXAMPLE_TYPE_COMPOUND
+        @temp_sink = CompoundExampleLookup.find_node_from_compund_example(@parent_pattern.evaluation_context.root_pattern.filters[0].source[0],
+                                                                          @example,
+                                                                          next_page_example)
+        @xpath = @parent_pattern.generalize ? XPathUtils.generate_XPath(@temp_sink, nil, false) :
+        XPathUtils.generate_XPath(@temp_sink, nil, true)
+      end
+    end
+
+    def generate_relative_XPath(parent_xpath)
+      @xpath = XPathUtils.generate_relative_XPath_from_XPaths(parent_xpath, @xpath) if (@xpath =~ /^\/html/) #TODO: should not rely on <html> being the root node
+    end
+
+    def to_sexp
+      [:str, @xpath]
+    end
+
+  end #End of class TreeFilter
+end #End of module Scrubyt

data/lib/scrubyt/core/scraping/pattern.rb

@@ -5,129 +5,234 @@ module Scrubyt
   ##
   #=<tt>Group more filters into one</tt>
   #
-  #Server as an umbrella for filters which are conceptually extracting 
+  #Server as an umbrella for filters which are conceptually extracting
   #the same thing - for example a price or a title or ...
   #
-  #Sometimes the same piece of information can not be extracted with one filter 
+  #Sometimes the same piece of information can not be extracted with one filter
   #across more result instances (for example a price has an XPath in record n,
   #but since in record n+1 has a discount price as well, the real price is pushed
   #to a different XPath etc) - in this case the more filters which extract the same
   #thing are hold in the same pattern.
   class Pattern
-    #Type of the pattern; 
-
-    # a root pattern represents a (surprise!) root pattern
-    PATTERN_TYPE_ROOT = 0x00
-    # a tree pattern represents a HTML region
-    PATTERN_TYPE_TREE = 0x01
-    # represents an attribute of the node extracted by the parent pattern
-    PATTERN_TYPE_ATTRIBUTE = 0x02
-    # represents a pattern which filters its output with a regexp
-    PATTERN_TYPE_REGEXP = 0x03
-    # represents a pattern which crawls to the detail page and extracts information from there
-    PATTERN_TYPE_DETAIL = 0x04
+    #Type of the pattern;
+
+    # TODO: Update documentation
+
+    #    # a root pattern represents a (surprise!) root pattern
+    #    PATTERN_TYPE_ROOT = :PATTERN_TYPE_ROOT
+    #    # a tree pattern represents a HTML region
+    #    PATTERN_TYPE_TREE = :PATTERN_TYPE_TREE
+    #    # represents an attribute of the node extracted by the parent pattern
+    #    PATTERN_TYPE_ATTRIBUTE = :PATTERN_TYPE_ATTRIBUTE
+    #    # represents a pattern which filters its output with a regexp
+    #    PATTERN_TYPE_REGEXP = :PATTERN_TYPE_REGEXP
+    #    # represents a pattern which crawls to the detail page and extracts information from there
+    #    PATTERN_TYPE_DETAIL_PAGE = :PATTERN_TYPE_DETAIL_PAGE
+    #    # represents a download pattern
+    #    PATTERN_TYPE_DOWNLOAD = :PATTERN_TYPE_DOWNLOAD
+    #    # write out the HTML subtree beginning at the matched element
+    #    PATTERN_TYPE_HTML_SUBTREE = :PATTERN_TYPE_HTML_SUBTREE
+
+    VALID_PATTERN_TYPES = [:root, :tree, :attribute, :regexp, :detail_page, :download, :html_subtree]
 
     #The pattern can be either a model pattern (in this case it is
     #written to the output) or a temp pattern (in this case it is skipped)
     #Will be implemented in a higher version (i.e. not 0.1.0) - for now, everything
     #is considered to be a model pattern
-    
+
     #Model pattern are shown in the output
-    OUTPUT_TYPE_MODEL = 0x10
-    #Temp patterns are skipped in the output (their ancestors are appended to the parent
-    #of the pattrern which was skipped
-    OUTPUT_TYPE_TEMP = 0x11
-    
-    #These fields can be set upon wrapper creation - i.e. a field which is public but not contained here can be accessed 
-    #from outside, but not set as a result of wrapper construction
-    SETTABLE_FIELDS = ['generalize', 'type', 'output_type', 'write_text']
-    
-    attr_accessor :name, :output_type, :generalize, :children, :filters, :parent, 
-                  :last_result, :result, :limit, 
-                  :examples, :parent_of_leaf, :evaluation_context, :type,
-                  :indices_to_extract, :evaluation_context, :referenced_extractor, 
-                  :referenced_pattern, :write_text
-    attr_reader   :generalize_set, :next_page_url, :result_indexer
-
-    def initialize (name, *args)
-      @name = name                #name of the pattern
-      parse_args(args)            #parse the rest of the arguments
-      @children = []              #child patterns
-      @filters = []               #filters of the wrapper
-      @result = Result.new        #hierarchical results of the pattern
-      if @examples == nil
-        filters << Scrubyt::Filter.new(self) #create a default filter
+    #    OUTPUT_TYPE_MODEL = :OUTPUT_TYPE_MODEL
+    #    #Temp patterns are skipped in the output (their ancestors are appended to the parent
+    #    #of the pattrern which was skipped
+    #    OUTPUT_TYPE_TEMP = :OUTPUT_TYPE_TEMP
+
+    VALID_OUTPUT_TYPES = [:model, :temp]
+
+    #These options can be set upon wrapper creation
+    VALID_OPTIONS = [:generalize, :type, :output_type, :write_text, :references, :limit, :default, :resolve]  + Scrubyt::CompoundExample::DESCRIPTORS
+
+    attr_accessor(:name, :options, :children, :constraints, :filters, :parent,
+                  :last_result, :result, :evaluation_context,
+                  :indices_to_extract, :referenced_extractor, :referenced_pattern,
+                  :source_file, :source_proc, :modifier_calls)
+
+    attr_reader(:next_page_url, :result_indexer)
+
+    option_reader(:type => :tree, :output_type => :model, :generalize => false,
+                  :write_text => lambda { @children.size == 0 }, :limit => nil,
+                  :default => nil, :resolve => :full)
+
+    def initialize(name, args=[], evaluation_context=nil, parent=nil, &block)
+      #init attributes
+      @name = name
+      @evaluation_context = evaluation_context
+      @parent = parent
+      @options = {}
+      @children = []
+      @filters = []
+      @constraints = []
+      @result = Result.new
+      @modifier_calls = []
+
+      #grab any examples that are defined
+      examples = look_for_examples(args)
+
+      #parse the options hash if provided
+      parse_options_hash(args[-1]) if args[-1].is_a? Hash
+
+      #perform checks for special cases
+      examples = check_if_shortcut_pattern() if examples == nil
+      check_if_detail_page(block)
+
+      #create filters
+      if examples == nil
+        @filters << Scrubyt::BaseFilter.create(self) #create a default filter
       else
-        @examples.each do |example|
-          filters << Scrubyt::Filter.new(self,example) #create a filter
+        examples.each do |example|
+          @filters << Scrubyt::BaseFilter.create(self,example) #create a filter
+        end
+      end
+
+      #by default, generalize direct children of the root pattern, but only in the case if
+      #@generalize was not set up explicitly
+      @options[:generalize] = true if parent && parent.type == :root && @options[:generalize].nil?
+
+      #parse child patterns if available
+      parse_child_patterns(&block) if ( !block.nil? && type != :detail_page )
+
+      #tree pattern only (TODO: subclass?)
+      if type == :tree
+        #generate xpaths and regexps
+        @filters.each do |filter|
+          filter.generate_XPath_for_example(false)
+          filter.generate_regexp_for_example
+        end
+        #when the xpaths of this pattern have been created, its children can make their xpaths relative
+        xpaths = @filters.collect { |filter| filter.xpath }
+        @children.each do |child|
+          child.generate_relative_XPaths xpaths
         end
-      end      
+      end
+    end
+
+    def generate_relative_XPaths(parent_xpaths)
+      return if type != :tree
+      raise ArgumentError.new if parent_xpaths.size != 1 && parent_xpaths.size != @filters.size #TODO: should be checked earlier with proper error message
+      @filters.each_index do |index|
+        @filters[index].generate_relative_XPath parent_xpaths[parent_xpaths.size == 1 ? 0 : index]
+      end
     end
-    
-    #Parse the args passed as *args; There is only one compulsory parameter to pattern: it's name
-    #All the other parameters can (but do not have to) be specified;
+
+    #Shortcut patterns, as their name says, are a shortcut for creating patterns
+    #from predefined rules; for example:
+    #
+    #  detail_url
+    #
+    #  is equivalent to
+    #
+    #  detail_url 'href', type => :attribute
+    #
+    #i.e. the system figures out on it's own that because of the postfix, the
+    #example should be looked up (but it should never override the user input!)
+    #another example (will be available later):
+    #
+    # every_img
+    #
+    # is equivivalent to
     #
-    #If an example is specified, it *MUST* be the first parameter; the order of the other
-    #parameters is irrelevant
-    def parse_args(args)
-      #Grab any examples that are defined!
-      look_for_examples(args)
-      args.each do |arg|
-        next if !arg.is_a? Hash        
-        arg.each do |k,v|
-          #Set only the setable fields
-          if SETTABLE_FIELDS.include? k.to_s 
-            #If the user is specifying a pattern type, turn it into the corresponding constant
-            v = "PATTERN_TYPE_#{v.to_s.upcase!}" if k.to_s == 'type'
-            v = "OUTPUT_TYPE_#{v.to_s.upcase!}" if k.to_s == 'output_type'
-            #Otherwise, if nothing special is happening, isntance_eval the hash pair
-            instance_eval("@#{k.to_s} = #{v}") 
+    # every_img '//img'
+    #
+    def check_if_shortcut_pattern()
+      if @name =~ /.+_url/
+        @options[:type] = :attribute
+        ['href']
+      end
+    end
+
+    #Check whether the currently created pattern is a detail pattern (i.e. it refrences
+    #a subextractor). Also check if the currently created pattern is
+    #an ancestor of a detail pattern , and store this in a hash if yes (to be able to
+    #traverse the pattern structure on detail pages as well).
+    def check_if_detail_page(block)
+      #return if !@options[:references]
+      #@options[:type] = :detail_page
+      #@referenced_extractor = @options[:references]
+      if @name =~ /.+_detail/
+        @options[:type] = :detail_page
+        @referenced_extractor = block
+        Scrubyt::Extractor.add_detail_extractor_to_pattern_name(block, self)
+      end
+    end
+
+    def parent_of_leaf
+      @children.inject(false) { |is_parent_of_leaf, child| is_parent_of_leaf || child.children.empty? }
+    end
+
+    def parse_child_patterns(&block)
+      context = Object.new
+      context.instance_eval do
+        def current=(value)
+          @current = value
+        end
+        def method_missing(method_name, *args, &block)
+          if method_name.to_s[0..0] == '_'
+            #add hash option
+            key = :"#{method_name.to_s[1..-1]}"
+            args.each do |arg|
+              current_value = @current.options[key]
+              if current_value.nil?
+                @current.options[key] = arg
+              else
+                @current.options[key] = [current_value] if !current_value.is_a Array
+                @current.options[key] << arg
+              end
+            end
+          else
+            #create child pattern
+            child = Scrubyt::Pattern.new(method_name.to_s, args, @current.evaluation_context, @current, &block)
+            @current.children << child
+            child
           end
-          #This flags says that the user explicitly wants to set generalization on a pattern
-          #In this case, of course, our heuristics do not apply - the users setting overrides
-          #it
-          @generalize_set = true if (k.to_s == 'generalize')        
         end
       end
-      #default settings - the user can override them, but if she did not do so,
-      #we will setup some meaningful defaults
-      @type ||= PATTERN_TYPE_TREE
-      @output_type ||= OUTPUT_TYPE_MODEL
-      #don't generalize by default
-      @generalize ||= false
-      #This flag indicates that the user set 'generalize' to some value;
-      #This way we can ensure that the explicit setting will not be overridden 
-      @generalize_set ||= false 
-    end    
-    
+      context.current = self
+      context.instance_eval(&block)
+    end
+
     #Dispatcher function; The class was already too big so I have decided to factor
-    #out some methods based on their functionality (like output, adding constraints) 
+    #out some methods based on their functionality (like output, adding constraints)
     #to utility classes.
     #
-    #The second function besides dispatching is to lookup the results in an evaluated 
+    #The second function besides dispatching is to lookup the results in an evaluated
     #wrapper, for example
     #
     # camera_data.item[1].item_name[0]
     def method_missing(method_name, *args, &block)
+      if @evaluation_context.evaluating_extractor_definition
+        @modifier_calls << [method_name, [:array, *args.collect { |arg| [:lit, arg] }]]
+      end
+
       case method_name.to_s
-        when 'select_indices'
-          @result_indexer = Scrubyt::ResultIndexer.new(*args)
-          self
-        when /^to_/
-          Scrubyt::ResultDumper.send(method_name.to_s, self)
-        when /^ensure_/
-          Scrubyt::ConstraintAdder.send(method_name, self, *args)
-        else
-          @children.each { |child| return child if child.name == method_name.to_s }
-          nil
+      when 'select_indices'
+        @result_indexer = Scrubyt::ResultIndexer.new(*args)
+        return self
+      when /^to_/
+        return Scrubyt::ResultDumper.send(method_name.to_s, self)
+      when /^ensure_/
+        @constraints << Scrubyt::ConstraintAdder.send(method_name, *args)
+        return self #To make chaining possible
+      else
+        @children.each { |child| return child if child.name == method_name.to_s }
       end
+
+      raise NoMethodError.new(method_name.to_s, method_name.to_s, args)
     end
 
     #Companion function to the previous one (Pattern::method_missing). It makes
     #inspecting results, like
     #
     #    camera_data.item[1].item_name[0]
-    #    
+    #
     #possible. The method Pattern::method missing handles the 'item', 'item_name' etc.
     #parts, while the indexing ([1], [0]) is handled by this function.
     #If you would like to select a different document than the first one (which is
@@ -142,105 +247,135 @@ module Scrubyt
         return nil if (@result.lookup(@parent.last_result)) == nil
         @last_result = @result.lookup(@parent.last_result)[index]
       end
-      self    
+      self
     end
-    
+
     ##
     #If export is called on the root pattern, it exports the whole extractor wher it is
     #defined; See export.rb for further details on the parameters
-    def export(file, output_file_name=nil, extractor_result_file_name=nil)
-      Scrubyt::Export.export(file, self, output_file_name, extractor_result_file_name)
+    def export(arg1, output_file_name=nil, extractor_result_file_name=nil)
+      #      require 'scrubyt/output/export_old'; Scrubyt::ExportOld.export(arg1, self, output_file_name, extractor_result_file_name) ; return
+      if File.exists? arg1
+        old_export(arg1, output_file_name, extractor_result_file_name)
+      else
+        new_export(arg1, output_file_name, extractor_result_file_name)
+      end
     end
-    
-    ##
-    #Add a filter to this pattern    
-    def add_filter(filter)
-      @filters << filter
-      return self
+
+    def old_export(input_file, output_file_name=nil, extractor_result_file_name=nil)
+      contents = open(input_file).read
+      wrapper_name = contents.scan(/\s+(.+)\s+=.*Extractor\.define.*/)[0][0]
+      Scrubyt::Export.export(self, wrapper_name, output_file_name, extractor_result_file_name)
     end
 
-    ##
-    #Add a child pattern to this pattern
-    def add_child_pattern(child)
-      child.parent = self
-      #by default, generalize direct children of the root pattern, but only in the case if
-      #@generalize was not set up explicitly
-      child.generalize = true if (!child.generalize_set && child.parent != nil && child.parent.parent == nil)
-      @children << child
+    def new_export(wrapper_name, output_file_name=nil, extractor_result_file_name=nil)
+      Scrubyt::Export.export(self, wrapper_name, output_file_name, extractor_result_file_name)
     end
-        
+
     ##
-    #Evaluate the pattern. This means evaluating all the filters and adding 
+    #Evaluate the pattern. This means evaluating all the filters and adding
     #their extracted instances to the array of results of this pattern
-    def evaluate      
-      #No need to evaluate if there is no parent pattern
-      return if @parent == nil
-      all_filter_results = []
-      @filters.each do |filter|
-        filter_index = @filters.index(filter)
-        filter_index = 0 if @parent.filters.size <= filter_index
-        filter.source = @parent.filters[filter_index].sink
-        filter.source.each do |source|          
-          r = filter.evaluate(source)
-          next if r == nil
-          if filter.constraints.size > 0
-            #in the beginning, keys of result_hash are made up of all the results of the filter
-            #with value = true; Later on, only those results will have 'true' value which are
-            #accepted with all filters
-            result_hash = {}
-            r.each { |res| result_hash[res] = true }
-            result_hash.keys.each do |res|
-              filter.constraints.each { |constraint| result_hash[res] &&= constraint.check(res) }
+    def evaluate(parent_filters)
+      if type != :root #TODO: should be removed, but there is more refactoring of filter handling needed to do so
+        all_filter_results = []
+        @filters.each do |filter|
+          filter_index = @filters.index(filter)
+          filter_index = 0 if parent_filters.size <= filter_index
+          filter.source = parent_filters[filter_index].sink
+          filter.source.each do |source|
+            results = filter.evaluate(source)
+            next if results == nil
+            #apply constraints
+            if @constraints.size > 0
+              results = results.select do |result|
+                @constraints.inject(true) { |accepted, constraint| accepted && constraint.check(result) }
+              end
             end
-            result = result_hash.reject {|k,v| k if !v}
-            sorted_result = r.reject {|e| !result.keys.include? e}
-            indexer = @result_indexer == nil ? sorted_result : @result_indexer.select_indices_to_extract(sorted_result)
-            add_result(filter, source, indexer)
-          else
-            indexer = @result_indexer == nil ? r : @result_indexer.select_indices_to_extract(r)
-            add_result(filter, source, indexer)
-          end#end of constraint check
-        end#end of source iteration
-      end#end of filter iteration    
-    end   
-    
-    def get_constraints
-      filters[0].constraints
-    end     
-       
-private        
+            #apply indexer
+            results = @result_indexer.select_indices_to_extract(results) if !@result_indexer.nil?
+            add_result(filter, source, results)
+          end
+        end
+      end
+
+      #evaluate children
+      @children.each { |child| child.evaluate(@filters) }
+
+      #do postprocessing
+    end
+
+    def to_sexp
+      #collect arguments
+      args = []
+      args.push(*@filters.to_sexp_array) if type != :detail_page #TODO: this if shouldn't be there
+      args.push(@options.to_sexp) if !@options.empty?
+
+      #build main call
+      sexp = [:fcall, @name, [:array, *args]]
+
+      if type == :detail_page
+        #add detail page extractor
+        detail_root = @evaluation_context.extractor.get_detail_extractor(self)
+        sexp = [:iter, sexp, nil, [:block, *detail_root.children.to_sexp_array ]]
+      else
+        #add child block if the pattern has children
+        sexp = [:iter, sexp, nil, [:block, *@children.to_sexp_array ]] if !@children.empty?
+      end
+
+      #add modifier calls - TODO: remove when everything is exported to the options hash
+      @modifier_calls.each do |modifier_sexp|
+        sexp = [:call, sexp, *modifier_sexp]
+      end
+
+      #return complete sexp
+      sexp
+    end
+
+    private
+    def parse_options_hash(hash)
+      #merge provided hash
+      @options.merge!(hash)
+      #check if valid
+      hash.each { |key, value| raise "Unknown pattern option: #{key.to_s}" if VALID_OPTIONS.index(key.to_sym).nil? }
+      raise "Invalid pattern type: #{type.to_s}" if VALID_PATTERN_TYPES.index(type.to_sym).nil?
+      raise "Invalid output type: #{output_type.to_s}" if VALID_OUTPUT_TYPES.index(output_type.to_sym).nil?
+    end
+
     def look_for_examples(args)
       if (args[0].is_a? String)
-        @examples = args.select {|e| e.is_a? String}
+        examples = args.select {|e| e.is_a? String}
         #Check if all the String parameters are really the first
-        #parameters 
-        args[0..@examples.size-1].each do |example|
+        #parameters
+        args[0..examples.size-1].each do |example|
           if !example.is_a? String
             puts 'FATAL: Problem with example specification'
           end
         end
       elsif (args[0].is_a? Regexp)
-        @examples = args.select {|e| e.is_a? Regexp}
+        examples = args.select {|e| e.is_a? Regexp}
         #Check if all the String parameters are really the first
-        #parameters 
-        args[0..@examples.size].each do |example|
+        #parameters
+        args[0..examples.size].each do |example|
           if !example.is_a? Regexp
             puts 'FATAL: Problem with example specification'
           end
         end
-        @type = PATTERN_TYPE_REGEXP
+        @options[:type] = :regexp
       elsif (args[0].is_a? Hash)
-        @examples = (args.select {|e| e.is_a? Hash}).select {|e| CompoundExample.compound_example?(e)}
-        @examples = nil if @examples == []
+        examples = (args.select {|e| e.is_a? Hash}).select {|e| CompoundExample.compound_example?(e)}
+        examples = nil if examples == []
       end
-      
+
+      @has_examples = !examples.nil?
+      examples
     end
-    
+
     def add_result(filter, source, results)
       results.each do |res|
         filter.sink << res
         @result.add_result(source, res)
-      end      
-    end # end of method generate_examples
+      end
+    end
+
   end #end of class Pattern
 end #end of module Scrubyt