metrocot 1.0.0

data/lib/metrocot.rb

@@ -0,0 +1,1112 @@
+
+#############################################################################
+#
+# Copyright (c) 2009 Metro Cascade Media Inc
+# 
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+#############################################################################
+#
+#  Helmut Hissen <helmut@zeebar.com> (Metro Cascade Media Inc)
+#  January 1 2009
+#
+#############################################################################
+#
+#  We are like tiny pleasantly chirping hex bugs coding away on the
+#  shoulders of Why so that we can create more, and do more with less
+#  code, not by virtue of any sharpness of mind on our part, or any
+#  other distinction, but because we are carried high and raised up
+#  by his giant size.
+#
+#############################################################################
+#
+
+class Metrocot < Object
+
+	VERSION = '1.0.0'
+
+	class MatchRange
+		
+		attr_accessor :node_scraper, :start_index, :start_offset, :end_index, :end_offset, :verbose
+
+
+		def initialize( node_scraper, start_index, start_offset, end_index, end_offset )
+			@node_scraper = node_scraper
+			@start_index  = start_index
+			@start_offset = start_offset
+			@end_index    = end_index
+			@end_offset   = end_offset
+			@verbose      = false
+		end
+
+
+		def hnodes
+			@node_scraper.flattened_hnodes
+		end
+
+
+		def crop( crop_start_index, crop_start_offset, crop_end_index, crop_end_offset )
+			MatchRange.new( node_scraper, crop_start_index, crop_start_offset, crop_end_index, crop_end_offset )
+		end
+
+
+		def empty?
+			return @start_index >= @end_index || (@start_index == @end_index && @start_offset >= end_offset)
+		end
+
+
+		def following( other_range )
+			MatchRange.new( node_scraper, other_range.end_index, other_range.end_offset, end_index, end_offset )
+		end
+
+
+		def tail( tail_start_index, tail_start_offset )
+			MatchRange.new( @node_scraper, tail_start_index, tail_start_offset, @end_index, @end_offset )
+		end
+
+
+		def head( head_end_index, head_end_offset )
+			MatchRange.new( @node_scraper, @start_index, @start_offset, head_end_index, head_end_offset )
+		end
+
+
+		def split_at( middle )
+			parts = []
+			if middle.start_index > 0 || middle.start_offset > 0
+				parts << crop_at( 0, 0, middle.start_index, middle.start_offset )
+			end
+			parts << middle
+			if middle.end_index < end_index || (middle.start_end_index == end_index && middle.end_offset < end_offset)
+				parts << crop( middle.end_index, middle.end_offset, end_index, end_offset )
+			end
+		end
+
+
+		def extend( other ) 
+
+			extended_range = MatchRange.new( @node_scraper, @start_index, @start_offset, @end_index, @end_offset )
+			if other.start_index < extended_range.start_index
+				extended_range.start_index = other.start_index 
+			elsif other.start_index == extended_range.start_index && other.start_offset < extended_range.start_offset
+				extended_range.start_offset = other.start_offset 
+			end
+
+			if other.end_index > extended_range.end_index
+				extended_range.end_index = other.end_index 
+			elsif other.end_index == extended_range.end_index && other.end_offset > extended_range.end_offset
+				extended_range.end_offset = other.end_offset 
+			end
+
+			extended_range
+
+		end
+
+
+		def describe
+			"[#{start_index}+#{start_offset} ... #{end_index}+#{end_offset}]"
+		end
+
+	end
+
+
+
+	class BasePattern
+
+		attr_accessor :pred, :succ, :source, :name, :matched, :node_scraper, :metrocot, :pattern_no
+
+		@@instance_count = 0
+
+		def initialize( source )
+			@source = source 
+			@pattern_no = @@instance_count
+			@@instance_count += 1
+		end
+
+		def optional
+			false
+		end
+
+		def log( s ) 
+			metrocot.log("#{self.description}: #{s}")
+		end
+
+		def dump( level, out ) 
+			out << "  " * level + description + " p=#{priority}\n"
+		end
+
+		def self.parse(s)
+			raise "not supported"
+		end
+
+		def description
+			self.class.name
+		end
+
+
+		def dump_match_map( out, level, match_map )
+			if match_map.is_a? Hash
+				out << "{\n"
+				level += 1
+				match_map.each { |key, value|
+					out << "  " * level + "#{key} => " 
+					dump_match_map( out, level, value )
+				}
+				level -= 1
+				out << "  " * level + "}\n"
+			elsif match_map.is_a? Array
+				out << "[\n"
+				level += 1
+				match_map.each { |value|
+					out << "  " * level 
+					dump_match_map( out, level, value )
+				}
+				level -= 1
+				out << "  " * level + "]\n"
+			elsif match_map.is_a? String
+				out << "\"" + match_map + "\"\n"
+			elsif match_map.is_a? Hpricot::Elem
+				out << "<" + match_map.stag.name + ">\n"
+			else
+				out << match_map.class.to_s + "\n"
+			end
+		end
+
+
+		def log_match_data( msg, match_range, match_map )
+			log("#{msg} #{match_range.describe} map:")
+			if @node_scraper.verbose	
+				if ! match_map.nil? && match_map != {} 
+					dump_match_map( STDOUT, 0, match_map )
+				end
+			end
+		end
+
+
+		def each_match( match_range, match_map )
+			log_match_data("each_match", match_range, match_map)
+		end
+	
+		def priority
+			0
+		end
+
+		def with_scanned_match_data( match_map, match_data )
+			
+			scanner = if name 
+				@node_scraper.scanner_by_name(name) 
+			else 
+				default_scanner
+			end
+
+			if scanner
+#				begin
+					match_map[name] = scanner.scan(match_data)
+#				rescue
+#					log("scanner error: #{$!}")
+#					return nil
+#				end
+			elsif name
+				match_map[name] = match_data
+			elsif self.is_a?(CompositePattern) && match_data.is_a?(Hash)
+				log("copying #{match_data.class.name} match data from #{self.description}")
+				match_data.each { |key, value|
+					match_map[key] = value
+				}
+			else
+				log("not carrying #{match_data.class.name} match data from #{self.description}")
+			end
+			
+			result = yield( match_map )
+			match_map.delete(name) if name
+			return result
+
+		end
+
+		def default_scanner
+			nil
+		end
+
+
+	end
+
+
+	class PathPattern  < BasePattern 
+
+		def initialize( source, path )
+			super(source)
+			@path = path
+		end
+
+		def self.parse( s ) 
+			return nil unless s.index(".") == 0
+			return nil if s.index("..") == 0
+			space_index = s.index(" ") || s.size
+			return nil if space_index == 1
+			self.new( s[0 .. space_index - 1], s[1 .. space_index - 1] )
+		end
+
+
+		def source
+			@source
+		end
+
+
+		def description
+			"path \"#{@path}\""
+		end
+	
+
+		def each_match( match_range, match_map )
+			super(match_range, match_map)
+			result = nil
+			search_root = match_range.node_scraper.hnode
+			search_root.search( @path ).each { |descendent|
+				nix = node_scraper.hnode_index[descendent]
+				next_node_nix = node_scraper.hnode_succ_index[descendent]
+				unless nix
+					@node_scraper.flattened_hnodes.each { |node|
+						log( "#{@node_scraper.hnode_index[node]}: #{node}" )
+					}
+					raise "no node index for #{descendent.class} #{descendent}" 
+				end
+				if nix < match_range.start_index 
+					log( "too far left: #{nix}" )
+					next 
+				end
+				if nix >= match_range.end_index 
+					log( "too far right: #{nix}" )
+					break 
+				end
+				log( "matched path at node #{nix}" )
+				result = with_scanned_match_data( match_map, descendent ) { |match_map|
+					yield( match_range.crop(nix, 0, next_node_nix, 0) , match_map )
+				}
+				break if result
+			}
+			result
+		end
+
+	
+		def priority
+			1
+		end
+
+	
+	end
+
+	
+	class OptSpacePattern < BasePattern 
+
+		def initialize
+			super(" ")
+		end
+
+		def description
+			"spaces"
+		end
+
+		def optional
+			true
+		end
+
+		def self.parse( s ) 
+			return nil unless s[0..0] == " "
+			OptSpacePattern.new
+		end
+
+		def priority
+			-7
+		end
+
+		def each_match( match_range, match_map )
+			super(match_range, match_map)
+			match_start_index = match_range.start_index
+			match_start_offset = match_range.start_offset
+			match_end_index = match_range.start_index
+			match_end_offset = match_range.start_offset
+
+			# consume rest of first text node  
+
+			hnodes = match_range.hnodes
+
+			if hnodes[match_start_index] && hnodes[match_start_index].text?
+					hnode_text = hnodes[match_start_index].inner_text
+					while match_end_offset < hnode_text.size && (/\s+/.=== hnode_text[match_start_offset .. match_end_offset])
+						match_end_offset += 1
+					end
+					
+					if match_end_offset > match_start_offset 
+						if match_end_offset >= hnode_text.size
+							match_range = match_range.tail( match_end_index + 1, 0 )
+							log( "matched entire string of #{match_end_offset - match_start_offset} spaces" )
+						else
+							match_range = match_range.tail( match_end_index, match_end_offset )
+							log( "matched first #{match_end_offset - match_start_offset} leading spaces" )
+						end
+					end
+			end
+
+			result = with_scanned_match_data( match_map, hnodes[match_start_index ... match_end_index] ) { |match_map|
+				yield( match_range, match_map ) 
+			}
+			result
+			
+		end
+
+	end
+
+
+	class OneOrMorePattern < BasePattern 
+
+		def initialize(repeatee) 
+			super( nil )
+			@repeatee = repeatee
+		end
+
+
+		def self.parse(s)
+			raise "not implemented"
+		end
+
+
+		def repeatee
+			@repeatee
+		end
+
+
+		def description
+			"one+ ##{pattern_no}"
+		end
+	
+
+		def dump( level, out ) 
+			out << "  " * level + "one or more p=#{priority}\n"
+			@repeatee.dump( level + 1, out )
+		end
+
+
+		def priority
+			@repeatee.priority
+		end
+
+
+		def consume_remaining_matches( match_range, match_map, matches )
+
+			log("consuming remaining matches in #{match_range}")
+			@repeatee.each_match( match_range, match_map ) { |r_match_range, r_match_map|
+				matches << r_match_range.hnodes[r_match_range.start_index ... r_match_range.end_index]
+				last_match_range = consume_remaining_matches( match_range.tail( r_match_range.end_index, r_match_range.end_offset ), match_map, matches )  
+				return last_match_range || r_match_range
+			}
+			return nil
+
+		end
+
+
+		def each_match( match_range, match_map )
+
+			super(match_range, match_map)
+
+			log("looking for first match in #{match_range}")
+			@repeatee.each_match( match_range, match_map ) { |first_match_range, first_match_map|
+				results = []
+				last_match_range = consume_remaining_matches( match_range.tail( first_match_range.end_index, first_match_range.end_offset ), match_map, results ) 
+					
+				combined_match_range = if last_match_range
+					match_range.crop( 
+							first_match_range.start_index, first_match_range.start_offset, 
+							last_match_range.end_index, last_match_range.end_offset
+					)
+				else
+					first_match_range
+				end
+	
+				log("combined match in #{combined_match_range}")
+
+				result = with_scanned_match_data( match_map, results ) { |match_map|
+					yield( combined_match_range, results )
+				}
+
+				if result 
+					log("one+ match done with #{result}")
+					return result 
+				else
+					log("one+ match not done")
+				end
+			}
+			
+			return nil
+		end
+
+	end
+
+
+	class AnythingPattern < BasePattern 
+
+		def description
+			"anything"
+		end
+
+		def self.parse( s ) 
+			return nil unless s.index("...") == 0
+			self.new("...")
+		end
+	
+
+		# always matches the whole thing because it will get matched last... 
+		# it just expands to fill whatever gap
+
+		def each_match( match_range, match_map )
+			with_scanned_match_data( match_map, match_range.hnodes[match_range.start_index .. match_range.end_index] ) { |match_map|
+				yield( match_range, match_map ) 
+			}
+		end
+
+
+		def priority
+			-7
+		end
+
+	end
+
+
+	class TextPattern < BasePattern 
+
+		def initialize( source, text )
+			super(source)
+			@text = text
+		end
+
+		def description
+			"text \"#{@text}\""
+		end
+
+		def self.parse( s ) 
+
+			if s.index("$") == 0
+				return self.new( "$", /[\r\n]/ )
+			end
+
+			if s.index("/") == 0
+
+				r_s = ""
+				s = s[1..-1]
+				src = "/"
+				while !s.empty?
+
+					if s.index("/") == 0
+						s = s[1..-1]
+						src << "/"
+						break
+					elsif s.index("\\/") == 0
+						s = s[2..-1]
+						r_s << "/"
+						src << "\\/"
+					else
+						r_s << s[0..0]
+						src << s[0..0]
+						s = s[1..-1]
+					end
+
+				end
+
+				regexp = Regexp.compile( r_s )
+				return self.new( src, regexp )
+
+			end
+
+			if s.index("\"") == 0
+
+				r_s = ""
+				s = s[1..-1]
+				src = "\""
+				while !s.empty?
+
+					if s.index("\"") == 0
+						s = s[1..-1]
+						src << "\""
+						break
+					elsif s.index("\\\"") == 0
+						s = s[2..-1]
+						r_s << "\""
+						src << "\\\""
+					else
+						r_s << s[0..0]
+						src << s[0..0]
+						s = s[1..-1]
+					end
+
+				end
+
+				return self.new( src, r_s )
+			end
+
+		end
+	
+		def priority
+			if name
+				if @text.is_a?(String)
+					-4
+				else
+					-5
+				end
+			else
+				if @text.is_a?(String)
+					-2
+				else
+					-3
+				end
+			end
+		end
+	
+		def each_match( match_range, match_map )
+
+			super(match_range, match_map)
+
+			match_start_index = match_range.start_index
+			match_start_offset = match_range.start_offset
+			match_end_index = match_range.start_index
+			match_end_offset = match_range.start_offset
+
+			# consume rest of first text node  
+
+			hnodes = match_range.hnodes
+
+			actual_match = nil
+
+			while match_start_index < match_range.end_index 
+
+				while match_start_index < match_range.end_index && ! hnodes[match_start_index].text?
+					log( "not text: ##{match_start_index} #{hnodes[match_start_index].class}" )
+					match_start_index += 1
+					match_start_offset = 0
+				end
+
+				unless match_start_index < match_range.end_index && hnodes[match_start_index].text?
+					log( "no match found" )
+					return nil
+				end
+
+				hnode_text = hnodes[match_start_index].inner_text
+
+				log( "trying text match on: #{hnode_text[match_start_offset .. -1]}" )
+
+				match_offset = hnode_text.index( @text, match_start_offset )
+
+				if match_offset
+
+					actual_match = if @text.is_a? Regexp
+						hnode_text[match_offset..-1][@text]
+					else
+						@text
+					end
+
+					match_end_offset = match_start_offset + actual_match.size
+					match_start_offset = match_start_offset + actual_match.size
+							
+					if match_end_offset >= match_start_offset 
+						if match_end_offset >= hnode_text.size
+							log( "matched entire string of #{match_end_offset - match_start_offset} chars" )
+						else
+							log( "matched first #{match_end_offset - match_start_offset} chars" )
+						end
+						break
+					end
+				end
+
+				match_start_index += 1
+				match_start_offset = 0
+		
+			end
+
+
+			result = with_scanned_match_data( match_map, actual_match ) { |match_map|
+				yield( match_range.crop( match_start_index, match_start_offset, match_start_index, match_end_offset), match_map ) 
+			}
+			result
+			
+		end
+
+	end
+
+
+	class CompositePattern < BasePattern 
+
+		attr_reader :parts
+
+		def initialize( parts = nil )
+			super(nil)
+			@parts = if parts.nil?
+				[]
+			else
+				parts.clone
+			end
+			succ_pred = nil
+			@parts.each { |part| 
+				part.pred = succ_pred
+				succ_pred.succ = part unless succ_pred.nil?
+				succ_pred = part
+			}
+		end
+
+
+		def dump( level, out ) 
+			out << "  " * level + "composite p=#{priority} [\n"
+			@parts.each { |part|
+				part.dump( level + 1, out )
+			}
+			out << "  " * level + "]\n"
+		end
+
+
+		def each_split_match( match_range, match_map, parts_by_priority, ppx, part_matches ) 
+
+			pattern = nil
+
+			while (ppx < parts_by_priority.size)
+				pattern = parts_by_priority[ppx]
+				break unless pattern.optional 
+				break unless pattern.pred && ! pattern.pred.matched
+				log("skipping optional #{pattern.description}")
+				ppx += 1
+			end
+
+			if ppx >= parts_by_priority.size
+				log("comp nothing left to do")
+				return yield( match_range, match_map ) 
+			end
+
+
+			#
+			# figure out which gap this pattern is supposed to fill
+			#
+
+			matched_on_right = pattern.succ
+
+			while matched_on_right && ! matched_on_right.matched
+				matched_on_right = matched_on_right.succ
+			end
+
+			if matched_on_right
+				log("comp matching must be left of #{matched_on_right.description}")
+				match_range = match_range.head(matched_on_right.matched.start_index, matched_on_right.matched.start_offset) 
+			end
+
+			matched_on_left = pattern.pred
+
+			while matched_on_left && ! matched_on_left.matched
+				matched_on_left = matched_on_left.pred
+			end
+
+			if matched_on_left
+				log("comp matching must be right of #{matched_on_left.description}")
+				match_range = match_range.tail(matched_on_left.matched.end_index, matched_on_left.matched.end_offset) 
+			end
+
+			log("comp matching sub-pattern: #{pattern.description} at #{match_range.describe}")
+
+			pattern.each_match( match_range, match_map ) { |part_match_range, match_map| 
+
+				pattern.matched = part_match_range
+
+				result = each_split_match( match_range, match_map, parts_by_priority, ppx + 1, part_matches ) { |sub_match_range, sub_match_map|
+					yield( sub_match_range, match_map )
+				}
+
+				pattern.matched = nil
+
+				if result
+					log("comp done, returning: #{result}")
+					return result 
+				else
+					log("comp not done")
+				end
+			}
+
+			return nil
+
+		end
+
+	
+		def each_match( match_range, match_map )  
+
+			@parts.each { |part| part.matched = nil }
+			
+			super(match_range, match_map)
+
+			# find the hightest priority part and divine up the children 
+
+			parts_by_priority = @parts.sort_by { |part| 0 - part.priority }
+			each_split_match( match_range, {}, parts_by_priority, 0, {} ) { |last_match_range, last_match_map|
+				comp_match_range = nil
+				@parts.each { |part|
+					next unless part.matched
+					if ! comp_match_range
+						comp_match_range = part.matched
+						next
+					end
+					comp_match_range = comp_match_range.extend(part.matched)
+				}
+				result = with_scanned_match_data( match_map, last_match_map ) { |match_map|
+					log_match_data("comp match trying", comp_match_range, match_map)
+					yield( comp_match_range, match_map )
+				}
+				if result
+					log("comp match done, returning: #{result}")
+					return result 
+				else
+					log("comp match not done")
+				end
+			}
+
+		end
+
+		def description
+			"comp ##{pattern_no}"
+		end
+	
+	end
+
+
+	class NodeScraper
+
+		attr_accessor :mcot, :root, :parent, :hnode, :pattern_classes, :top_part_names, :verbose
+
+
+		def initialize( mcot, parent, root, hnode )
+			@mcot         = mcot 
+			@parent       = parent 
+			@root         = root || self
+			@hnode        = hnode 
+			@verbose      = mcot.verbose
+		end
+
+
+		def log( s ) 
+			mcot.log( s ) if @verbose
+		end
+
+
+		def descend( path )
+
+			results = {}
+			@hnode.search( path ).each { |hchild| 
+				results[hchild] = yield( NodeScraper.new( @mcot, self, @root, hchild ) ) 
+			}
+			results
+
+		end
+
+
+		def flatten_hnodes( ix, node )
+			@flattened_hnodes << node
+			@hnode_index[node] = ix
+			ix += 1 
+			if node.elem?
+				node.children.each { |child| 
+					ix = flatten_hnodes(ix, child) 
+				}
+			end
+			@hnode_succ_index[node] = ix
+			ix
+		end
+
+
+		def build_hnode_index
+
+			@flattened_hnodes = []
+			@hnode_index = {}
+			@hnode_succ_index = {}
+
+			n = flatten_hnodes( 0, hnode )
+
+			log( "built index for #{n} hnodes" )
+
+		end
+
+
+		def flattened_hnodes
+			unless @flattened_hnodes
+				build_hnode_index
+			end
+			return @flattened_hnodes
+		end
+
+
+		def hnode_index
+			unless @hnode_index
+				build_hnode_index
+			end
+			return @hnode_index
+		end
+
+
+		def hnode_succ_index
+			unless @hnode_succ_index
+				build_hnode_succ_index
+			end
+			return @hnode_succ_index
+		end
+
+
+		def collect_gen( pattern_s, call_with, &block )
+			pattern = @mcot.compile_pattern( pattern_s, self )
+			top_part_names = []
+			if pattern.is_a? CompositePattern
+				pattern.parts.each { |part|
+					top_part_names << part.name if part.name
+				}
+			end
+			log("top part names: #{top_part_names.join(", ")}")
+			build_hnode_index
+			pattern.dump( 0, $stdout ) if @verbose
+			results = []
+			match_range = MatchRange.new( self, 0, 0, flattened_hnodes.size, 0)
+			while ! match_range.empty?
+				result = nil
+				pattern.each_match( match_range, {} )  { |sub_match_range, match_map|
+					match_list = []
+					block_args = if (call_with == :positional) && top_part_names.size > 0
+						top_part_names.collect { |top_name|
+							match_map[top_name]
+						}
+					else
+						match_map
+					end
+					log("calling scan block with: #{block_args.join(", ")}")
+					result = block.call( *block_args )
+					if result
+						results << result
+						match_range = match_range.following( sub_match_range )
+					end
+		
+					result
+				}
+
+				break unless result
+			end
+			results
+		end
+
+
+		def collect( pattern_s, &block )
+			collect_gen( pattern_s, :positional, &block )
+		end
+
+
+		def collect_hashed( pattern_s, &block )
+			collect_gen( pattern_s, :map, &block )
+		end
+
+		def scanner_by_name( name )
+			return mcot.scanner_by_name(name)
+		end
+
+	end
+
+
+	def log( s )
+		puts( s ) if @verbose
+	end
+
+
+
+	def scanner_by_name( name )
+		@scanners[name]
+	end
+
+
+	def compile_pattern( pattern_s, node_scraper )	
+
+#		if @compiled_patterns.key? pattern_s 
+#			return @compiled_patterns[ pattern_s ] 
+#		end
+
+		s = pattern_s
+		patterns = []
+
+		log("compiling: #{s}")
+		
+		while !s.empty? 
+
+			log("left: #{s}")
+
+			if s.index(")") == 0
+				break
+			end
+
+			name = nil
+			s.scan( /^(\w+)=/ ) { |match| 
+				log "got name #{match.class} (#{match.size})"
+				name = match[0]
+			}
+			s = s[(name.length + 1) .. -1] if name
+
+			log("after name #{name}: #{s}")
+
+			pattern = nil
+
+			[PathPattern, TextPattern, AnythingPattern, OptSpacePattern].each { |pattern_class|
+				pattern = pattern_class.parse(s)
+				if pattern 
+					pattern.metrocot = self
+					pattern.node_scraper = node_scraper
+					break 
+				end
+				log "not a #{pattern_class}"
+			}
+
+			if pattern 
+				s = s[pattern.source.size .. -1]
+				patterns << pattern
+				log("found: #{pattern.description}")
+				if name
+					log("scanned as: #{name}")
+					pattern.name = name.to_sym
+				end
+				next
+			end
+		
+			if s[0..0] == "+" 
+				raise "+ must follow pattern" unless patterns.size > 0
+				raise "+ applied twice does not make sense" if patterns[-1].is_a? OneOrMorePattern
+				pattern = OneOrMorePattern.new( patterns[-1] ) 
+				pattern.metrocot = self
+				pattern.node_scraper = node_scraper
+				patterns[-1] = pattern
+				log("now one or more: #{pattern.repeatee}")
+				s = s[1 .. -1]
+				next
+			end
+
+			if s[0..0] == "(" 
+				pattern = compile_pattern( s[1 .. -1], node_scraper )
+				close_par_index = pattern.source.size + 1
+				raise "expected ')' found '#{s[0..0]}'" unless s[close_par_index..close_par_index] == ")"
+				s = s[close_par_index + 1 .. -1]
+				log("found nested: #{pattern.description} \"#{pattern.source}\"")
+				patterns << pattern
+				if name
+					pattern.name = name.to_sym
+				end
+				next
+			end
+
+			raise "unrecognizable pattern: \"#{s[0..10]}...\""
+				
+		end
+		
+		pattern = if patterns.size > 1
+			CompositePattern.new( patterns )
+		elsif patterns.size == 1
+			patterns[0]
+		else 
+			nil
+		end
+
+		if pattern
+			pattern.metrocot = self
+			pattern.node_scraper = node_scraper
+			pattern.source = pattern_s[0 .. (0 - (1 + s.size))] 
+		end
+
+		return @compiled_patterns[ pattern_s ] = pattern
+		
+	end
+
+
+	attr_accessor :verbose
+
+
+	def initialize( scanners )
+
+		@scanners = {}
+		@compiled_patterns = {}
+		
+		scanners.each { |name, value|
+			if value.is_a? Class
+				@scanners[name] = value.new
+			else
+				@scanners[name] = value
+			end
+		}
+
+		@verbose = false
+
+		log("scanners: #{@scanners.inspect}")
+
+	end
+
+
+	def scrape(doc) 
+		NodeScraper.new( self, nil, nil, doc )
+	end
+
+
+	module Scanners
+		
+		class BaseScanner
+			def scan(data)
+				data.to_s
+			end
+		end
+
+		class DateTimeScanner < BaseScanner
+			def scan( data )
+				if data.is_a? Hpricot::Elem
+					data = data.inner_text
+				end
+				Time.parse(data)
+			end
+		end
+
+		class TextLookupScanner < BaseScanner
+		end
+
+		class TextileScanner < BaseScanner
+			def scan( data )
+				if data.is_a? Hpricot::Elem
+					data = data.inner_text
+				end
+			end
+		end
+
+		class TextScanner < BaseScanner
+			def scan( data )
+				if data.is_a? Hpricot::Elem
+					data = data.inner_text
+				else
+					data = data.to_s
+				end
+				data
+			end
+		end
+
+		class LineScanner < BaseScanner
+		end
+
+
+	end
+
+end
+
+#
+#############################################################################
+#