metrocot 1.0.0 → 1.0.2

data/History.txt

@@ -1,3 +1,13 @@
+=== 1.0.2 / 2009-01-08
+
+  * added docs and examples
+  * created tests for examples and made those work as well
+  * added StrippingTextScanner
+
+=== 1.0.1 / 2009-01-03
+
+  * checked in with some initial docs
+
 === 1.0.0 / 2009-01-02
 
 * First working version

data/README.txt

@@ -4,9 +4,9 @@
 
 == DESCRIPTION:
 
-Metrocot builds on top of Hpricot to allow scraping of list data from HTML pages 
-with a minimum of code and page specific information.  The specification is done 
-is a very compact readable format.
+Metrocot builds on Hpricot to allow scraping of list data from HTML pages 
+with a minimum of code and page specific information.  The specification is 
+done in a very compact readable format.
 
 
 == FEATURES/PROBLEMS:
@@ -19,39 +19,39 @@ is a very compact readable format.
 
 == SYNOPSIS:
 
-require 'rubygems'
-require 'metrocot'
+    require 'rubygems'
+    require 'metrocot'
 
-class Event < Object
+    class Event < Object
 
-    attr_accessor :starts_at, :title, :description, :url
+        attr_accessor :starts_at, :title, :description, :url
             
-    def initialize( starts_at, title, description, url )
-        @starts_at = starts_at
-        @title = title
-        @description = description
-        @url = url
+        def initialize( starts_at, title, description, url )
+            @starts_at = starts_at
+            @title = title
+            @description = description
+            @url = url
+        end
+    
     end
-
-end
         
-mce_url = "http://www.musiccorner.ca/calendar.html"
-mce_doc = open(URI.parse(mce_url)) { |data| Hpricot(data) }
+    mce_url = "http://www.musiccorner.ca/calendar.html"
+    mce_doc = open(URI.parse(mce_url)) { |data| Hpricot(data) }
 
-scraper = Metrocot.new(
+    scraper = Metrocot.new(
             :starts_at => Metrocot::Scanners::DateTimeScanner,
             :description => Metrocot::Scanners::TextScanner,
-            :title => Metrocot::Scanners::TextScanner
-)
+            :title => Metrocot::Scanners::StrippingTextScanner
+    )
         
-mce_events = scraper.scrape(mce_doc).descend("//div[@id='content']/table/tr/td") { |td|
-    td.collect( "starts_at=.//h3 ... title=.//h2 ... description=((.//p )+)" ) { |starts_at, title, description| Event.new( starts_at, title, description, mce_url ) }
-}.values.flatten
-
-puts "Found #{mce_events.size} mce events:"
-mce_events.each_with_index { |event, event_index|
-    puts "%3d %20s %s" % [event_index, event.starts_at, event.title]
-}
+    mce_events = scraper.scrape(mce_doc).descend("//div[@id='content']/table/tr/td") { |td|
+        td.collect( "starts_at=.//h3 ... title=.//h2 ... description=((.//p )+)" ) { |starts_at, title, description| Event.new( starts_at, title, description, mce_url ) }
+    }.values.flatten
+
+    puts "Found #{mce_events.size} mce events:"
+    mce_events.each_with_index { |event, event_index|
+        puts "%3d %20s %s" % [event_index, event.starts_at, event.title]
+    }
 
 
 == REQUIREMENTS:

data/Rakefile

@@ -5,8 +5,11 @@ require 'hoe'
 require './lib/metrocot.rb'
 
 Hoe.new('metrocot', Metrocot::VERSION) do |p|
-  p.rubyforge_name = 'metrocot' # if different than lowercase project name
+  p.rubyforge_name = 'metrocot' 
   p.developer('Helmut Hissen', 'helmut@zeebar.com')
+  p.summary = "An Hpricot based tool for harvesting list-like data from HTML pages."
+  p.remote_rdoc_dir = '' # Release to root
 end
 
+
 # vim: syntax=Ruby

data/lib/metrocot.rb

@@ -1,4 +1,8 @@
 
+# Author::	Helmut Hissen (mailto:helmut@zeebar.com) 
+# Copyright::	Copyright(c) 2009 Metro Cascade Media, Inc.
+# License::	Distributed under the BSD open source license
+#
 #############################################################################
 #
 # Copyright (c) 2009 Metro Cascade Media Inc
@@ -24,9 +28,6 @@
 #
 #############################################################################
 #
-#  Helmut Hissen <helmut@zeebar.com> (Metro Cascade Media Inc)
-#  January 1 2009
-#
 #############################################################################
 #
 #  We are like tiny pleasantly chirping hex bugs coding away on the
@@ -38,9 +39,39 @@
 #############################################################################
 #
 
+# == Purpose
+# This class implements the main Metrocot HTML scanner and a number of handy
+# input scanners (for grabbing time, numbers, or text from HTML).  The purpose
+# of the Metrocot is to scan a XML dom for the patterns specified in the 
+# Metrocot pattern language.
+#
+# == Pattern Language
+# The Metrocot pattern language allows for the following types of patterns:
+# 
+# +...+:: matches anything
+# +"some string":: matches that string 
+# +/(some|pattern)/ matches that regexp pattern
+# +./HPRICOT_PATH+:: matches a certain type of dom subtree  
+# +SPACE+:: matches zero or more white spaces
+# +(PATTERN_A PATTERN_B):: matches PATTERN_A followed by PATTERN_B
+# +PATTERN\++:: matches one or more occurrences of PATTERN 
+#
+# == Usage
+# 0) create a Metricot and define the types of fields you want to extract (and their names).
+# 1) use Hpricot to get the doc's dom
+# 2) use descend(xpath) to create a NodeScraper rooted at the Hpricot node(s) matching the xpath
+# 3) use collect(pattern) to collect all entries found in the HTML which match the Metricot pattern
+
 class Metrocot < Object
 
-	VERSION = '1.0.0'
+	VERSION = '1.0.2'
+
+
+	# represents a subtree withing a metrocot dom.  the semantics are roughly equivalent 
+	# to what you get from select-dragging your mouse pointer through a section of an html
+	# doc in your web browser.  Thats is, a range specifies the first and last node in the
+	# pre-fix traversal of the dom.  Additionally, the first and last node may be truncated
+	# (at their tail and head respectively) if they are text nodes.
 
 	class MatchRange
 		
@@ -126,6 +157,9 @@ class Metrocot < Object
 	end
 
 
+	#
+	# base class for all other patterns.  Provides some reasonable default behaviours.
+	#
 
 	class BasePattern
 
@@ -156,7 +190,11 @@ class Metrocot < Object
 		end
 
 		def description
-			self.class.name
+			if name 
+				"#{self.class.name} \"#{name}\""
+			else
+				self.class.name
+			end
 		end
 
 
@@ -247,6 +285,8 @@ class Metrocot < Object
 	end
 
 
+	# matches a certain Hpricot path
+
 	class PathPattern  < BasePattern 
 
 		def initialize( source, path )
@@ -311,6 +351,8 @@ class Metrocot < Object
 	
 	end
 
+
+	# matches zero or more white spaces
 	
 	class OptSpacePattern < BasePattern 
 
@@ -332,47 +374,58 @@ class Metrocot < Object
 		end
 
 		def priority
-			-7
+			-8
 		end
 
 		def each_match( match_range, match_map )
+
+			result = nil
+
 			super(match_range, match_map)
-			match_start_index = match_range.start_index
+
+			match_start_index  = match_range.start_index
 			match_start_offset = match_range.start_offset
-			match_end_index = match_range.start_index
-			match_end_offset = match_range.start_offset
+			match_end_index    = match_range.start_index
+			match_end_offset   = match_range.start_offset
+
+			raise "negative range #{match_range}" if match_start_index > match_end_index
+			raise "negative range #{match_range}" if match_start_index == match_end_index && match_start_offset > match_end_offset
 
 			# consume rest of first text node  
 
 			hnodes = match_range.hnodes
+			hnode_text = nil
 
 			if hnodes[match_start_index] && hnodes[match_start_index].text?
-					hnode_text = hnodes[match_start_index].inner_text
-					while match_end_offset < hnode_text.size && (/\s+/.=== hnode_text[match_start_offset .. match_end_offset])
-						match_end_offset += 1
-					end
+	
+				hnode_text = hnodes[match_start_index].inner_text
+				while match_end_offset < hnode_text.size && (/^\s+$/.=== hnode_text[match_start_offset .. match_end_offset])
+					match_end_offset += 1
+				end
 					
-					if match_end_offset > match_start_offset 
-						if match_end_offset >= hnode_text.size
-							match_range = match_range.tail( match_end_index + 1, 0 )
-							log( "matched entire string of #{match_end_offset - match_start_offset} spaces" )
-						else
-							match_range = match_range.tail( match_end_index, match_end_offset )
-							log( "matched first #{match_end_offset - match_start_offset} leading spaces" )
-						end
+				if match_end_offset >= match_start_offset 
+					if match_end_offset >= hnode_text.size
+						log( "matched entire string of #{match_end_offset - match_start_offset} spaces" )
+					else
+						log( "matched first #{match_end_offset - match_start_offset} leading spaces" )
 					end
+				end
 			end
 
+			sp_match_range = match_range.crop( match_start_index, match_start_offset, match_end_index, match_end_offset )
+
 			result = with_scanned_match_data( match_map, hnodes[match_start_index ... match_end_index] ) { |match_map|
-				yield( match_range, match_map ) 
+				yield( sp_match_range, match_map ) 
 			}
-			result
 			
+			result
 		end
 
 	end
 
 
+	# matches one or more occurences of some other pattern
+
 	class OneOrMorePattern < BasePattern 
 
 		def initialize(repeatee) 
@@ -458,6 +511,8 @@ class Metrocot < Object
 	end
 
 
+	# Matches anything.
+
 	class AnythingPattern < BasePattern 
 
 		def description
@@ -474,9 +529,25 @@ class Metrocot < Object
 		# it just expands to fill whatever gap
 
 		def each_match( match_range, match_map )
-			with_scanned_match_data( match_map, match_range.hnodes[match_range.start_index .. match_range.end_index] ) { |match_map|
-				yield( match_range, match_map ) 
-			}
+
+			result = if match_range.start_index > match_range.end_index 
+				log( "empty range" )
+				nil
+			elsif match_range.start_index > match_range.end_index && match_range.start_offset > match_range.end_offset
+				log( "empty node" )
+				nil
+			elsif match_range.start_index == match_range.end_index || (match_range.start_index + 1 == match_range.end_index && match_range.end_offset == 0) && match_range.hnodes[match_range.start_index].text?
+				log( "single text node range #{match_range.describe}" )
+				raise "bad range #{match_range.describe}" if match_range.hnodes[match_range.start_index].nil?
+				with_scanned_match_data( match_map, match_range.hnodes[match_range.start_index].inner_text[match_range.start_offset...match_range.end_offset] ) { |match_map|
+					yield( match_range, match_map ) 
+				}
+			else
+				log( "multi node range #{match_range.describe}" )
+				with_scanned_match_data( match_map, match_range.hnodes[match_range.start_index ... match_range.end_index] ) { |match_map|
+					yield( match_range, match_map ) 
+				}
+			end
 		end
 
 
@@ -487,6 +558,8 @@ class Metrocot < Object
 	end
 
 
+	# Matches a certain text string or regex pattern
+
 	class TextPattern < BasePattern 
 
 		def initialize( source, text )
@@ -580,10 +653,9 @@ class Metrocot < Object
 
 			super(match_range, match_map)
 
-			match_start_index = match_range.start_index
-			match_start_offset = match_range.start_offset
-			match_end_index = match_range.start_index
-			match_end_offset = match_range.start_offset
+			match_index = match_range.start_index
+			match_offset = match_range.start_offset
+
 
 			# consume rest of first text node  
 
@@ -591,62 +663,72 @@ class Metrocot < Object
 
 			actual_match = nil
 
-			while match_start_index < match_range.end_index 
+			while match_index < match_range.end_index || (match_index == match_range.end_index && match_offset < match_range.end_offset)
 
-				while match_start_index < match_range.end_index && ! hnodes[match_start_index].text?
-					log( "not text: ##{match_start_index} #{hnodes[match_start_index].class}" )
-					match_start_index += 1
-					match_start_offset = 0
+				while (match_index < match_range.end_index || (match_index == match_range.end_index && match_offset < match_range.end_offset)) && ! hnodes[match_index].text?
+					log( "not text: ##{match_index} #{hnodes[match_index].class}" )
+					match_index += 1
+					match_offset = 0
 				end
 
-				unless match_start_index < match_range.end_index && hnodes[match_start_index].text?
+				unless (match_index < match_range.end_index || (match_index == match_range.end_index && match_offset < match_range.end_offset)) && hnodes[match_index].text?
 					log( "no match found" )
 					return nil
 				end
 
-				hnode_text = hnodes[match_start_index].inner_text
+				hnode_text = if match_index == match_range.end_index
+					hnodes[match_index].inner_text[0...match_range.end_offset]
+				else
+					hnodes[match_index].inner_text
+				end
 
-				log( "trying text match on: #{hnode_text[match_start_offset .. -1]}" )
+				log( "trying text match on: #{hnode_text[match_offset .. -1]}" )
 
-				match_offset = hnode_text.index( @text, match_start_offset )
+				next_match_offset = hnode_text.index( @text, match_offset )
 
-				if match_offset
+				if next_match_offset.nil?
+					log( "no match found for #{@text}" )
+					match_index += 1
+					match_offset = 0
+					next
+				end
 
-					actual_match = if @text.is_a? Regexp
-						hnode_text[match_offset..-1][@text]
-					else
-						@text
-					end
+				actual_match = if @text.is_a? Regexp
+					hnode_text[next_match_offset..-1][@text]
+				else
+					@text
+				end
+
+				log( "next text match at #{match_index}.#{next_match_offset}: #{actual_match}" )
 
-					match_end_offset = match_start_offset + actual_match.size
-					match_start_offset = match_start_offset + actual_match.size
+				match_start_offset = next_match_offset
+				match_end_offset = match_start_offset + actual_match.size
 							
-					if match_end_offset >= match_start_offset 
-						if match_end_offset >= hnode_text.size
-							log( "matched entire string of #{match_end_offset - match_start_offset} chars" )
-						else
-							log( "matched first #{match_end_offset - match_start_offset} chars" )
-						end
-						break
-					end
+				if match_end_offset >= hnode_text.size
+					log( "matched entire string of #{match_end_offset - match_start_offset} chars" )
+				else
+					log( "matched first #{match_end_offset - match_start_offset} chars" )
 				end
 
-				match_start_index += 1
-				match_start_offset = 0
+				result = with_scanned_match_data( match_map, actual_match ) { |match_map|
+					yield( match_range.crop( match_index, match_start_offset, match_index, match_end_offset), match_map ) 
+				}
+				
+				return result if result
+
+				match_offset = match_end_offset
 		
 			end
 
-
-			result = with_scanned_match_data( match_map, actual_match ) { |match_map|
-				yield( match_range.crop( match_start_index, match_start_offset, match_start_index, match_end_offset), match_map ) 
-			}
-			result
+			return nil
 			
 		end
 
 	end
 
 
+	# Matches a series of patterns
+
 	class CompositePattern < BasePattern 
 
 		attr_reader :parts
@@ -676,7 +758,7 @@ class Metrocot < Object
 		end
 
 
-		def each_split_match( match_range, match_map, parts_by_priority, ppx, part_matches ) 
+		def each_split_match( comp_match_range, match_map, parts_by_priority, ppx, part_matches ) 
 
 			pattern = nil
 
@@ -690,10 +772,13 @@ class Metrocot < Object
 
 			if ppx >= parts_by_priority.size
 				log("comp nothing left to do")
-				return yield( match_range, match_map ) 
+				return yield( comp_match_range, match_map ) 
 			end
 
 
+			match_range = comp_match_range
+			log("comp matching sub-pattern #{pattern.description} within #{match_range.describe}")
+
 			#
 			# figure out which gap this pattern is supposed to fill
 			#
@@ -718,15 +803,25 @@ class Metrocot < Object
 			if matched_on_left
 				log("comp matching must be right of #{matched_on_left.description}")
 				match_range = match_range.tail(matched_on_left.matched.end_index, matched_on_left.matched.end_offset) 
+			elsif matched_on_right
+				right_node = match_range.hnodes[matched_on_right.matched.start_index]
+				parent_of_right_node = right_node && right_node.parent
+				parent_ix_of_right_node = parent_of_right_node && node_scraper.hnode_index[parent_of_right_node]
+				if parent_ix_of_right_node && parent_ix_of_right_node >= match_range.start_index 
+					match_range = match_range.tail(parent_ix_of_right_node + 1, 0)
+					log("restricting left boundary to #{match_range} because would otherwise include subtree with right peer")
+				end
 			end
 
-			log("comp matching sub-pattern: #{pattern.description} at #{match_range.describe}")
+			log("comp matching sub-pattern #{pattern.description} at #{match_range.describe}")
 
 			pattern.each_match( match_range, match_map ) { |part_match_range, match_map| 
 
 				pattern.matched = part_match_range
 
-				result = each_split_match( match_range, match_map, parts_by_priority, ppx + 1, part_matches ) { |sub_match_range, sub_match_map|
+				log("found sub-pattern #{pattern.description} at #{part_match_range.describe}")
+
+				result = each_split_match( comp_match_range, match_map, parts_by_priority, ppx + 1, part_matches ) { |sub_match_range, sub_match_map|
 					yield( sub_match_range, match_map )
 				}
 
@@ -785,6 +880,9 @@ class Metrocot < Object
 	end
 
 
+	# rooted at a node in the dom, the node srcaper is used to collect all matches of 
+	# patterns.  
+
 	class NodeScraper
 
 		attr_accessor :mcot, :root, :parent, :hnode, :pattern_classes, :top_part_names, :verbose
@@ -883,15 +981,16 @@ class Metrocot < Object
 				result = nil
 				pattern.each_match( match_range, {} )  { |sub_match_range, match_map|
 					match_list = []
-					block_args = if (call_with == :positional) && top_part_names.size > 0
-						top_part_names.collect { |top_name|
+					result = if (call_with == :positional) && top_part_names.size > 0
+						block_args = top_part_names.collect { |top_name|
 							match_map[top_name]
 						}
+						log("calling pos scan block with: #{block_args.inspect}")
+						result = block.call( *block_args )
 					else
-						match_map
+						log("calling hash scan block with: #{match_map.inspect}")
+						result = block.call( match_map )
 					end
-					log("calling scan block with: #{block_args.join(", ")}")
-					result = block.call( *block_args )
 					if result
 						results << result
 						match_range = match_range.following( sub_match_range )
@@ -905,16 +1004,51 @@ class Metrocot < Object
 			results
 		end
 
+		# collects all occurrences of the data matching the pattern by calling the 
+		# yield block for everything part of the dom subtree matching the pattern.
+		# The block can reject the dom match by returning nil.  Anything other than
+		# nil will be appended to the list returned at the end.  
+		#
+		# Unlike collect_hashed(), the block will be given a list of parameter 
+		# values matching the list of named fields in the pattern.
+		#
+		# === Example
+		#
+		#     mcot.scrape(doc).descend( "//ul/li" ) { |li| 
+		#       li.collect( "liker=... \"likes\" likee=..." ) { |likes, liked| 
+		#         [ likes, liked ] 
+		#       }
+		#     }
+		#
 
 		def collect( pattern_s, &block )
 			collect_gen( pattern_s, :positional, &block )
 		end
 
 
+		# collects all occurrences of the data matching the pattern by calling the 
+		# yield block for everything part of the dom subtree matching the pattern.
+		# The block can reject the dom match by returning nil.  Anything other than
+		# nil will be appended to the list returned at the end.  
+		#
+		# Unlike collect(), the block will be given a map of parameter values 
+		# keyed by the names of the named fields in the pattern.
+		#
+		# === Example
+		#
+		#     mcot.scrape(doc).descend( "//ul/li" ) { |li| 
+		#       li.collect_hashed( "killer=... verb=/(stabbed|shot|strangled)/ victim=... \"(with|using)\" weapon=..." ) { |map|
+		#         Murder.new( map )
+		#       }
+		#     }
+
 		def collect_hashed( pattern_s, &block )
 			collect_gen( pattern_s, :map, &block )
 		end
 
+
+		# returns the scanner declared with that name when the metrocot was created 
+
 		def scanner_by_name( name )
 			return mcot.scanner_by_name(name)
 		end
@@ -922,17 +1056,125 @@ class Metrocot < Object
 	end
 
 
+	# Some useful scanners which should cover a good number of common 
+	# parsing scenarios.
+
+	module Scanners
+		
+		class BaseScanner
+			def scan(data)
+				data.to_s
+			end
+		end
+
+
+		# Scans the hpricot element or text for a date in one of the 
+		# various formats accepted by Time.  Dependingon the kinds of
+		# dates expected in the doc, this may not be sufficient and 
+		# you may have to create a custom scanner.
+
+		class DateTimeScanner < BaseScanner
+			def scan( data )
+				if data.is_a? Hpricot::Elem
+					data = data.inner_text
+				end
+				Time.parse(data)
+			end
+		end
+
+
+		# Scans the dom subtree and converts it to textile where possible
+		# returning a string containing the Textile version
+
+		class TextileScanner < BaseScanner
+			def scan( data )
+				if data.is_a? Hpricot::Elem
+					data = data.inner_text
+				end
+			end
+		end
+
+
+		# just pulls out the plain text.  This will probably be the most
+		# commonly used scanner.
+
+		class TextScanner < BaseScanner
+			def scan( data )
+				if data.is_a? Hpricot::Elem
+					data = data.inner_text
+				else
+					data = data.to_s
+				end
+				data
+			end
+		end
+
+
+		# just pulls out the plain text and strips it.  This will probably one of the 
+		# most commonly used scanner.
+
+		class StrippingTextScanner < BaseScanner
+			def scan( data )
+				if data.is_a? Hpricot::Elem
+					data = data.inner_text
+				else
+					data = data.to_s
+				end
+				data && data.strip
+			end
+		end
+
+		# pulls out text and then dicards everything up to the end of line
+
+		class LineScanner < BaseScanner
+		end
+
+	end
+
 	def log( s )
 		puts( s ) if @verbose
 	end
 
 
+	# given a Symbol of a scanner (name), return a handle for that scanner
+	# declared when the metrocot was created.
 
 	def scanner_by_name( name )
 		@scanners[name]
 	end
 
 
+
+	attr_accessor :verbose
+
+
+	def initialize( scanners )
+
+		@scanners = {}
+		@compiled_patterns = {}
+		
+		scanners.each { |name, value|
+			if value.is_a? Class
+				@scanners[name] = value.new
+			else
+				@scanners[name] = value
+			end
+		}
+
+		@verbose = false
+
+		log("scanners: #{@scanners.inspect}")
+
+	end
+
+	#	
+	#
+
+	def scrape(doc) 
+		NodeScraper.new( self, nil, nil, doc )
+	end
+
+
 	def compile_pattern( pattern_s, node_scraper )	
 
 #		if @compiled_patterns.key? pattern_s 
@@ -1031,80 +1273,6 @@ class Metrocot < Object
 		
 	end
 
-
-	attr_accessor :verbose
-
-
-	def initialize( scanners )
-
-		@scanners = {}
-		@compiled_patterns = {}
-		
-		scanners.each { |name, value|
-			if value.is_a? Class
-				@scanners[name] = value.new
-			else
-				@scanners[name] = value
-			end
-		}
-
-		@verbose = false
-
-		log("scanners: #{@scanners.inspect}")
-
-	end
-
-
-	def scrape(doc) 
-		NodeScraper.new( self, nil, nil, doc )
-	end
-
-
-	module Scanners
-		
-		class BaseScanner
-			def scan(data)
-				data.to_s
-			end
-		end
-
-		class DateTimeScanner < BaseScanner
-			def scan( data )
-				if data.is_a? Hpricot::Elem
-					data = data.inner_text
-				end
-				Time.parse(data)
-			end
-		end
-
-		class TextLookupScanner < BaseScanner
-		end
-
-		class TextileScanner < BaseScanner
-			def scan( data )
-				if data.is_a? Hpricot::Elem
-					data = data.inner_text
-				end
-			end
-		end
-
-		class TextScanner < BaseScanner
-			def scan( data )
-				if data.is_a? Hpricot::Elem
-					data = data.inner_text
-				else
-					data = data.to_s
-				end
-				data
-			end
-		end
-
-		class LineScanner < BaseScanner
-		end
-
-
-	end
-
 end
 
 #

data/test/test_metrocot.rb

@@ -20,21 +20,32 @@ class TestMetrocot < Test::Unit::TestCase
     end
   end
 
+  class Murder < Object
+    attr_accessor :killer, :mod, :weapon, :victim
+    def initialize( map )
+        @killer = map[:killer]
+        @mod = map[:mod]
+        @weapon = map[:weapon]
+        @victim = map[:victim]
+    end
+  end
+
+
   def test_nothing
 
 	html = "<html><head></head><body><h1>hello!</h1><h2>A</h2><p>a 1</p><p>a 2</p><h2>B</h2><p>b 1</p><p>b 2</p><h2>C</h2><p>c 1</p><p>c 2</p></body></html>"
 
 	doc = Hpricot(html)
 
-	scraper = Metrocot.new(
+	mcot = Metrocot.new(
    		:a => Metrocot::Scanners::TextScanner,
    		:b => Metrocot::Scanners::TextScanner,
    		:c => Metrocot::Scanners::TextScanner
 	)
 
-#	scraper.verbose = true
+#	mcot.verbose = true
 
-	assert_equal( [[]], scraper.scrape(doc).descend("//html/body") { |td|
+	assert_equal( [[]], mcot.scrape(doc).descend("//html/body") { |td|
     		td.collect( "a=.//h3 b=.//p c=.//p" ) { |a, b, c|
         		Abc.new( a, b, c )
     		}
@@ -42,21 +53,22 @@ class TestMetrocot < Test::Unit::TestCase
 
   end
 
+
   def test_abc
 
-	html = "<html><head></head><body><h1>hello!</h1><h2>A</h2><p>a 1</p><p>a 2</p><h2>B</h2><p>b 1</p><p>b 2</p><h2>C</h2><p>c 1</p><p>c 2</p></body></html>"
+	html = "<html><head></head><body><h1>hello!</h1><h2>A</h2><h3>where are the cartoon foxes?</h3><p>a 1</p><p>a 2</p><h2>B</h2><p>b 1</p><p>b 2</p><h2>C</h2><p>c 1</p><p>c 2</p><p>Inquiring minds want to know!</p></body></html>"
 
 	doc = Hpricot(html)
 
-	scraper = Metrocot.new(
+	mcot = Metrocot.new(
    		:a => Metrocot::Scanners::TextScanner,
    		:b => Metrocot::Scanners::TextScanner,
    		:c => Metrocot::Scanners::TextScanner
 	)
 
-#	scraper.verbose = true
+#	mcot.verbose = true
 
-	abcs = scraper.scrape(doc).descend("//html/body") { |td|
+	abcs = mcot.scrape(doc).descend("//html/body") { |td|
     		td.collect( "a=.//h2 b=.//p c=.//p" ) { |a, b, c|
         		Abc.new( a, b, c )
     		}
@@ -66,5 +78,57 @@ class TestMetrocot < Test::Unit::TestCase
 
   end
 
+
+  def test_murder
+
+	html = %{
+<html>
+	<head>
+		<title>Murder by Numbers</title>
+	</head>
+	<body>
+		<h1>The Who Killed Who Hoedown</h1>
+		<ul>
+			<li>Bob strangled the plumber with piano wire.</li>
+			<li>Collonel Clinket stabbed Aunt Elizabeth with a screw driver, while Dick shot Harry with a hunting rifle.</li>
+			<li>Other than that, not much happened here today.  Honestly!</li>
+			<li>Accidents do and will happen and that's what happened last Friday.</li>
+		</ul>
+		<p>
+			ps: Collonel Clinket would have stabbed Aunt Elizabeth with an ice pick if that had been available.
+		</p>
+	</body>
+</html>
+}
+
+	doc = Hpricot(html)
+
+	mcot = Metrocot.new(
+   		:killer => Metrocot::Scanners::StrippingTextScanner,
+   		:victim => Metrocot::Scanners::StrippingTextScanner,
+   		:mod    => Metrocot::Scanners::StrippingTextScanner,
+   		:weapon => Metrocot::Scanners::StrippingTextScanner
+	)
+
+#	mcot.verbose = true
+
+	murder_strings = []
+        mcot.scrape(doc).descend( "//ul/li" ) { |li|
+        	li.collect_hashed( "killer=... mod=/(stabbed|shot|strangled)/ victim=... /(with|using)/ weapon=... /[.,]/" ) { |map| Murder.new( map ) }
+        }.each { |node, murders|
+		murders.each { |murder|
+			ms = "    %-20s %-20s %-10s %-20s" % [murder.killer, murder.victim, murder.mod, murder.weapon]
+#			puts ms
+			murder_strings << ms
+		}
+	}
+
+	assert_equal([
+		"    Bob                  the plumber          strangled  piano wire          ",
+ 		"    Collonel Clinket     Aunt Elizabeth       stabbed    a screw driver      ",
+ 		"    while Dick           Harry                shot       a hunting rifle     "].sort, murder_strings.sort)
+
+  end
+
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: metrocot
 version: !ruby/object:Gem::Version 
-  version: 1.0.0
+  version: 1.0.2
 platform: ruby
 authors: 
 - Helmut Hissen
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-01-05 00:00:00 -08:00
+date: 2009-01-08 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -22,7 +22,7 @@ dependencies:
       - !ruby/object:Gem::Version 
         version: 1.8.2
     version: 
-description: Metrocot builds on top of Hpricot to allow scraping of list data from HTML pages  with a minimum of code and page specific information.  The specification is done  is a very compact readable format.
+description: Metrocot builds on Hpricot to allow scraping of list data from HTML pages  with a minimum of code and page specific information.  The specification is  done in a very compact readable format.
 email: 
 - helmut@zeebar.com
 executables: 
@@ -67,6 +67,6 @@ rubyforge_project: metrocot
 rubygems_version: 1.2.0
 signing_key: 
 specification_version: 2
-summary: Metrocot builds on top of Hpricot to allow scraping of list data from HTML pages  with a minimum of code and page specific information
+summary: An Hpricot based tool for harvesting list-like data from HTML pages.
 test_files: 
 - test/test_metrocot.rb