metrocot 1.0.0

data/History.txt

@@ -0,0 +1,7 @@
+=== 1.0.0 / 2009-01-02
+
+* First working version
+
+  * simple object collection based on single string specification
+  * scans several test URLs without problems
+

data/Manifest.txt

@@ -0,0 +1,7 @@
+History.txt
+Manifest.txt
+README.txt
+Rakefile
+bin/metrocot
+lib/metrocot.rb
+test/test_metrocot.rb

data/README.txt

@@ -0,0 +1,98 @@
+= metrocot
+
+* http://www.metrocascade.com/mdn/opensource/metrocot (url)
+
+== DESCRIPTION:
+
+Metrocot builds on top of Hpricot to allow scraping of list data from HTML pages 
+with a minimum of code and page specific information.  The specification is done 
+is a very compact readable format.
+
+
+== FEATURES/PROBLEMS:
+
+* extremely concise specification via single string parameter (not even a DSL)
+* leverages Hpricot to locate reference points in input data  
+* comes with standard scanners for Time, String, etc
+* modular data validation through use of user defined scanners
+
+
+== SYNOPSIS:
+
+require 'rubygems'
+require 'metrocot'
+
+class Event < Object
+
+    attr_accessor :starts_at, :title, :description, :url
+            
+    def initialize( starts_at, title, description, url )
+        @starts_at = starts_at
+        @title = title
+        @description = description
+        @url = url
+    end
+
+end
+        
+mce_url = "http://www.musiccorner.ca/calendar.html"
+mce_doc = open(URI.parse(mce_url)) { |data| Hpricot(data) }
+
+scraper = Metrocot.new(
+            :starts_at => Metrocot::Scanners::DateTimeScanner,
+            :description => Metrocot::Scanners::TextScanner,
+            :title => Metrocot::Scanners::TextScanner
+)
+        
+mce_events = scraper.scrape(mce_doc).descend("//div[@id='content']/table/tr/td") { |td|
+    td.collect( "starts_at=.//h3 ... title=.//h2 ... description=((.//p )+)" ) { |starts_at, title, description| Event.new( starts_at, title, description, mce_url ) }
+}.values.flatten
+
+puts "Found #{mce_events.size} mce events:"
+mce_events.each_with_index { |event, event_index|
+    puts "%3d %20s %s" % [event_index, event.starts_at, event.title]
+}
+
+
+== REQUIREMENTS:
+
+* Hpricot
+
+== INSTALL:
+
+* sudo gem install metricot
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2009 Metro Cascade Media Inc
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
+== ALL HAIL WHY
+
+We are like tiny pleasantly chirping hex bugs coding away on the
+shoulders of Why so that we can create more, and do more with less
+code, not by virtue of any sharpness of mind on our part, or any
+other distinction, but because we are carried high and raised up
+by his giant size.
+

data/Rakefile

@@ -0,0 +1,12 @@
+# -*- ruby -*-
+
+require 'rubygems'
+require 'hoe'
+require './lib/metrocot.rb'
+
+Hoe.new('metrocot', Metrocot::VERSION) do |p|
+  p.rubyforge_name = 'metrocot' # if different than lowercase project name
+  p.developer('Helmut Hissen', 'helmut@zeebar.com')
+end
+
+# vim: syntax=Ruby

data/bin/metrocot

@@ -0,0 +1,109 @@
+#!/usr/bin/env ruby 
+############################################################################
+#
+# Copyright (c) 2009 Metro Cascade Media Inc
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+#############################################################################
+#
+#  Helmut Hissen <helmut@zeebar.com> (Metro Cascade Media Inc)
+#  January 1 2009
+#
+#############################################################################
+#
+#  We are like tiny pleasantly chirping hex bugs coding away on the 
+#  shoulders of Why so that we can create more, and do more with less 
+#  code, not by virtue of any sharpness of mind on our part, or any 
+#  other distinction, but because we are carried high and raised up 
+#  by his giant size.
+#
+#############################################################################
+#
+
+
+require 'rubygems'
+require 'hpricot'
+require 'metrocot'
+require 'open-uri'
+
+# example url - these guys give amazing concerts for kids
+url = "http://www.musiccorner.ca/calendar.html"
+
+# example xpath root - depends on the structure of the document
+xpath_root = "//div[@id='content']/table/tr/td"
+
+# esample metrocot scan specification - also depends on page
+# notice how the values we pull out get names and that these 
+# names map to a specific scanner decleared when the scraper 
+# was created.
+#
+# the following patterns are supported:
+#   "ABC"                   a fixed string
+#   /[abc]+/                Regexp
+#   ...                     anything
+#   ./h1/p                  Hpricot path 
+#   (PATTERN1 PATTERN2)     composite pattern
+#   PATTERN1+               one or more occurence of
+#   SPACE                   optional white space
+#   NAME=PATTERN            pull data out and scan, then collect
+#
+mspec = "starts_at=.//h3 ... title=.//h2 ... description=((.//p )+)"
+
+if ARGV.size > 0
+	url = ARGV[0]
+	if ARGV.size > 1
+		xpath_root = ARGV[1]
+		if ARGV.size > 2
+			mspec = ARGV[2]
+		end
+	end
+end
+
+
+doc = open(URI.parse(url)) { |data| Hpricot(data) }
+
+class Event < Object
+    attr_accessor :starts_at, :title, :description, :url
+    def initialize( starts_at, title, description, url )
+        @starts_at = starts_at
+        @title = title
+        @description = description
+        @url = url
+    end
+end
+
+scraper = Metrocot.new(
+   :starts_at => Metrocot::Scanners::DateTimeScanner,
+   :description => Metrocot::Scanners::TextScanner,
+   :title => Metrocot::Scanners::TextScanner
+)
+
+events = scraper.scrape(doc).descend(xpath_root) { |td|
+    td.collect( mspec ) { |starts_at, title, description|
+        Event.new( starts_at, title, description, url )
+    }
+}.values.flatten
+
+puts "Found #{events.size} events:"
+events.each_with_index { |event, event_index|
+    puts "%3d %20s %s" % [event_index, event.starts_at, event.title]
+}
+