object-scraper 0.0.2

data/Manifest

@@ -0,0 +1,10 @@
+Manifest
+README.rdoc
+Rakefile
+lib/object-scraper.rb
+lib/object-scraper/scraper.rb
+object-scraper.gemspec
+spec/data/twitter.html
+spec/object-scraper/scraper_spec.rb
+spec/spec.opts
+spec/spec_helper.rb

data/README.rdoc

@@ -0,0 +1,53 @@
+= Object Scraper
+
+== Description
+
+Object scraper is a thin wrapper for hpricot to enable receipt-like
+extraction of ruby objects from various web sites.
+
+== Install
+
+=== Gem
+
+  gem install object-scraper --source http://gemcutter.org
+
+=== Rails
+
+  config.gem 'object-scraper', :source => 'http://gemcutter.org'
+
+== Example
+
+  class Entry < Object
+    attr_accessor :text, :date
+  end
+  
+  uri     = "http://twitter.com/twitter"
+  pattern = ".status"
+
+  Scraper.define(:twitter, :class => :entry, :source => uri, :node => pattern) do |s|
+    s.text { |node| node.at(".entry-content").inner_html }
+    s.date { |node| DateTime.parse(node.at(".timestamp")[:data][/\'.*\'/].delete("'")) }
+  end
+
+  @objects = Scraper.parse(:twitter)
+
+== Advanced Example
+
+It is possible to use other existing HTML parsers instead of hpricot.
+Just overwrite the according proc object.
+
+  require 'nokogiri'
+  Scraper.scrape_source_with = Proc.new { |source| Nokogiri::HTML(source) }
+  
+  Scraper.define(:twitter, :class => :entry, :source => uri, :node => pattern) do |s|
+    # initialize your objects here accordingly
+  end
+
+== Rails
+
+All scraper definitions sitting in RAILS_ROOT/scrapers will be taken into account
+automatically when you use object-scraper as a gem in your rails project.
+
+== Author
+
+- Maintained by {Enrico Genauck}[mailto:kontakt@enricogenauck.de]

data/Rakefile

@@ -0,0 +1,16 @@
+require 'rubygems'
+require 'rake'
+require 'echoe'
+
+Echoe.new('object-scraper', '0.0.2') do |p|
+  p.summary         = "Recipe like object extraction from HTML sources"
+  p.description     = "Object scraper is a thin wrapper for hpricot to enable recipe-like extraction of ruby objects from various web sites."
+  p.url             = "http://github.com/enricogenauck/object-scraper"
+  p.author          = "Enrico Genauck"
+  p.email           = "kontakt@enricogenauck.de"
+  p.ignore_pattern  = ["tmp/*", "script/*"]
+  p.development_dependencies = []
+  p.runtime_dependencies = ["hpricot >=0.8.2"]
+end
+
+Dir["#{File.dirname(__FILE__)}/tasks/*.rake"].sort.each { |ext| load ext }

data/lib/object-scraper/scraper.rb

@@ -0,0 +1,128 @@
+class Scraper
+
+  # Raised when a scraper is defined with the same name as a previously-defined scraper.
+  class DuplicateDefinitionError < RuntimeError
+  end
+
+  class << self
+    attr_accessor :scrapers
+    attr_accessor :scrape_source_with
+    attr_accessor :definition_file_paths
+  end
+
+  self.scrapers = {}
+  self.scrape_source_with = Proc.new { |source| Hpricot(source) }
+  self.definition_file_paths = %w(scrapers)
+
+  attr_reader :scraper_source, :scraper_node
+
+  def self.define(name, options = {}, &block)
+    instance = Scraper.new(name, options, &block)
+
+    if self.scrapers[name] 
+      raise DuplicateDefinitionError, "Scraper already defined: #{name}"
+    end
+
+    self.scrapers[name] = instance
+  end
+
+  def initialize(name, options = {}, &block) #:nodoc:
+    assert_valid_options(options)
+    @objects  = []
+    @class    = class_for(options[:class])
+    @scraper_source   = options[:source]
+    @scraper_node     = options[:node]
+    @block    = block
+  end
+
+  def self.get(name)
+    scraper_by_name(name)
+  end
+
+  def self.parse(name)
+    scraper_by_name(name).parse
+  end
+
+  def parse
+    doc = open(@scraper_source) { |f| Scraper.scrape_source_with.call(f) }
+    doc.search(@scraper_node).each do |n|
+      @current_node   = n
+      @current_object = @class.new
+      @objects << @current_object
+      @block.call(self)
+    end
+    @objects
+  end
+
+  def self.scraper_by_name(name)
+    scrapers[name.to_sym] or raise ArgumentError, "No such scraper: #{name.to_s}"
+  end
+
+  def method_missing(symbol, *args, &block)
+    if block_given?
+      @current_object.send("#{symbol}=", yield(@current_node))
+    else
+      @current_object.send("#{symbol}=", args.first)
+    end
+  end
+  
+  def self.find_definitions
+    definition_file_paths.each do |path|
+      require("#{path}.rb") if File.exists?("#{path}.rb")
+
+      if File.directory? path
+        Dir[File.join(path, '*.rb')].each do |file|
+          require file
+        end
+      end
+    end
+  end
+
+  private
+
+  def class_for(class_or_to_s)
+   if class_or_to_s.respond_to?(:to_sym)
+     Object.const_get(variable_name_to_class_name(class_or_to_s))
+   else
+     class_or_to_s
+   end
+  end
+  
+  def scraper_name_for(class_or_to_s)
+    if class_or_to_s.respond_to?(:to_sym)
+      class_or_to_s.to_sym
+    else
+      class_name_to_variable_name(class_or_to_s).to_sym
+    end
+  end
+
+  def class_name_to_variable_name(name)
+    name.to_s.gsub(/::/, '/').
+      gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
+      gsub(/([a-z\d])([A-Z])/,'\1_\2').
+      tr("-", "_").
+      downcase
+  end
+
+  def variable_name_to_class_name(name)
+    name.to_s.
+      gsub(/\/(.?)/) { "::#{$1.upcase}" }.
+      gsub(/(?:^|_)(.)/) { $1.upcase }
+  end
+
+  def assert_valid_options(options)
+    invalid_keys = options.keys - [:class, :source, :node] 
+    unless invalid_keys == []
+      raise ArgumentError, "Unknown arguments: #{invalid_keys.inspect}"
+    end
+    unless options[:class]
+      raise ArgumentError, "Missing argument: :class"
+    end
+    unless options[:source]
+      raise ArgumentError, "Missing argument: :source"
+    end
+    unless options[:node]
+      raise ArgumentError, "Missing argument: :node"
+    end
+  end
+end

data/lib/object-scraper.rb

@@ -0,0 +1,20 @@
+require 'object-scraper/scraper'
+require 'open-uri'
+require 'hpricot'
+
+# Shortcut for Scraper.extract
+#
+# Example:
+#   Scraper(:my_space)
+def Scraper(name)
+  Scraper.get(name)
+end
+
+if defined? Rails.configuration
+  Rails.configuration.after_initialize do
+    Scraper.definition_file_paths = [File.join(RAILS_ROOT, 'scrapers')]
+    Scraper.find_definitions
+  end
+else
+  Scraper.find_definitions
+end

data/object-scraper.gemspec

@@ -0,0 +1,33 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{object-scraper}
+  s.version = "0.0.2"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Enrico Genauck"]
+  s.date = %q{2009-12-10}
+  s.description = %q{Object scraper is a thin wrapper for hpricot to enable recipe-like extraction of ruby objects from various web sites.}
+  s.email = %q{kontakt@enricogenauck.de}
+  s.extra_rdoc_files = ["README.rdoc", "lib/object-scraper.rb", "lib/object-scraper/scraper.rb"]
+  s.files = ["Manifest", "README.rdoc", "Rakefile", "lib/object-scraper.rb", "lib/object-scraper/scraper.rb", "object-scraper.gemspec", "spec/data/twitter.html", "spec/object-scraper/scraper_spec.rb", "spec/spec.opts", "spec/spec_helper.rb"]
+  s.homepage = %q{http://github.com/enricogenauck/object-scraper}
+  s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Object-scraper", "--main", "README.rdoc"]
+  s.require_paths = ["lib"]
+  s.rubyforge_project = %q{object-scraper}
+  s.rubygems_version = %q{1.3.5}
+  s.summary = %q{Recipe like object extraction from HTML sources}
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<hpricot>, [">= 0.8.2"])
+    else
+      s.add_dependency(%q<hpricot>, [">= 0.8.2"])
+    end
+  else
+    s.add_dependency(%q<hpricot>, [">= 0.8.2"])
+  end
+end

data/spec/data/twitter.html

@@ -0,0 +1,731 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+  <head>
+        <script type="text/javascript">
+//<![CDATA[
+var page={};var onCondition=function(D,C,A,B){D=D;A=A?Math.min(A,5):5;B=B||100;if(D()){C()}else{if(A>1){setTimeout(function(){onCondition(D,C,A-1,B)},B)}}};
+//]]>
+</script>
+    <meta content="text/html; charset=utf-8" http-equiv="Content-Type" />
+<meta content="en-us" http-equiv="Content-Language" />
+<meta content="Always wondering what's happening. " name="description" />
+<meta content="no" http-equiv="imagetoolbar" />
+<meta content="width = 780" name="viewport" />
+<meta content="4FTTxY4uvo0RZTMQqIyhh18HsepyJOctQ+XTOu1zsfE=" name="verify-v1" />
+<meta content="1" name="page" />
+<meta content="NOODP" name="robots" />
+<meta content="n" name="session-loggedin" />
+<meta content="twitter" name="page-user-screen_name" />
+    <title id="page_title">Twitter (twitter) on Twitter</title>
+    <link href="http://a1.twimg.com/a/1259091217/images/favicon.ico" rel="shortcut icon" type="image/x-icon" />
+<link href="http://a1.twimg.com/a/1259091217/images/twitter_57.png" rel="apple-touch-icon" />
+    <link rel="alternate" href="http://twitter.com/statuses/user_timeline/783214.rss" title="twitter's Tweets" type="application/rss+xml" />
+    <link rel="alternate" href="http://twitter.com/favorites/783214.rss" title="twitter's Favorites" type="application/rss+xml" />
+  
+    
+    <link href="http://a1.twimg.com/a/1259091217/stylesheets/twitter.css?1259117012" media="screen" rel="stylesheet" type="text/css" />
+        <style type="text/css">
+      
+        body { background: #C0DEED url('http://a3.twimg.com/a/1259091217/images/bg-clouds.png') repeat-x; }
+      
+      
+      body {
+        background-position: 0 0;
+        padding-top: 0;
+      }
+      
+    </style>
+      <link href="http://a0.twimg.com/a/1259091217/stylesheets/following.css?1259117012" media="screen, projection" rel="stylesheet" type="text/css" />
+
+  </head>
+
+  <body class="account safari" id="profile">    
+    <script type="text/javascript">
+//<![CDATA[
+if (window.top !== window.self) {document.write = "";window.top.location = window.self.location; setTimeout(function(){document.body.innerHTML='';},1);window.self.onload=function(evt){document.body.innerHTML='';};}
+//]]>
+</script>
+    
+    
+    <div id="dim-screen"></div>
+    <ul id="accessibility" class="offscreen">
+  <li><a href="#content" accesskey="0">Skip past navigation</a></li>
+  <li>On a mobile phone? Check out <a href="http://m.twitter.com/">m.twitter.com</a>!</li>
+  <li><a href="#footer" accesskey="2">Skip to navigation</a></li>
+    <li><a href="#signin">Skip to sign in form</a></li>
+</ul>
+
+    
+
+    <div id="container" class="subpage">
+            <span id="loader" style="display:none"><img alt="Loader" src="http://a0.twimg.com/a/1259091217/images/loader.gif" /></span>
+      <div id="header">
+  <a href="http://twitter.com/" title="Twitter / Home" accesskey="1" id="logo">
+          <img alt="Twitter.com" height="36" src="http://a0.twimg.com/a/1259091217/images/twitter_logo_header.png" width="155" />    
+  </a>
+            <form method="post" id="sign_out_form" action="/sessions/destroy" style="display:none;">
+        <input name="authenticity_token" value="7e394453cc9d849cab133ccb3180e8ae4e6a0258" type="hidden" />
+      </form>
+
+      <ul class="top-navigation round">
+                  <li><a href="/login" accesskey="l">Login</a></li>
+          <li class="signup-link"><a href="/signup">Join Twitter!</a></li>
+              </ul>
+      </div>
+
+
+
+  <div id="profilebox_outer">
+    <div id="profilebird"><img alt="Profile_bird" height="48" id="profilebirdimg" src="http://a0.twimg.com/a/1259091217/images/profile_bird.png" width="48" /></div>
+    <div id="profilebox" class="clearfix">
+      <div id="profiletext">
+        <h1>Hey there! <strong>twitter</strong> is using Twitter.</h1>
+        <h2>Twitter is a free service that lets you keep in touch with people through the exchange of quick, frequent answers to one simple question: What's happening? <strong>Join today</strong> to start receiving <strong>twitter's</strong> tweets.</h2>
+      </div>
+      <div id="profilebutton">
+        <form name="account_signup_form" id="account_signup_form" action="/signup">
+          <input id="follow" name="follow" type="hidden" value="twitter" />
+          <input class="profilesubmit" id="join" name="commit" type="submit" value="Join today!" />
+        </form>
+        <p><small>Already using Twitter<br /> from your phone?  <a href="/account/complete">Click here</a>.</small></p>
+      </div>
+    </div>
+  </div>
+
+
+      
+
+      
+        <div class="content-bubble-arrow"></div>
+      
+      
+        <table cellspacing="0" class="columns">
+          <tbody>
+            <tr>
+              <td id="content" class="round-left column">
+                                <div class="wrapper">
+                  
+                  
+
+
+  
+
+
+
+
+  <div class="profile-user">
+    <div id="user_783214" class="user ">
+      <h2 class="thumb clearfix">
+        <a href="/account/profile_image/twitter?hreflang=en"><img alt="" border="0" height="73" id="profile-image" src="http://a1.twimg.com/profile_images/75075164/twitter_bird_profile_bigger.png" valign="middle" width="73" /></a>
+        <div class="screen-name">twitter</div>
+      </h2>
+    </div>
+  </div>
+
+
+<div class="section">
+
+  <div id="timeline_heading" style="display: none;">
+    <h1 id="heading"></h1>
+  </div>
+  <ol id='timeline' class='statuses'>
+          <li class="hentry u-twitter status latest-status" id="status_6191506635">
+    <span class="status-body">
+            <span class="entry-content">SMS delivery issues on AT&T <a href="http://bit.ly/7JFJ6H" class="tweet-url web" rel="nofollow" target="_blank">http://bit.ly/7JFJ6H</a></span>
+        <span class="meta entry-meta">
+      <a class="entry-date" rel="bookmark" href="http://twitter.com/twitter/status/6191506635">
+        <span class="published timestamp" data="{time:'Mon Nov 30 04:10:51 +0000 2009'}">about 8 hours ago</span>
+      </a>
+      <span>from <a href="http://twitterfeed.com" rel="nofollow">twitterfeed</a></span>
+      
+          </span>
+      </span>
+</li>
+          <li class="hentry u-twitter status" id="status_5989297065">
+    <span class="status-body">
+            <span class="entry-content">Fixing elevated error rate on twitter.com <a href="http://bit.ly/4xRf8U" class="tweet-url web" rel="nofollow" target="_blank">http://bit.ly/4xRf8U</a></span>
+        <span class="meta entry-meta">
+      <a class="entry-date" rel="bookmark" href="http://twitter.com/twitter/status/5989297065">
+        <span class="published timestamp" data="{time:'Mon Nov 23 22:13:27 +0000 2009'}">2:13 PM Nov 23rd</span>
+      </a>
+      <span>from <a href="http://twitterfeed.com" rel="nofollow">twitterfeed</a></span>
+      
+          </span>
+      </span>
+</li>
+          <li class="hentry u-twitter status" id="status_5875860574">
+    <span class="status-body">
+            <span class="entry-content">Abonnez-vous à @<a class="tweet-url username" href="/Twitter_FR">Twitter_FR</a>, le compte officiel de Twitter en français!</span>
+        <span class="meta entry-meta">
+      <a class="entry-date" rel="bookmark" href="http://twitter.com/twitter/status/5875860574">
+        <span class="published timestamp" data="{time:'Fri Nov 20 01:34:31 +0000 2009'}">5:34 PM Nov 19th</span>
+      </a>
+      <span>from web</span>
+      
+          </span>
+      </span>
+</li>
+          <li class="hentry u-twitter status" id="status_5875627626">
+    <span class="status-body">
+            <span class="entry-content">Nouvelle saveur : Twitter en Français! <a href="http://bit.ly/DeCHQ" class="tweet-url web" rel="nofollow" target="_blank">http://bit.ly/DeCHQ</a></span>
+        <span class="meta entry-meta">
+      <a class="entry-date" rel="bookmark" href="http://twitter.com/twitter/status/5875627626">
+        <span class="published timestamp" data="{time:'Fri Nov 20 01:25:37 +0000 2009'}">5:25 PM Nov 19th</span>
+      </a>
+      <span>from web</span>
+      
+          </span>
+      </span>
+</li>
+          <li class="hentry u-twitter status" id="status_5874789939">
+    <span class="status-body">
+            <span class="entry-content">RT @<a class="tweet-url username" href="/macgill">macgill</a>: Released refreshed Twitter privacy policy <a href="http://bit.ly/kYyQ6" class="tweet-url web" rel="nofollow" target="_blank">http://bit.ly/kYyQ6</a> and a new helpful trademark page <a href="http://bit.ly/2iGZgV" class="tweet-url web" rel="nofollow" target="_blank">http://bit.ly/2iGZgV</a>  Check  </span>
+    <a href="http://twitter.com/twitter/status/5874789939">...</a>    <span class="meta entry-meta">
+      <a class="entry-date" rel="bookmark" href="http://twitter.com/twitter/status/5874789939">
+        <span class="published timestamp" data="{time:'Fri Nov 20 00:53:33 +0000 2009'}">4:53 PM Nov 19th</span>
+      </a>
+      <span>from web</span>
+      
+          </span>
+      </span>
+</li>
+          <li class="hentry u-twitter status" id="status_5870049749">
+    <span class="status-body">
+            <span class="entry-content">Think globally, Tweet locally with geotagging. Opt-in! <a href="http://blog.twitter.com/2009/11/think-globally-tweet-locally.html" class="tweet-url web" rel="nofollow" target="_blank">http://blog.twitter.com/200...</a></span>
+        <span class="meta entry-meta">
+      <a class="entry-date" rel="bookmark" href="http://twitter.com/twitter/status/5870049749">
+        <span class="published timestamp" data="{time:'Thu Nov 19 21:54:41 +0000 2009'}">1:54 PM Nov 19th</span>
+      </a>
+      <span>from web</span>
+      
+          </span>
+      </span>
+</li>
+          <li class="hentry u-twitter status" id="status_5867297191">
+    <span class="status-body">
+            <span class="entry-content">RT @<a class="tweet-url username" href="/davewiner">davewiner</a>: The new Retweet is cool! (Scripting News) <a href="http://r2.ly/wvz4" class="tweet-url web" rel="nofollow" target="_blank">http://r2.ly/wvz4</a></span>
+        <span class="meta entry-meta">
+      <a class="entry-date" rel="bookmark" href="http://twitter.com/twitter/status/5867297191">
+        <span class="published timestamp" data="{time:'Thu Nov 19 20:09:04 +0000 2009'}">12:09 PM Nov 19th</span>
+      </a>
+      <span>from web</span>
+      
+          </span>
+      </span>
+</li>
+          <li class="hentry u-twitter status" id="status_5866757295">
+    <span class="status-body">
+            <span class="entry-content">What did the general say when he found out his army was going to lose the war? Retweet! (feature now available to all users)</span>
+        <span class="meta entry-meta">
+      <a class="entry-date" rel="bookmark" href="http://twitter.com/twitter/status/5866757295">
+        <span class="published timestamp" data="{time:'Thu Nov 19 19:48:01 +0000 2009'}">11:48 AM Nov 19th</span>
+      </a>
+      <span>from web</span>
+      
+          </span>
+      </span>
+</li>
+          <li class="hentry u-twitter status" id="status_5865461320">
+    <span class="status-body">
+            <span class="entry-content">Quick question: What's happening? <a href="http://blog.twitter.com/2009/11/whats-happening.html" class="tweet-url web" rel="nofollow" target="_blank">http://blog.twitter.com/200...</a></span>
+        <span class="meta entry-meta">
+      <a class="entry-date" rel="bookmark" href="http://twitter.com/twitter/status/5865461320">
+        <span class="published timestamp" data="{time:'Thu Nov 19 18:57:36 +0000 2009'}">10:57 AM Nov 19th</span>
+      </a>
+      <span>from web</span>
+      
+          </span>
+      </span>
+</li>
+          <li class="hentry u-twitter status" id="status_5838277342">
+    <span class="status-body">
+            <span class="entry-content">A picture is worth more than 140 characters. Shout out to @<a class="tweet-url username" href="/flickr">flickr</a> for their handy Flickr2Twitter app! <a href="/search?q=%23appwednesday" title="#appwednesday" class="tweet-url hashtag">#appwednesday</a></span>
+        <span class="meta entry-meta">
+      <a class="entry-date" rel="bookmark" href="http://twitter.com/twitter/status/5838277342">
+        <span class="published timestamp" data="{time:'Wed Nov 18 21:47:50 +0000 2009'}">1:47 PM Nov 18th</span>
+      </a>
+      <span>from web</span>
+      
+          </span>
+      </span>
+</li>
+          <li class="hentry u-twitter status" id="status_5812855148">
+    <span class="status-body">
+            <span class="entry-content">Wrong profile background pictures <a href="http://bit.ly/6lMhG" class="tweet-url web" rel="nofollow" target="_blank">http://bit.ly/6lMhG</a></span>
+        <span class="meta entry-meta">
+      <a class="entry-date" rel="bookmark" href="http://twitter.com/twitter/status/5812855148">
+        <span class="published timestamp" data="{time:'Wed Nov 18 01:11:32 +0000 2009'}">5:11 PM Nov 17th</span>
+      </a>
+      <span>from <a href="http://twitterfeed.com" rel="nofollow">twitterfeed</a></span>
+      
+          </span>
+      </span>
+</li>
+          <li class="hentry u-twitter status" id="status_5783517157">
+    <span class="status-body">
+            <span class="entry-content">Another first from the UK - MMS your pics to Twitter! <a href="http://bit.ly/5cm7R" class="tweet-url web" rel="nofollow" target="_blank">http://bit.ly/5cm7R</a></span>
+        <span class="meta entry-meta">
+      <a class="entry-date" rel="bookmark" href="http://twitter.com/twitter/status/5783517157">
+        <span class="published timestamp" data="{time:'Tue Nov 17 02:01:27 +0000 2009'}">6:01 PM Nov 16th</span>
+      </a>
+      <span>from web</span>
+      
+          </span>
+      </span>
+</li>
+          <li class="hentry u-twitter status" id="status_5782562902">
+    <span class="status-body">
+            <span class="entry-content">Breaking in the new office with an awesome performance by @<a class="tweet-url username" href="/1republic">1republic</a>. Streaming live at <a href="http://bit.ly/4C8Hvv" class="tweet-url web" rel="nofollow" target="_blank">http://bit.ly/4C8Hvv</a>.</span>
+        <span class="meta entry-meta">
+      <a class="entry-date" rel="bookmark" href="http://twitter.com/twitter/status/5782562902">
+        <span class="published timestamp" data="{time:'Tue Nov 17 01:25:13 +0000 2009'}">5:25 PM Nov 16th</span>
+      </a>
+      <span>from web</span>
+      
+          </span>
+      </span>
+</li>
+          <li class="hentry u-twitter status" id="status_5781410369">
+    <span class="status-body">
+            <span class="entry-content">Maintenance window Tuesday, November 17th at 11p Pacific <a href="http://bit.ly/4AqkLs" class="tweet-url web" rel="nofollow" target="_blank">http://bit.ly/4AqkLs</a></span>
+        <span class="meta entry-meta">
+      <a class="entry-date" rel="bookmark" href="http://twitter.com/twitter/status/5781410369">
+        <span class="published timestamp" data="{time:'Tue Nov 17 00:41:43 +0000 2009'}">4:41 PM Nov 16th</span>
+      </a>
+      <span>from <a href="http://twitterfeed.com" rel="nofollow">twitterfeed</a></span>
+      
+          </span>
+      </span>
+</li>
+          <li class="hentry u-twitter status" id="status_5778942639">
+    <span class="status-body">
+            <span class="entry-content">Official pics from the shiny new Twitter HQ! <a href="http://bit.ly/2oVk85" class="tweet-url web" rel="nofollow" target="_blank">http://bit.ly/2oVk85</a></span>
+        <span class="meta entry-meta">
+      <a class="entry-date" rel="bookmark" href="http://twitter.com/twitter/status/5778942639">
+        <span class="published timestamp" data="{time:'Mon Nov 16 23:08:56 +0000 2009'}">3:08 PM Nov 16th</span>
+      </a>
+      <span>from web</span>
+      
+          </span>
+      </span>
+</li>
+          <li class="hentry u-twitter status" id="status_5772564273">
+    <span class="status-body">
+            <span class="entry-content">Feels like the 1st day of school at the new office! Here's one last tribute to the old HQ: <a href="http://bit.ly/1BkDvi" class="tweet-url web" rel="nofollow" target="_blank">http://bit.ly/1BkDvi</a></span>
+        <span class="meta entry-meta">
+      <a class="entry-date" rel="bookmark" href="http://twitter.com/twitter/status/5772564273">
+        <span class="published timestamp" data="{time:'Mon Nov 16 18:56:52 +0000 2009'}">10:56 AM Nov 16th</span>
+      </a>
+      <span>from web</span>
+      
+          </span>
+      </span>
+</li>
+          <li class="hentry u-twitter status" id="status_5742847664">
+    <span class="status-body">
+            <span class="entry-content">Tweeps from 21 states and 5 countries gathered with @<a class="tweet-url username" href="/nasa">nasa</a> to tweet first-hand about the space shuttle launch tomorrow. <a href="http://bit.ly/nyOrn" class="tweet-url web" rel="nofollow" target="_blank">http://bit.ly/nyOrn</a></span>
+        <span class="meta entry-meta">
+      <a class="entry-date" rel="bookmark" href="http://twitter.com/twitter/status/5742847664">
+        <span class="published timestamp" data="{time:'Sun Nov 15 19:05:27 +0000 2009'}">11:05 AM Nov 15th</span>
+      </a>
+      <span>from web</span>
+      
+          </span>
+      </span>
+</li>
+          <li class="hentry u-twitter status" id="status_5687484356">
+    <span class="status-body">
+            <span class="entry-content">Missing "in reply to" links <a href="http://bit.ly/3HIBNe" class="tweet-url web" rel="nofollow" target="_blank">http://bit.ly/3HIBNe</a></span>
+        <span class="meta entry-meta">
+      <a class="entry-date" rel="bookmark" href="http://twitter.com/twitter/status/5687484356">
+        <span class="published timestamp" data="{time:'Fri Nov 13 18:52:05 +0000 2009'}">10:52 AM Nov 13th</span>
+      </a>
+      <span>from <a href="http://twitterfeed.com" rel="nofollow">twitterfeed</a></span>
+      
+          </span>
+      </span>
+</li>
+          <li class="hentry u-twitter status" id="status_5686368225">
+    <span class="status-body">
+            <span class="entry-content">Lots to look forward to next week with the big move to our new headquarters. @<a class="tweet-url username" href="/sara">sara</a> has outdone herself on the interior design! <a href="/search?q=%23twitterhq" title="#twitterhq" class="tweet-url hashtag">#twitterhq</a></span>
+        <span class="meta entry-meta">
+      <a class="entry-date" rel="bookmark" href="http://twitter.com/twitter/status/5686368225">
+        <span class="published timestamp" data="{time:'Fri Nov 13 18:09:17 +0000 2009'}">10:09 AM Nov 13th</span>
+      </a>
+      <span>from web</span>
+      
+          </span>
+      </span>
+</li>
+          <li class="hentry u-twitter status" id="status_5632730783">
+    <span class="status-body">
+            <span class="entry-content">Retweet feature temporarily disabled <a href="http://bit.ly/TU4h3" class="tweet-url web" rel="nofollow" target="_blank">http://bit.ly/TU4h3</a></span>
+        <span class="meta entry-meta">
+      <a class="entry-date" rel="bookmark" href="http://twitter.com/twitter/status/5632730783">
+        <span class="published timestamp" data="{time:'Wed Nov 11 23:18:48 +0000 2009'}">3:18 PM Nov 11th</span>
+      </a>
+      <span>from <a href="http://twitterfeed.com" rel="nofollow">twitterfeed</a></span>
+      
+          </span>
+      </span>
+</li>
+  </ol>  <div id="pagination">
+    <a href="/twitter?max_id=6191506635&amp;page=2&amp;twttr=true" class="round more" id="more" rel="next">more</a>  </div>
+
+</div>
+
+
+
+                </div>
+                              </td>
+              
+                <td id="side_base" class="column round-right">
+                                  
+                  <div id="side">
+                      
+<div id="profile" class="section profile-side">
+  <span class="section-links">
+      </span>
+  <address>
+    <ul class="about vcard entry-author">
+      
+      
+      
+        <li><span class="label">Name</span> <span class="fn">Twitter</span></li>
+                  <li><span class="label">Location</span> <span class="adr">San Francisco, CA</span></li>
+                          <li><span class="label">Web</span> <a href="http://twitter.com" class="url" rel="me nofollow" target="_blank">http://twitter.com</a></li>
+                          <li id="bio"><span class="label">Bio</span> <span class="bio">Always wondering what's happening. </span></li>
+              
+    </ul>
+  </address>
+        
+
+
+<div class="stats">
+  <table>
+    <tr>
+      <td>
+
+<a href="/twitter/following" id="following_count_link" class="link-following_page" rel="me" title="See who twitter is following">
+  <span id="following_count" class="stats_count numeric">123 </span>
+  <span class="label">Following</span>
+</a>
+
+</td>
+      <td>
+
+<a href="/twitter/followers" id="follower_count_link" class="link-followers_page" rel="me" title="See who's following twitter">
+  <span id="follower_count" class="stats_count numeric">2,700,999 </span>
+  <span class="label">Followers</span>
+</a>
+
+</td>
+              <td>
+
+<a href="/twitter/lists/memberships" id="lists_count_link" class="link-lists_page" rel="me" title="See which lists twitter is on">
+  <span id="lists_count" class="stats_count numeric">12,392 </span>
+  <span class="label">Listed</span>
+</a>
+
+</td>
+                </tr>
+  </table>
+</div>
+  
+</div>
+
+  <ul id="primary_nav" class="sidebar-menu">
+    <li id="profile_tab"><a href="/twitter" accesskey="u"><span id="update_count" class="stat_count">592</span><span>Tweets</span></a></li>
+    <li id="profile_favorites_tab"><a href="http://twitter.com/twitter/favorites" accesskey="f"><span>Favorites</span></a></li>
+  </ul>
+  
+  
+        <hr/>
+    <div id="side_lists">
+      <h2 class="sidebar-title"><span>Lists</span></h2>
+      
+        <ul class="sidebar-menu lists-links">
+          <li><a href="/twitter/team" class="list_574" data="{&quot;dispatch_action&quot;:&quot;list&quot;,&quot;mode&quot;:&quot;public&quot;,&quot;description&quot;:&quot;&quot;,&quot;uri&quot;:&quot;/twitter/team&quot;,&quot;subscriber_count&quot;:60480,&quot;slug&quot;:&quot;team&quot;,&quot;full_name&quot;:&quot;@twitter/team&quot;,&quot;user&quot;:&quot;twitter&quot;,&quot;name&quot;:&quot;Team&quot;,&quot;id&quot;:574,&quot;member_count&quot;:124}" title="@twitter/Team"><span>@twitter/<wbr/><b>team</b></span></a></li>
+        </ul>
+        <p class="sidebar-menu sidebar-menu-actions">
+          <span class="view-all"><a href="http://twitter.com/twitter/lists">View all</a></span>
+        </p>
+          </div>
+      
+
+<hr/>
+
+
+  <div id="following">
+    <h2 class="sidebar-title" id="fm_menu"><span>Following</span></h2>
+    
+    <div class="sidebar-menu">
+  <div id="following_list">
+    
+  <span class="vcard">
+    <a href="/troyholden" class="url" hreflang="en" rel="contact" title="troyholden"><img alt="troyholden" class="photo fn" height="24" src="http://a3.twimg.com/profile_images/471907441/4002551589_23daaeeca3_mini.jpg" width="24" /></a>  </span>
+
+
+  <span class="vcard">
+    <a href="/twitter_fr" class="url" hreflang="en" rel="contact" title="Twitter Français"><img alt="Twitter Français" class="photo fn" height="24" src="http://a3.twimg.com/profile_images/534477089/twitter_bird_profile_mini.png" width="24" /></a>  </span>
+
+
+  <span class="vcard">
+    <a href="/jreichhold" class="url" hreflang="en" rel="contact" title="jreichhold"><img alt="jreichhold" class="photo fn" height="24" src="http://a3.twimg.com/profile_images/54857067/Photo_1_mini.jpg" width="24" /></a>  </span>
+
+
+  <span class="vcard">
+    <a href="/imownbey" class="url" hreflang="en" rel="contact" title="Ian Miles Ownbey"><img alt="Ian Miles Ownbey" class="photo fn" height="24" src="http://a1.twimg.com/profile_images/64790496/Photo_2_mini.jpg" width="24" /></a>  </span>
+
+
+  <span class="vcard">
+    <a href="/dongwang218" class="url" hreflang="en" rel="contact" title="Dong Wang"><img alt="Dong Wang" class="photo fn" height="24" src="http://a1.twimg.com/profile_images/382072008/dong_mini.jpg" width="24" /></a>  </span>
+
+
+  <span class="vcard">
+    <a href="/bsuto" class="url" hreflang="en" rel="contact" title="Brian Sutorius"><img alt="Brian Sutorius" class="photo fn" height="24" src="http://a1.twimg.com/profile_images/452951844/Screen_shot_2009-10-04_at_3.39.37_PM_mini.png" width="24" /></a>  </span>
+
+
+  <span class="vcard">
+    <a href="/BFF" class="url" hreflang="en" rel="contact" title="Brandi"><img alt="Brandi" class="photo fn" height="24" src="http://a1.twimg.com/profile_images/511212714/care-bears_best-friends_mini.jpg" width="24" /></a>  </span>
+
+
+  <span class="vcard">
+    <a href="/dino" class="url" hreflang="en" rel="contact" title="Dino"><img alt="Dino" class="photo fn" height="24" src="http://a3.twimg.com/profile_images/508083749/Photo_2_mini.jpg" width="24" /></a>  </span>
+
+
+  <span class="vcard">
+    <a href="/francesca" class="url" hreflang="en" rel="contact" title="Francesca"><img alt="Francesca" class="photo fn" height="24" src="http://a1.twimg.com/profile_images/115081740/Singlephoto_mini.jpg" width="24" /></a>  </span>
+
+
+  <span class="vcard">
+    <a href="/th" class="url" hreflang="en" rel="contact" title="taylor harwin"><img alt="taylor harwin" class="photo fn" height="24" src="http://a1.twimg.com/profile_images/492316898/IMG_0058_mini.JPG" width="24" /></a>  </span>
+
+
+  <span class="vcard">
+    <a href="/taylorharwin" class="url" hreflang="fr" rel="contact" title="Taylor Harwin"><img alt="Taylor Harwin" class="photo fn" height="24" src="http://a1.twimg.com/profile_images/475719652/2847_626906472242_105819_36830072_2929508_n_mini.jpg" width="24" /></a>  </span>
+
+
+  <span class="vcard">
+    <a href="/mischahere" class="url" hreflang="en" rel="contact" title="Mischa Nachtigal"><img alt="Mischa Nachtigal" class="photo fn" height="24" src="http://a1.twimg.com/profile_images/435895728/waldo_mini.jpg" width="24" /></a>  </span>
+
+
+  <span class="vcard">
+    <a href="/cayley" class="url" hreflang="en" rel="contact" title="Cayley Torgeson"><img alt="Cayley Torgeson" class="photo fn" height="24" src="http://a1.twimg.com/profile_images/446923096/IMG_0072_mini.jpg" width="24" /></a>  </span>
+
+
+  <span class="vcard">
+    <a href="/meetutkarsh" class="url" hreflang="en" rel="contact" title="Utkarsh Srivastava"><img alt="Utkarsh Srivastava" class="photo fn" height="24" src="http://a1.twimg.com/profile_images/193389882/IMG_1906_mini.JPG" width="24" /></a>  </span>
+
+
+  <span class="vcard">
+    <a href="/twitter_es" class="url" hreflang="en" rel="contact" title="Twitter Español"><img alt="Twitter Español" class="photo fn" height="24" src="http://a3.twimg.com/profile_images/504883319/twitter_bird_profile_mini.png" width="24" /></a>  </span>
+
+
+  <span class="vcard">
+    <a href="/rion" class="url" hreflang="en" rel="contact" title="Rion"><img alt="Rion" class="photo fn" height="24" src="http://a1.twimg.com/profile_images/182109900/Photo_8_mini.jpg" width="24" /></a>  </span>
+
+
+  <span class="vcard">
+    <a href="/nancyjconnery" class="url" hreflang="en" rel="contact" title="Nancy  Connery"><img alt="Nancy  Connery" class="photo fn" height="24" src="http://a1.twimg.com/profile_images/93080012/Mommy_and_Morgan_mini.jpg" width="24" /></a>  </span>
+
+
+  <span class="vcard">
+    <a href="/tamtam2" class="url" hreflang="en" rel="contact" title="tamtam2"><img alt="tamtam2" class="photo fn" height="24" src="http://a1.twimg.com/profile_images/458987118/Taimay_Red_Outside_2_half_mini.jpg" width="24" /></a>  </span>
+
+
+  <span class="vcard">
+    <a href="/em33" class="url" hreflang="en" rel="contact" title="Emee"><img alt="Emee" class="photo fn" height="24" src="http://a3.twimg.com/profile_images/61480913/em_mini.PNG" width="24" /></a>  </span>
+
+
+  <span class="vcard">
+    <a href="/andr8a" class="url" hreflang="en" rel="contact" title="andr8a"><img alt="andr8a" class="photo fn" height="24" src="http://a3.twimg.com/profile_images/81960525/kitty_mini.jpg" width="24" /></a>  </span>
+
+
+  <span class="vcard">
+    <a href="/keerthi" class="url" hreflang="en" rel="contact" title="Keerthi Prakash"><img alt="Keerthi Prakash" class="photo fn" height="24" src="http://a3.twimg.com/profile_images/104888409/keerthi_2_2__mini.jpg" width="24" /></a>  </span>
+
+
+  <span class="vcard">
+    <a href="/Lukester" class="url" hreflang="en" rel="contact" title="Luke "><img alt="Luke " class="photo fn" height="24" src="http://a1.twimg.com/profile_images/543545486/image_mini.jpg" width="24" /></a>  </span>
+
+
+  <span class="vcard">
+    <a href="/sean" class="url" hreflang="ja" rel="contact" title="Sean"><img alt="Sean" class="photo fn" height="24" src="http://a3.twimg.com/profile_images/468339255/645882267_8CRyH-L_mini.jpg" width="24" /></a>  </span>
+
+
+  <span class="vcard">
+    <a href="/che" class="url" hreflang="en" rel="contact" title="Cheryl Palarca"><img alt="Cheryl Palarca" class="photo fn" height="24" src="http://a1.twimg.com/profile_images/399472616/Picture_2a_mini.jpg" width="24" /></a>  </span>
+
+
+  <span class="vcard">
+    <a href="/lg" class="url" hreflang="en" rel="contact" title="Larry Gadea"><img alt="Larry Gadea" class="photo fn" height="24" src="http://a1.twimg.com/profile_images/53283340/q90406211_4464_mini.jpg" width="24" /></a>  </span>
+
+
+  <span class="vcard">
+    <a href="/tiger" class="url" hreflang="en" rel="contact" title="Emily"><img alt="Emily" class="photo fn" height="24" src="http://a3.twimg.com/profile_images/257293259/724px-Tigerente_mini.jpg" width="24" /></a>  </span>
+
+
+  <span class="vcard">
+    <a href="/mattknox" class="url" hreflang="en" rel="contact" title="matt knox"><img alt="matt knox" class="photo fn" height="24" src="http://a3.twimg.com/profile_images/257712871/n14602342_30779909_9881_mini.jpg" width="24" /></a>  </span>
+
+
+  <span class="vcard">
+    <a href="/Magnuson" class="url" hreflang="en" rel="contact" title="Charles Magnuson"><img alt="Charles Magnuson" class="photo fn" height="24" src="http://a3.twimg.com/profile_images/371840833/twitter_pic_mini.jpg" width="24" /></a>  </span>
+
+
+  <span class="vcard">
+    <a href="/Charles" class="url" hreflang="en" rel="contact" title="Charles"><img alt="Charles" class="photo fn" height="24" src="http://a3.twimg.com/profile_images/529789365/4099673652_248e60ccf5_mini.jpg" width="24" /></a>  </span>
+
+
+  <span class="vcard">
+    <a href="/rsarver" class="url" hreflang="en" rel="contact" title="Ryan Sarver"><img alt="Ryan Sarver" class="photo fn" height="24" src="http://a3.twimg.com/profile_images/53700173/2448434960_65aba38823_t_mini.jpg" width="24" /></a>  </span>
+
+
+  <span class="vcard">
+    <a href="/ElizaSwan" class="url" hreflang="en" rel="contact" title="Robin"><img alt="Robin" class="photo fn" height="24" src="http://a3.twimg.com/profile_images/260125397/icon_mini.png" width="24" /></a>  </span>
+
+
+  <span class="vcard">
+    <a href="/sam" class="url" hreflang="en" rel="contact" title="Sam Luckenbill"><img alt="Sam Luckenbill" class="photo fn" height="24" src="http://a3.twimg.com/profile_images/65334155/601879580_479440f611_mini.jpg" width="24" /></a>  </span>
+
+
+  <span class="vcard">
+    <a href="/noradio" class="url" hreflang="en" rel="contact" title="Marcel Molina"><img alt="Marcel Molina" class="photo fn" height="24" src="http://a3.twimg.com/profile_images/53473799/marcel-euro-rails-conf_mini.jpg" width="24" /></a>  </span>
+
+
+  <span class="vcard">
+    <a href="/bakineggs" class="url" hreflang="en" rel="contact" title="Dan Barry"><img alt="Dan Barry" class="photo fn" height="24" src="http://a1.twimg.com/profile_images/59318058/n30408690_33614537_9476_square_mini.jpg" width="24" /></a>  </span>
+
+
+  <span class="vcard">
+    <a href="/emaland" class="url" hreflang="en" rel="contact" title="emaland"><img alt="emaland" class="photo fn" height="24" src="http://a3.twimg.com/profile_images/68049269/n707915915_121_mini.jpg" width="24" /></a>  </span>
+
+
+  <span class="vcard">
+    <a href="/ablegrape" class="url" hreflang="en" rel="contact" title="Doug Cook"><img alt="Doug Cook" class="photo fn" height="24" src="http://a1.twimg.com/profile_images/76892852/sticker2_mini.jpg" width="24" /></a>  </span>
+
+
+  </div>
+    <div id="friends_view_all">
+    <a href="/twitter/following" rel="me">View all&hellip;</a>
+  </div>
+  
+</div>
+
+    
+  </div>
+
+
+
+
+  <div id="rssfeed">
+    <hr/>
+    <a href="/statuses/user_timeline/783214.rss" class="xref rss profile-rss" rel="alternate" type="application/rss+xml">RSS feed of twitter's tweets</a>
+     <a href="/favorites/783214.rss" class="xref rss favorites-rss" rel="alternate" type="application/rss+xml">RSS feed of twitter's favorites</a>
+  </div>
+
+
+                  </div>
+                                </td>
+              
+            </tr>
+          </tbody>
+        </table>
+      
+
+      
+  <div id="footer"
+        class="round">
+      <h3 class="offscreen">Footer</h3>
+      
+      
+      <ul>
+          <li class="first">&copy; 2009 Twitter</li>
+          <li><a href="/about#about">About Us</a></li>
+          <li><a href="/about#contact">Contact</a></li>
+          <li><a href="http://blog.twitter.com">Blog</a></li>
+          <li><a href="http://status.twitter.com">Status</a></li>
+                      <li><a href="/goodies">Goodies</a></li>
+                    <li><a href="http://apiwiki.twitter.com/">API</a></li>
+                      <li><a href="http://business.twitter.com/twitter101">Business</a></li>
+                    <li><a href="http://help.twitter.com">Help</a></li>
+          <li><a href="/jobs">Jobs</a></li>
+          <li><a href="/tos">Terms</a></li>
+          <li><a href="/privacy">Privacy</a></li>
+      </ul>
+  </div>
+
+
+
+      <hr />
+
+    </div>
+
+    
+    
+    <script src="http://ajax.googleapis.com/ajax/libs/jquery/1.3.0/jquery.min.js" type="text/javascript"></script>
+<script src="http://a0.twimg.com/a/1259091217/javascripts/twitter.js?1259117012" type="text/javascript"></script>
+<script src="http://a2.twimg.com/a/1259091217/javascripts/lib/jquery.tipsy.min.js?1259117012" type="text/javascript"></script>
+<script type="text/javascript">
+//<![CDATA[
+  page.user_screenname = 'twitter';
+  page.user_fullname = 'Twitter';
+      page.controller_name = 'AccountController';
+      page.action_name = 'profile';
+      twttr.form_authenticity_token = '7e394453cc9d849cab133ccb3180e8ae4e6a0258';
+      // FIXME: Reconcile with the kinds on the Status model.
+      twttr.statusKinds = {
+        UPDATE: 1,
+        SHARE: 2
+      };
+      twttr.ListPerUserLimit = 20;
+      
+    
+//]]>
+</script>
+<script type="text/javascript">
+//<![CDATA[
+
+      $( function () {
+          initializePage();
+
+      });
+    
+//]]>
+</script>
+
+        <!-- BEGIN google analytics -->
+
+  <script type="text/javascript">
+    var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
+    document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
+  </script>
+
+  <script type="text/javascript">
+
+    try {
+      var pageTracker = _gat._getTracker("UA-30775-6");
+      pageTracker._setDomainName("twitter.com");
+            pageTracker._setVar('Not Logged In');
+            pageTracker._setVar('lang: en');
+            pageTracker._initData();
+                              pageTracker._trackPageview('/profile/not_logged_in/twitter');
+                  } catch(err) { }
+
+  </script>
+
+  <!-- END google analytics -->
+
+
+
+
+
+    <div id="notifications"></div>
+
+
+    
+    <!-- ERB -->
+  </body>
+
+</html>

data/spec/object-scraper/scraper_spec.rb

@@ -0,0 +1,62 @@
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'spec_helper'))
+
+describe Scraper do
+  before :all do
+    @uri     = File.expand_path(File.join(File.dirname(__FILE__), '..', 'data', 'twitter.html' ))
+    @pattern = ".status"
+    class Entry < Object
+      attr_accessor :text, :date
+    end
+  end
+  
+  before :each do
+    Scraper.scrapers = {}
+  end
+  
+  describe "defining a scraper" do    
+    it "should create a new scraper using the specified name" do
+      Scraper.define(:s, :class => :entry, :source => @uri, :node => @pattern) {}
+
+      Scraper(:s).scraper_source.should == @uri
+    end
+
+    it "should be set with and without block" do
+      Scraper.define(:s, :class => :entry, :source => @uri, :node => @pattern) do |s|
+        s.text "foo"
+        s.date {"bar"}
+      end
+
+      @objects = Scraper.parse(:s)
+      @objects.first.text.should == "foo"
+      @objects.first.date.should == "bar"
+    end
+    
+    it "should get the objects from twitter" do
+      Scraper.define(:twitter, :class => :entry, :source => @uri, :node => @pattern) do |s|
+        s.text { |node| node.at(".entry-content").inner_html }
+        s.date { |node| DateTime.parse(node.at(".timestamp")[:data][/\'.*\'/].delete("'")) }
+      end
+      
+      @objects = Scraper.parse(:twitter)
+      @objects.size.should == 20
+      @objects.first.text.should == "SMS delivery issues on AT&T <a href=\"http://bit.ly/7JFJ6H\" class=\"tweet-url web\" rel=\"nofollow\" target=\"_blank\">http://bit.ly/7JFJ6H</a>"
+      @objects.first.date.should == DateTime.parse("Mon Nov 30 04:10:51 +0000 2009")
+    end
+    
+    it "should use a different html parser" do
+      require 'nokogiri'
+      Scraper.scrape_source_with = Proc.new { |source| Nokogiri::HTML(source) }
+
+      Scraper.define(:twitter, :class => :entry, :source => @uri, :node => @pattern) do |s|
+        s.text { |node| node.at(".entry-content").inner_html }
+        s.date { |node| DateTime.parse(node.at(".timestamp")[:data][/\'.*\'/].delete("'")) }
+      end
+      
+      @objects = Scraper.parse(:twitter)
+      @objects.size.should == 20
+      @objects.first.text.should == "SMS delivery issues on AT&amp;T <a href=\"http://bit.ly/7JFJ6H\" class=\"tweet-url web\" rel=\"nofollow\" target=\"_blank\">http://bit.ly/7JFJ6H</a>"
+      @objects.first.date.should == DateTime.parse("Mon Nov 30 04:10:51 +0000 2009")
+    end
+    
+  end
+end

data/spec/spec.opts

@@ -0,0 +1,2 @@
+--format progress
+--color

data/spec/spec_helper.rb

@@ -0,0 +1,7 @@
+$: << File.join(File.dirname(__FILE__), '..', 'lib')
+$: << File.join(File.dirname(__FILE__))
+
+require 'rubygems'
+require 'spec'
+require 'spec/autorun'
+require 'object-scraper'

metadata

@@ -0,0 +1,80 @@
+--- !ruby/object:Gem::Specification 
+name: object-scraper
+version: !ruby/object:Gem::Version 
+  version: 0.0.2
+platform: ruby
+authors: 
+- Enrico Genauck
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-12-10 00:00:00 +01:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: hpricot
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.8.2
+    version: 
+description: Object scraper is a thin wrapper for hpricot to enable recipe-like extraction of ruby objects from various web sites.
+email: kontakt@enricogenauck.de
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+- lib/object-scraper.rb
+- lib/object-scraper/scraper.rb
+files: 
+- Manifest
+- README.rdoc
+- Rakefile
+- lib/object-scraper.rb
+- lib/object-scraper/scraper.rb
+- object-scraper.gemspec
+- spec/data/twitter.html
+- spec/object-scraper/scraper_spec.rb
+- spec/spec.opts
+- spec/spec_helper.rb
+has_rdoc: true
+homepage: http://github.com/enricogenauck/object-scraper
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --line-numbers
+- --inline-source
+- --title
+- Object-scraper
+- --main
+- README.rdoc
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "1.2"
+  version: 
+requirements: []
+
+rubyforge_project: object-scraper
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: Recipe like object extraction from HTML sources
+test_files: []
+