libcraigscrape 0.6 → 0.6.5

data/CHANGELOG

@@ -1,5 +1,13 @@
 == Change Log
 
+=== Release 0.6.5 (Jun 8, 2009)
+- Added PostFull::deleted_by_author? , added test case for said condition
+- Fixed a bug that caused the library to die in weird ways if there wasn't a title tag on a parsed page
+- Apparently Craigslist starting gzip-encoding *some* listings. Added gzip decoding support
+- Found a bug when parsing the location field on some full_posts in the apa sections
+- Added support for file:// uri's int he scrape_* functions, and revised the tests to use these uri's
+- Fixed a bug that caused errors to be raised with legitimately empty listing pages
+
 === Release 0.6.0 (May 21, 2009)
 - Added PostFull::flagged_for_removal?
 - Fixed a couple small parse bugs found in production

data/Rakefile

@@ -11,7 +11,7 @@ include FileUtils
 RbConfig = Config unless defined? RbConfig
 
 NAME = "libcraigscrape"
-VERS = ENV['VERSION'] || "0.6"
+VERS = ENV['VERSION'] || "0.6.5"
 PKG = "#{NAME}-#{VERS}"
 
 RDOC_OPTS = ['--quiet', '--title', 'The libcraigscrape Reference', '--main', 'README', '--inline-source']
@@ -57,8 +57,11 @@ Rake::RDocTask.new do |rdoc|
 end
 
 Rake::GemPackageTask.new(SPEC) do |p|
-    p.need_tar = true
-    p.gem_spec = SPEC
+  p.need_tar = false
+  p.need_tar_gz = true
+  p.need_tar_bz2 = true
+  p.need_zip = true
+  p.gem_spec = SPEC
 end
 
 task "lib" do
@@ -74,35 +77,3 @@ task :uninstall => [:clean] do
   sh %{sudo gem uninstall #{NAME}}
 end
 
-task :pkg_archives do
-  base_dir = File.dirname __FILE__
-  package_name = '%s-%s' % [NAME,VERS]
-  packages_base = "#{base_dir}/pkg"
-  packaging_dir = '%s/%s' % [ packages_base,package_name ] 
-  
-  begin
-    # First we create a proper package-X.X directory:
-    PKG_FILES.each do |p_f|
-      base_file = '%s/%s' % [base_dir, p_f]
-      packaged_file = '%s/%s' % [packaging_dir, p_f]
-      packaged_file_dirname = File.dirname packaged_file
-  
-      # We really don't care to do anything about these - we'll recreate it when/if its needed
-      next if File.directory? base_file
-  
-      FileUtils.mkdir_p packaged_file_dirname unless File.directory? packaged_file_dirname
-      
-      FileUtils.cp base_file, packaged_file unless File.exists? packaged_file
-    end
-    
-    # Remove any old archives we'd be replacing:
-    %w(zip tar.bz2).each{ |ext| FileUtils.rm "#{packaging_dir}.#{ext}" if File.exist? "#{packaging_dir}.#{ext}" }
-    
-    # Now let's create some archives:
-    sh %{cd #{packages_base} && tar -cjvf  #{package_name}.tar.bz2 #{package_name}}
-    sh %{cd #{packages_base} && zip -r #{package_name}.zip #{package_name}}
-  ensure
-    # Delete that temp directory we created at the start here
-    FileUtils.rmtree packaging_dir
-  end
-end

data/lib/libcraigscrape.rb

@@ -2,10 +2,11 @@
 #
 # All of libcraigscrape's objects and methods are loaded when you use <tt>require 'libcraigscrape'</tt> in your code.
 #
+require 'net/http'
+require 'zlib'
 
 require 'rubygems'
 require 'hpricot'
-require 'net/http'
 require 'htmlentities'
 require 'activesupport'
 
@@ -31,10 +32,14 @@ class CraigScrape
   end
   
   module ParseObjectHelper  #:nodoc:
+    private
     def he_decode(text)
       HTMLEntities.new.decode text
     end
   end
+
+  class BadUrlError < StandardError #:nodoc:
+  end
   
   class ParseError < StandardError #:nodoc: 
   end
@@ -92,7 +97,7 @@ class CraigScrape
       
       title = page.at('title')
       @title = he_decode title.inner_html if title
-      @title = nil if @title.length ==0
+      @title = nil if @title and @title.length ==0
       
       @full_section = []
       (page/"div[@class='bchead']//a").each do |a|
@@ -125,18 +130,56 @@ class CraigScrape
       # This will make it easier for the next guy to work with if wants to parse out the information we're disgarding...
       parse_craig_body Hpricot.parse(craigbody_as_s) if craigbody_as_s
       
+      # We'll first set these edge cases to false, unless the block below decides otherwise
+      @flagged_for_removal = false
+      @deleted_by_author = false
+      
+      # Time to check for errors and edge cases
+      if [@contents,@posting_id,@post_time,@title].all?{|f| f.nil?}
+        case @header.gsub(HTML_TAG, "")
+          when "This posting has been flagged for removal"
+            @flagged_for_removal = true
+          when "This posting has been deleted by its author."
+            @deleted_by_author = true
+        end
+      end
+      
       # Validate that required fields are present:
-      raise ParseError, "Unable to parse PostFull: %s" % page.to_html if !flagged_for_removal? and [
+      raise ParseError, "Unable to parse PostFull: %s" % page.to_html if !flagged_for_removal? and !deleted_by_author? and [
         @contents,@posting_id,@post_time,@header,@title,@full_section
       ].any?{|f| f.nil? or (f.respond_to? :length and f.length == 0)}
     end
+        
+    # Returns true if this Post was parsed, and merely a 'Flagged for Removal' page
+    def flagged_for_removal?; @flagged_for_removal; end
+
+    # Returns true if this Post was parsed, and represents a 'This posting has been deleted by its author.' notice
+    def deleted_by_author?; @deleted_by_author; end
+    
+    # Returns the price (as float) of the item, as best ascertained by the post header
+    def price
+      $1.to_f if @title and @header and PRICE.match(@header.gsub(/#{@title}/, ''))
+    end
+    
+    # Returns the post contents with all html tags removed
+    def contents_as_plain
+      @contents.gsub HTML_TAG, "" if @contents
+    end
+    
+    private
     
     # I left this here as a stub, since someone may want to parse more then what I'm currently scraping from this part of the page
     def parse_craig_body(craigbody_els)  #:nodoc:
       # Location (when explicitly defined):
-      cursor = craigbody_els.at 'ul'
-      cursor = cursor.at 'li' if cursor
-      @location = he_decode $1 if cursor and LOCATION.match cursor.inner_html
+      cursor = craigbody_els.at 'ul' unless @location
+      
+      # Apa section includes other things in the li's (cats/dogs ok fields)
+      cursor.children.each do |li|
+        if LOCATION.match li.inner_html
+          @location = he_decode($1) and break
+          break
+        end
+      end if cursor
 
       # Real estate listings can work a little different for location:
       unless @location
@@ -151,24 +194,6 @@ class CraigScrape
       
       @images = (img_table / 'img').collect{|i| i[:src]} if img_table
     end
-    
-    # Returns true if this Post was parsed, and merely a 'Flagged for Removal' page
-    def flagged_for_removal?
-      (
-        [@contents,@posting_id,@post_time,@title].all?{|f| f.nil?} and 
-        @header.gsub(HTML_TAG, "") == "This posting has been flagged for removal"
-      )
-    end
-    
-    # Returns the price (as float) of the item, as best ascertained by the post header
-    def price
-      $1.to_f if @title and @header and PRICE.match(@header.gsub(/#{@title}/, ''))
-    end
-    
-    # Returns the post contents with all html tags removed
-    def contents_as_plain
-      @contents.gsub HTML_TAG, "" if @contents
-    end
   end
 
   # Listings represents a parsed Craigslist listing page and is generally returned by CraigScrape.scrape_listing
@@ -207,7 +232,7 @@ class CraigScrape
       @next_page_href = next_link[:href] if next_link
       
       # Validate that required fields are present:
-      raise ParseError, "Unable to parse Listings: %s" % page.to_html unless @posts.length > 0
+      raise ParseError, "Unable to parse Listings: %s" % page.to_html if tags_worth_parsing.length > 0 and  @posts.length == 0
     end
     
   end
@@ -304,7 +329,7 @@ class CraigScrape
     
     # Requests and returns the PostFull object that corresponds with this summary's full_url
     def full_post
-      @full_post = CraigScrape.scrape_full_post(full_url) if @full_post.nil? and full_url
+      @full_post ||= CraigScrape.scrape_full_post full_url if full_url
       
       @full_post
     end
@@ -366,42 +391,51 @@ class CraigScrape
   def self.scrape_posts_since(listing_url, newer_then)
     self.scrape_until(listing_url) {|post| post.date <= newer_then}
   end
-  
+
   def self.fetch_url(uri) #:nodoc:
-    fetch_attempts = 0
-    
-    begin
-      # This handles the redirects for us
-      uri_dest = ( uri.class == String ) ? URI.parse(uri) : uri 
-  
-      logger.info "Requesting: %s" % uri_dest.to_s if logger
-      
-      resp, data = Net::HTTP.new( uri_dest.host, uri_dest.port).get uri_dest.request_uri, nil
-  
-      if resp.response.code == "200"
-        data
-      elsif resp.response['Location']
-        redirect_to = resp.response['Location']
-        self.fetch_url(redirect_to)
-      else
-        # Sometimes Craigslist seems to return 404's for no good reason, and a subsequent fetch will give you what you want
-        error_description = 'Unable to fetch "%s" (%s)' % [ uri_dest.to_s, resp.response.code ]
+    uri_dest = ( uri.class == String ) ? URI.parse(uri) : uri 
+
+    logger.info "Requesting: %s" % uri_dest.to_s if logger
 
-        logger.info error_description if logger
+    case uri_dest.scheme
+      when 'file'
+        File.read uri_dest.path
+      when /^http[s]?/
+        fetch_attempts = 0
         
-        raise FetchError, error_description
-      end
-    rescue FetchError => err
-      fetch_attempts += 1
+        begin
+          # This handles the redirects for us          
+          resp, data = Net::HTTP.new( uri_dest.host, uri_dest.port).get uri_dest.request_uri, nil
       
-      if retries_on_fetch_fail <= CraigScrape.retries_on_fetch_fail
-        sleep CraigScrape.sleep_between_fetch_retries if CraigScrape.sleep_between_fetch_retries
-        retry
+          if resp.response.code == "200"
+            # Check for gzip, and decode:
+            data = Zlib::GzipReader.new(StringIO.new(data)).read if resp.response.header['Content-Encoding'] == 'gzip'
+            
+            data
+          elsif resp.response['Location']
+            redirect_to = resp.response['Location']
+            self.fetch_url(redirect_to)
+          else
+            # Sometimes Craigslist seems to return 404's for no good reason, and a subsequent fetch will give you what you want
+            error_description = 'Unable to fetch "%s" (%s)' % [ uri_dest.to_s, resp.response.code ]
+    
+            logger.info error_description if logger
+            
+            raise FetchError, error_description
+          end
+        rescue FetchError => err
+          fetch_attempts += 1
+          
+          if retries_on_fetch_fail <= CraigScrape.retries_on_fetch_fail
+            sleep CraigScrape.sleep_between_fetch_retries if CraigScrape.sleep_between_fetch_retries
+            retry
+          else
+            raise err
+          end
+        end
       else
-        raise err
-      end
+        raise BadUrlError, "Unknown URI scheme for the url: #{uri_dest.to_s}"
     end
-
   end
   
   def self.uri_from_href(base_uri, href) #:nodoc:

data/test/listing_samples/empty_listings.html

@@ -0,0 +1,128 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+<html><head>
+<title>treasure coast arts/crafts for sale classifieds - craigslist</title>
+
+<meta name="description" content="craigslist arts/crafts for sale classifieds for treasure coast ">
+<meta name="keywords" content="treasure coast arts/crafts for sale craigslist, classifieds, want ads ">
+
+
+
+<link rel=alternate type="application/rss+xml" href="index.rss" title="RSS feed for craigslist | arts/crafts for sale in treasure coast ">
+<link rel="stylesheet" title="craigslist" href="http://www.craigslist.org/styles/craigslist.css" type="text/css" media="all">
+</head>
+
+<body class="toc">
+
+<a name="top"></a>
+
+<div class="bchead"><span id="ef">
+			
+			[ <a href="http://www.craigslist.org/about/help/">help</a> ]
+			[ <a href="https://post.craigslist.org/psl/S">post</a> ]</span>
+			
+				<a href="/"> treasure coast craigslist</a>  &gt; <a href="/art/">arts/crafts for sale</a></div>
+
+<blockquote>
+<form action="/search/art" method="get" onsubmit="ckCAbb();">
+
+	<script type="text/javascript"><!--
+		function ckCAbb() {
+			t = document.getElementById("cAbb");
+			if (t.value == "art") { t.disabled = true; }
+		}
+	-->
+	</script>
+
+	<table width="95%" cellpadding="2" style="white-space: nowrap; background:#eee; border:1px solid gray;" summary="">
+		<tr>
+			<td align="right" width="1">search for:</td>
+			<td width="30%"><input id="query" name="query" size="30" value=""> in:
+				<select id="cAbb" name="catAbbreviation">
+<option value="ccc">all community<option value="eee">all event<option value="sss">all for sale / wanted<option disabled value="">--<option value="art" selected>   art &amp; crafts
+<option value="pts">   auto parts
+<option value="bab">   baby &amp; kid stuff
+<option value="bar">   barter
+<option value="bik">   bicycles
+<option value="boa">   boats
+<option value="bks">   books
+<option value="bfs">   business
+<option value="cta">   cars &amp; trucks - all
+<option value="ctd">   cars &amp; trucks - by dealer
+<option value="cto">   cars &amp; trucks - by owner
+<option value="emd">   cds / dvds / vhs
+<option value="clo">   clothing
+<option value="clt">   collectibles
+<option value="sys">   computers &amp; tech
+<option value="ele">   electronics
+<option value="grd">   farm &amp; garden
+<option value="zip">   free stuff
+<option value="fua">   furniture - all
+<option value="fud">   furniture - by dealer
+<option value="fuo">   furniture - by owner
+<option value="tag">   games &amp; toys
+<option value="gms">   garage sales
+<option value="for">   general
+<option value="hsh">   household
+<option value="wan">   items wanted
+<option value="jwl">   jewelry
+<option value="mat">   materials
+<option value="mcy">   motorcycles/scooters
+<option value="msg">   musical instruments
+<option value="pho">   photo/video
+<option value="rvs">   recreational vehicles
+<option value="spo">   sporting goods
+<option value="tix">   tickets
+<option value="tls">   tools
+<option disabled value="">--<option value="ggg">all gigs<option value="hhh">all housing<option value="jjj">all jobs<option value="ppp">all personals<option value="res">all resume<option value="bbb">all services offered</select>
+				<input type="submit" value="Search">
+			</td><td>
+				<label><input type="checkbox" name="srchType" value="T"
+					title="check this box to search only posting titles"> only search titles</label>
+			</td>
+		</tr>
+	
+		<tr>
+			<td align="right" width="1">price:</td>
+			<td><input name="minAsk" size="6" value="min" onfocus="value=''">&nbsp;<input name="maxAsk" size="6" value="max" onfocus="value=''">&nbsp;</td>
+			<td align="left"><label><input type="checkbox" name="hasPic" value="1"> has image</label></td>
+		</tr></table></form></blockquote><span id="showPics"></span><span id="hidePics"></span>
+
+<blockquote>
+<table width="95%" summary="">
+        <tr>
+        	<td valign="top">[ Mon, 08 Jun 17:37:29 ]</td>
+			<td valign="top" id="messages"><span class="hl"> [ <a href="http://www.recalls.gov/">avoid recalled items</a> ] </span> <span class="hl"> [ <a href="/about/prohibited.items">partial list of prohibited items</a> ] </span> <span class="hl"> [<a href="/cgi-bin/success.stories.cgi">success story?</a>]</span> <span class="hl"> [ <b><a href="/about/scams">AVOIDING SCAMS &amp; FRAUD</a></b> ] </span> <span class="hl"> [ <b><a href="/about/safety">PERSONAL SAFETY TIPS</a></b> ] </span> </td>
+        </tr>
+</table>
+
+
+
+
+
+<div id="footer">
+	<hr>
+	<span id="copy">
+		Copyright &copy; 2009 craigslist, inc.<br>
+		<a href="#top">Back to top of page</a>
+	</span>
+	<span class="rss">
+		<a class="l" href="http://treasure.craigslist.org/art/index.rss">RSS</a>
+		<a href="http://www.craigslist.org/about/rss">(?)</a><br>
+		<a class="y" href="http://add.my.yahoo.com/rss?url=http://treasure.craigslist.org/art/index.rss">add to My Yahoo!</a>  
+	</span>
+</div>
+<br><br>
+
+<div id="floater">&nbsp;</div>
+
+</blockquote>
+<script type="text/javascript" src="http://www.craigslist.org/js/jquery.js"></script><script type="text/javascript" src="http://www.craigslist.org/js/tocs.js"></script>
+<script type="text/javascript">
+<!--
+	initImgs();
+-->
+</script>
+
+
+</body>
+</html>

data/test/post_samples/1207457727.html

@@ -0,0 +1,92 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
+<html>
+<head>
+	<title>2bth for no deposit req</title>
+	<meta name="robots" content="NOARCHIVE,NOFOLLOW">
+	<link rel="stylesheet" title="craigslist" href="http://www.craigslist.org/styles/craigslist.css" type="text/css" media="all">
+</head>
+
+ <body onload="initFlag(1207457727)" class="posting">
+
+<div class="bchead">
+<a id="ef" href="/email.friend?postingID=1207457727">email this posting to a friend</a>
+<a href="http://miami.craigslist.org">south florida craigslist</a>
+        &gt; <a href="/brw/">broward county</a> &gt;  <a href="/brw/apa/">apts/housing for rent</a>
+</div>
+
+	<div id="flags">
+		<div id="flagMsg">
+			please <a href="http://www.craigslist.org/about/help/flags_and_community_moderation">flag</a> with care:
+		</div>
+		<div id="flagChooser">
+			<br>
+			<a class="fl" id="flag16" href="/flag/?flagCode=16&amp;postingID=1207457727"
+				title="Wrong category, wrong site, discusses another post, or otherwise misplaced">
+				miscategorized</a>
+			<br>
+
+			<a class="fl" id="flag28" href="/flag/?flagCode=28&amp;postingID=1207457727"
+				title="Violates craigslist Terms Of Use or other posted guidelines">
+				prohibited</a>
+			<br>
+
+			<a class="fl" id="flag15" href="/flag/?flagCode=15&amp;postingID=1207457727"
+				title="Posted too frequently, in multiple cities/categories, or is too commercial">
+				spam/overpost</a>
+			<br>
+
+			<a class="fl" id="flag9" href="/flag/?flagCode=9&amp;postingID=1207457727"
+				title="Should be considered for inclusion in the Best-Of-Craigslist">
+				best of craigslist</a>
+			<br>
+		</div>
+	</div>
+
+<div id="tsb">
+		<a href="http://www.craigslist.org/about/FHA.html">Stating a discriminatory preference in a housing post is illegal - please flag discriminatory posts as prohibited</a></div> <div id="tsb"> <em>Avoid scams and fraud by dealing locally!</em> Beware any arrangement involving Western Union, Moneygram, wire transfer, or a landlord/owner who is out of the country or cannot meet you in person. <a href="http://www.craigslist.org/about/scams.html">More info</a></div>
+
+
+<h2>$1350 / 3br - 2bth for no deposit req (Coral Springs)</h2>
+<hr>
+Reply to: <a href="mailto:hous-ccpap-1207457727@craigslist.org?subject=%241350%20%2F%203br%20-%202bth%20for%20no%20deposit%20req%20(Coral%20Springs)">hous-ccpap-1207457727@craigslist.org</a> <sup>[<a href="http://www.craigslist.org/about/help/replying_to_posts" target="_blank">Errors when replying to ads?</a>]</sup><br>
+Date: 2009-06-05,  6:56PM EDT<br>
+<br>
+<br>
+<div id="userbody">
+<p><br>Call!! asking for a new owner.<br>  no deposit required rent to own properties. <br> <br> Defaulting payment records are not a problem, <br> we will help you protect the previous owners credit history! 2&#48;&#50;-56&#55;-637&#49;  <br><br></p>
+
+
+<br><br><ul>
+<li>cats are OK - purrr
+<li>dogs are OK - wooof
+<li> Location: Coral Springs
+<li>it's NOT ok to contact this poster with services or other commercial interests</ul>
+
+		<table summary="craigslist hosted images">
+			<tr>
+				<td align="center"><img src="http://images.craigslist.org/3k43pe3o8ZZZZZZZZZ9655022102a3ea51624.jpg" alt="image 1207457727-0"></td>
+				<td align="center"><img src="http://images.craigslist.org/3n13m53p6ZZZZZZZZZ96596515e51237a179c.jpg" alt="image 1207457727-1"></td>
+			</tr>
+			<tr>
+				<td align="center"><img src="http://images.craigslist.org/3od3p33leZZZZZZZZZ9656d614da8e3a51dd9.jpg" alt="image 1207457727-2"></td>
+				<td align="center"><img src="http://images.craigslist.org/3pb3oa3leZZZZZZZZZ965eb60e4d2344019fb.jpg" alt="image 1207457727-3"></td>
+			</tr>
+		</table>
+
+</div>
+PostingID: 1207457727<br>
+
+<br>
+
+<hr>
+<ul class="clfooter">
+	<li>Copyright &copy; 2009 craigslist, inc.</li>
+	<li><a href="http://www.craigslist.org/about/terms.of.use.html">terms of use</a></li>
+	<li><a href="http://www.craigslist.org/about/privacy_policy">privacy policy</a></li>
+	<li><a href="/forums/?forumID=8">feedback forum</a></li>
+</ul>
+<script type="text/javascript" src="http://www.craigslist.org/js/jquery.js"></script>
+<script type="text/javascript" src="http://www.craigslist.org/js/postings.js"></script>
+</body>
+</html>
+

data/test/post_samples/this_post_has_been_deleted_by_its_author.html

@@ -0,0 +1,37 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
+<html>
+<head>
+        <title></title>
+        <meta name="robots" content="NOARCHIVE,NOFOLLOW" />
+        <link href="http://www.craigslist.org/styles/craigslist.css" title="craigslist" rel="stylesheet" media="all" type="text/css" />
+</head>
+
+ <body class="posting" onload="initFlag(1187861811)">
+
+<div class="bchead">
+
+<a href="http://miami.craigslist.org">south florida craigslist</a>
+        &gt; <a href="/brw/">broward county</a> &gt;  <a href="/brw/cto/">cars &amp; trucks - by owner</a>
+</div>
+
+
+
+<hr />
+<br />
+<br />
+<h2>This posting has been deleted by its author.</h2>
+<h5>(The title on the listings page will be removed in just a few minutes.)</h5>
+
+<br /><br />
+
+<hr />
+<ul class="clfooter">
+        <li>Copyright &copy; 2009 craigslist, inc.</li>
+        <li><a href="http://www.craigslist.org/about/terms.of.use.html">terms of use</a></li>
+        <li><a href="http://www.craigslist.org/about/privacy_policy">privacy policy</a></li>
+        <li><a href="/forums/?forumID=8">feedback forum</a></li>
+</ul>
+<script src="http://www.craigslist.org/js/jquery.js" type="text/javascript"></script>
+<script src="http://www.craigslist.org/js/postings.js" type="text/javascript"></script>
+</body>
+</html>

data/test/test_craigslist_listing.rb

@@ -119,8 +119,8 @@ EOD
     assert_equal 25.0, six.price
   end
 
-  def test_listings_parse    
-    category = CraigScrape::Listings.new read_as_hpricot('listing_samples/category_output.html')
+  def test_listings_parse 
+    category = CraigScrape.scrape_listing relative_uri_for('listing_samples/category_output.html')
     assert_equal 'index100.html', category.next_page_href
     assert_equal 100, category.posts.length
     category.posts[0..80].each do |l|
@@ -128,19 +128,19 @@ EOD
       assert_equal 18, l.date.day
     end
     
-    category2 = CraigScrape::Listings.new read_as_hpricot('listing_samples/category_output_2.html')
+    category2 = CraigScrape.scrape_listing relative_uri_for('listing_samples/category_output_2.html')
     assert_equal 'index900.html', category2.next_page_href
     assert_equal 100, category2.posts.length
     
-    long_search = CraigScrape::Listings.new read_as_hpricot('listing_samples/long_search_output.html')
+    long_search = CraigScrape.scrape_listing relative_uri_for('listing_samples/long_search_output.html')
     assert_equal '/search/rea?query=house&minAsk=min&maxAsk=max&bedrooms=&s=800', long_search.next_page_href
     assert_equal 100, long_search.posts.length
     
-    short_search = CraigScrape::Listings.new read_as_hpricot('listing_samples/short_search_output.html')
+    short_search = CraigScrape.scrape_listing relative_uri_for('listing_samples/short_search_output.html')
     assert_equal nil, short_search.next_page_href
     assert_equal 93, short_search.posts.length
     
-    mia_fua_index8900_052109 = CraigScrape::Listings.new read_as_hpricot('listing_samples/mia_fua_index8900.5.21.09.html')
+    mia_fua_index8900_052109 = CraigScrape.scrape_listing relative_uri_for('listing_samples/mia_fua_index8900.5.21.09.html')
     assert_equal 'index9000.html', mia_fua_index8900_052109.next_page_href
     assert_equal 100, mia_fua_index8900_052109.posts.length
     mia_fua_index8900_052109.posts[0..13].each do |l|
@@ -151,10 +151,14 @@ EOD
       assert_equal 5, l.date.month
       assert_equal 14, l.date.day
     end
+    
+    empty_listings = CraigScrape.scrape_listing relative_uri_for('listing_samples/empty_listings.html')
+    assert_equal nil, empty_listings.next_page_href
+    assert_equal [], empty_listings.posts
   end
   
   def test_posting_parse
-    posting0 = CraigScrape::PostFull.new read_as_hpricot('post_samples/posting0.html')
+    posting0 = CraigScrape.scrape_full_post relative_uri_for('post_samples/posting0.html')
     assert_equal "Has storage for videos/dvds. About 2 ft high by 21/2 ft widw. Almond/light beige color", posting0.contents
     assert_equal ["south florida craigslist", "miami / dade", "furniture - by owner"], posting0.full_section
     assert_equal "tv cart on wheels - $35 (NMB)", posting0.header
@@ -167,7 +171,7 @@ EOD
     assert_equal "Has storage for videos/dvds. About 2 ft high by 21/2 ft widw. Almond/light beige color",posting0.contents_as_plain
     assert_equal 35.0, posting0.price
     
-    posting1 = CraigScrape::PostFull.new read_as_hpricot('post_samples/posting1.html')
+    posting1 = CraigScrape.scrape_full_post relative_uri_for('post_samples/posting1.html')
     assert_equal "Residential income property\227Investors this property is for you! This duplex has a 2bedroom/1bath unit on each side. It features updated kitchens and baths (new tubs, toilet, sink, vanities), ceramic tile flooring throughout, separate water and electric meters and on site laundry facilities. It is also closed to the Galleria, beaches and downtown Fort Lauderdale! \r<br />\n\r<br />\nJe parle le Fran\347ais\r<br />\n\r<br />\nThis property is being offered by Blaunch Perrier, Broker Associate, Atlantic Properties International. Blaunch can be reached at 954-593-0077. For additional property information you may also visit www.garylanham.com\r<br />\n\r<br />", posting1.contents
     assert_equal ["south florida craigslist", "broward county", "real estate - by broker"], posting1.full_section
     assert_equal "$189900 / 4br - Investment Property--Duplex in Fort Lauderdale", posting1.header
@@ -180,7 +184,7 @@ EOD
     assert_equal "Residential income property\227Investors this property is for you! This duplex has a 2bedroom/1bath unit on each side. It features updated kitchens and baths (new tubs, toilet, sink, vanities), ceramic tile flooring throughout, separate water and electric meters and on site laundry facilities. It is also closed to the Galleria, beaches and downtown Fort Lauderdale! \r\n\r\nJe parle le Fran\347ais\r\n\r\nThis property is being offered by Blaunch Perrier, Broker Associate, Atlantic Properties International. Blaunch can be reached at 954-593-0077. For additional property information you may also visit www.garylanham.com\r\n\r", posting1.contents_as_plain
     assert_equal 189900.0, posting1.price
 
-    posting2 = CraigScrape::PostFull.new read_as_hpricot('post_samples/posting2.html')
+    posting2 = CraigScrape.scrape_full_post relative_uri_for('post_samples/posting2.html')
     assert_equal 15775, posting2.contents.length # This is easy, and probably fine enough
     assert_equal ["south florida craigslist", "broward county", "cars & trucks - by dealer"], posting2.full_section
     assert_equal "PRESENTING A ELECTRON BLUE METALLIC 2002 CHEVROLET CORVETTE Z06 6 SPEE - $23975 (Fort Lauderdale)", posting2.header
@@ -193,7 +197,7 @@ EOD
     assert_equal "\302\240 Sheehan Buick Pontiac GMC \302\240 Pompano Beach, FL(754) 224-3257 \302\240PRESENTING A ELECTRON BLUE METALLIC 2002 CHEVROLET CORVETTE Z06 6 SPEED FLORIDA DRIVEN SMOKIN' SPORTS CAR!2002 Chevrolet Corvette Z06 Florida Driven AutoCheck Certified 5.7L V8 6sp2 Door Coupe.\302\240Price: \302\240 $23,975Exterior:Electron Blue MetallicInterior:BlackStock#:P5110AVIN:1G1YY12S625129021FREE AutoCheck Vehicle ReportMileage:63,560Transmission:6 Speed ManualEngine:V8 5.7L OHVWarranty:Limited WarrantyTitle:Clear\302\273\302\240View All 58 Photos\302\273\302\240View Full Vehicle Details\302\273\302\240Ask the Seller a Question\302\273\302\240E-mail this to a Friend\302\240 DescriptionPRESENTING A ELECTRON BLUE METALLIC 2002 CHEVROLET CORVETTE Z06 6 SPEED FLORIDA DRIVEN SMOKIN' SPORTS CAR!\r\n\r\nLOADED WITH BLACK LEATHER BUCKET SEATS, POWER DRIVERS SEAT, DUAL ZONE CLIMATE CONTROL, 4 WHEEL ABS BRAKES, POWER STEERING AND BRAKES, REAR LIMITED SLIP DIFFERENTIAL, STABILITY CONTROL, CRUISE CONTROL, TLT STEERING WHEEL, POWER WINDOWS AND LOCKS, AUTOMATIC ON/OFF HEADLAMPS, FOG LIGHTS, DUAL AIR BAG SAFETY, AM/FM STEREO CD PLAYER, INTERMITTENT WINDSHIELD WIPERS AND SO MUCH MORE - THIS CAR IS TOTALLY HOT WITH GREAT LOW MILES!\r\n\r\nPlease call us to make your deal now at 1-888-453-5244. Please visit our Website at www.sheehanautoplex.com ***View 50+ Pictures of this vehicle - a complete description including standard features and all added options & a FREE AUTO CHECK REPORT at www.sheehanautoplex.com. ***Financing for Everyone - Good credit - bad credit - divorce - charge off's - NO PROBLEM. To complete a secure credit application, please visit our website at www.sheehanautoplex.com ***The largest Dealer in the State of Florida - We export all over the world - For details please visit www.sheehanautoplex.com ***Sheehan Autoplex takes great pride in our outstanding customer service and has been recognized by the following associations - BBB (Better Business Bureau) - NIADA - and the FIADA. Call us to get your best deal. CALL NOW. 1-888-453-5244\302\240 Contact Sheehan Buick Pontiac GMCPhone:(754) 224-3257Fax:(954) 781-9050Phone:(754) 224-3257E-mail:sales@proauto.comBusiness HoursWeekdays:9:00 AM to 9:00 PMSat:9:00 AM to 6:00 PMSun:",posting2.contents_as_plain
     assert_equal 23975.0, posting2.price
 
-    posting3 = CraigScrape::PostFull.new read_as_hpricot('post_samples/posting3.html')
+    posting3 = CraigScrape.scrape_full_post relative_uri_for('post_samples/posting3.html')
     assert_equal "1992 Twin Turbo 300ZX. This car is pearl white outside and Camel leather interior with suede accents. Motor was re-done from the ground up two years ago. 23,000 on new motor rebuild! New Leather seats and center arm rest done also two years ago. Has Alpine Am/Fm Cd with Ipod cable, Viper pager alarm New! JL Audio Amp & JLAudio sub box custom made. Mtx mids& highs component speakers sparate tweeter. Car runs strong & straight. Just detailed the interior. Exterior should be painted. This car once painted will sell for over $10,000. \r<br />\nCome get a great deal now! offers and trades will be considered. 786-303-6550 Manny", posting3.contents
     assert_equal ["south florida craigslist", "miami / dade", "cars & trucks - by owner"], posting3.full_section
     assert_equal "300ZX Nissan Twin Turbo 1992 - $5800 (N.Miami/ Hialeah)", posting3.header
@@ -207,7 +211,7 @@ EOD
     assert_equal 5800.0, posting3.price
 
     # This one ended up being quite a curveball since the user uploaded HTML was such junk:
-    posting4 = CraigScrape::PostFull.new read_as_hpricot('post_samples/posting4.html')
+    posting4 = CraigScrape.scrape_full_post relative_uri_for('post_samples/posting4.html')
     assert_equal 20640, posting4.contents.length
     assert_equal ["south florida craigslist", "broward county", "real estate - by broker"], posting4.full_section
     assert_equal "$225000 / 3br - Palm Aire Golf Corner Unit!", posting4.header
@@ -220,7 +224,7 @@ EOD
     assert_equal 6399,posting4.contents_as_plain.length
     assert_equal 225000.0, posting4.price
     
-    posting5 = CraigScrape::PostFull.new read_as_hpricot('post_samples/posting5.html')
+    posting5 = CraigScrape.scrape_full_post relative_uri_for('post_samples/posting5.html')
     assert_equal true, posting5.flagged_for_removal?
     assert_equal nil, posting5.contents
     assert_equal ["south florida craigslist", "palm beach co", "apts/housing for rent"], posting5.full_section
@@ -233,6 +237,35 @@ EOD
     assert_equal [],  posting5.images
     assert_equal nil, posting5.contents_as_plain
     assert_equal nil, posting5.price
+    
+    posting_deleted = CraigScrape.scrape_full_post relative_uri_for('post_samples/this_post_has_been_deleted_by_its_author.html')
+    assert_equal true, posting_deleted.deleted_by_author?
+    assert_equal nil, posting_deleted.contents
+    assert_equal ["south florida craigslist", "broward county", "cars & trucks - by owner"], posting_deleted.full_section
+    assert_equal "This posting has been deleted by its author.", posting_deleted.header
+    assert_equal nil, posting_deleted.title
+    assert_equal nil, posting_deleted.location
+    assert_equal nil, posting_deleted.posting_id
+    assert_equal nil, posting_deleted.reply_to 
+    assert_equal nil, posting_deleted.post_time
+    assert_equal [],  posting_deleted.images
+    assert_equal nil, posting_deleted.contents_as_plain
+    assert_equal nil, posting_deleted.price
+
+    posting6 = CraigScrape.scrape_full_post relative_uri_for('post_samples/1207457727.html')
+    assert_equal "<p><br />Call!! asking for a new owner.<br />  no deposit required rent to own properties. <br /> <br /> Defaulting payment records are not a problem, <br /> we will help you protect the previous owners credit history! 202-567-6371  <br /><br /></p>",posting6.contents
+    assert_equal "Call!! asking for a new owner.  no deposit required rent to own properties.   Defaulting payment records are not a problem,  we will help you protect the previous owners credit history! 202-567-6371  ",posting6.contents_as_plain
+    assert_equal false,posting6.deleted_by_author?
+    assert_equal false,posting6.flagged_for_removal?
+    assert_equal ["south florida craigslist", "broward county", "apts/housing for rent"],posting6.full_section
+    assert_equal "$1350 / 3br - 2bth for no deposit req (Coral Springs)",posting6.header
+    assert_equal ["http://images.craigslist.org/3k43pe3o8ZZZZZZZZZ9655022102a3ea51624.jpg", "http://images.craigslist.org/3n13m53p6ZZZZZZZZZ96596515e51237a179c.jpg", "http://images.craigslist.org/3od3p33leZZZZZZZZZ9656d614da8e3a51dd9.jpg", "http://images.craigslist.org/3pb3oa3leZZZZZZZZZ965eb60e4d2344019fb.jpg"],posting6.images
+    assert_equal 'Coral Springs',posting6.location
+    assert_equal [0, 56, 18, 5, 6, 2009, 5, 156, true, "EDT"],posting6.post_time.to_a
+    assert_equal 1207457727,posting6.posting_id
+    assert_equal 1350.0,posting6.price
+    assert_equal "hous-ccpap-1207457727@craigslist.org",posting6.reply_to
+    assert_equal "2bth for no deposit req",posting6.title    
   end
 
   private
@@ -242,4 +275,26 @@ EOD
       File.open('%s/%s' % [File.dirname(__FILE__), test_file]).read
     )
   end
+  
+  def relative_uri_for(filename)
+    'file://%s/%s' % [File.dirname(File.expand_path(__FILE__)), filename]
+  end
+  
+  def pp_assertions(obj, obj_name)
+    probable_accessors = (obj.methods-obj.class.superclass.methods)
+    
+    puts
+    probable_accessors.sort.each do |m|
+      val = obj.send(m.to_sym)
+      
+      # There's a good number of transformations worth doing here, I'll just start like this for now:
+      if val.kind_of? Time
+        # I've decided this is the the easiest way to understand and test a time
+        val = val.to_a
+        m = "#{m}.to_a"
+      end
+      
+      puts "assert_equal %s, %s.%s" % [val.inspect,obj_name,m]
+    end    
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: libcraigscrape
 version: !ruby/object:Gem::Version 
-  version: "0.6"
+  version: 0.6.5
 platform: ruby
 authors: 
 - Chris DeRose, DeRose Technologies, Inc.
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-05-21 00:00:00 -04:00
+date: 2009-06-08 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -68,6 +68,7 @@ files:
 - test/listing_samples
 - test/listing_samples/category_output.html
 - test/listing_samples/short_search_output.html
+- test/listing_samples/empty_listings.html
 - test/listing_samples/mia_fua_index8900.5.21.09.html
 - test/listing_samples/category_output_2.html
 - test/listing_samples/long_search_output.html
@@ -78,6 +79,8 @@ files:
 - test/post_samples/posting0.html
 - test/post_samples/posting5.html
 - test/post_samples/posting3.html
+- test/post_samples/this_post_has_been_deleted_by_its_author.html
+- test/post_samples/1207457727.html
 - test/post_samples/posting2.html
 - test/google.html
 - lib/libcraigscrape.rb