libcraigscrape 0.6.5 → 0.7.0

data/CHANGELOG

@@ -1,5 +1,22 @@
 == Change Log
 
+=== Release 0.7.0 (Jul 5, 2009)
+- A good bit of refactoring
+- Eager-loading in the Post object without the need of the full_post method
+- full_post is no longer needed or available
+- Added a Base Scraper object. Maybe I'll make that its own gem...
+- Post/Listing constructors now take either a Hash or a url (string) to use for its scrape. Regardless, either should always have a url object association now
+- Removed the PostSummary object, now we just have Posting/ Posts include all the functionality of both PostFull and PostSummary. Be careful with Posting if you're trying to minimize bandwidth. the rdoc labels which methods wont/will/might cause a page load.
+- Things should fail better (always outputting relevant html that was unable to be parsed)
+- Removed Posting::date in favor of Posting::post_date
+- Posting::full_url is now url
+- Post::href is no longer needed. Use Post::uri.path instead
+- Posting.images renamed to Posting.pics, and a new method Posting.images better reflects the craigslist designations
+- Fixed a bug with some search listings, where the last page might not get scrapped b/c craigslist doesn't correctly include the next page link. see 'test_nasty_search_listings' for an example
+- On some listings (with the empty h4 tags) we were eager-loading when we could have made some safe assumptions instead. Fixed.
+- Preliminary support for GeoListing scrapes.. (Not exactly sure where this will go - but I have ideas..)
+- Adjusted Posting::images and Posting::pics to better match the craigslist media-dicotomy notation.
+
 === Release 0.6.5 (Jun 8, 2009)
 - Added PostFull::deleted_by_author? , added test case for said condition
 - Fixed a bug that caused the library to die in weird ways if there wasn't a title tag on a parsed page

data/Rakefile

@@ -11,7 +11,7 @@ include FileUtils
 RbConfig = Config unless defined? RbConfig
 
 NAME = "libcraigscrape"
-VERS = ENV['VERSION'] || "0.6.5"
+VERS = ENV['VERSION'] || "0.7.0"
 PKG = "#{NAME}-#{VERS}"
 
 RDOC_OPTS = ['--quiet', '--title', 'The libcraigscrape Reference', '--main', 'README', '--inline-source']

data/bin/craigwatch

@@ -193,10 +193,10 @@ class CraigReportDefinition #:nodoc:
       
       if full_post_has or full_post_has_no
         # We're going to download the page, so let's make sure we didnt hit a "This posting has been flagged for removal"
-        return false if post.full_post.title.nil?
+        return false if post.system_post?
         
-        return false unless matches_all? full_post_has, post.full_post.contents_as_plain
-        return false unless doesnt_match_any? full_post_has_no, post.full_post.contents_as_plain
+        return false unless matches_all? full_post_has, post.contents_as_plain
+        return false unless doesnt_match_any? full_post_has_no, post.contents_as_plain
       end
 
       true
@@ -303,7 +303,7 @@ ActiveRecord::Base.logger = Logger.new STDERR if craig_report.debug_database?
 ActiveRecord::Base.establish_connection craig_report.tracking_database
 
 # Initialize CraigScrape (sorta)
-CraigScrape.logger = Logger.new STDERR if craig_report.debug_craigscrape?
+CraigScrape::Scraper.logger = Logger.new STDERR if craig_report.debug_craigscrape?
 
 # Perform migrations if needed?
 ActiveRecord::Schema.define do
@@ -340,18 +340,18 @@ craig_report.each_search do |search|
   # Let's collect all the summaries that could apply:
   new_summaries = {}
   search.listing.each do |listing|
-    CraigScrape.scrape_until(listing){|p| p.date <= last_tracked_at or already_tracked_urls.include? p.full_url }.each do |p_s|
-      new_summaries[p_s.full_url] = p_s unless new_summaries.has_key? p_s.full_url
+    CraigScrape.scrape_until(listing){|p| p.post_date <= last_tracked_at or already_tracked_urls.include? p.url }.each do |p_s|
+      new_summaries[p_s.url] = p_s unless new_summaries.has_key? p_s.url
     end
   end
 
   # Let's flatten the unique'd hash into a more useable array:
-  new_summaries = new_summaries.values.sort{|a,b| a.date <=> b.date} # oldest goes to bottom
+  new_summaries = new_summaries.values.sort{|a,b| a.post_date <=> b.post_date} # oldest goes to bottom
   
   # Let's tag all the newest tracked posts that should go into the database: 
   # NOTE: Since all the dates are at_begining_of_day, we'll effectively have a chunk of dates tied for latest
-  new_summaries.reject{|p| p.date < new_summaries.last.date}.each do |p_s|
-    newly_tracked_posts << search_track.tracked_posts.build( :url => p_s.full_url, :created_at => p_s.date)
+  new_summaries.reject{|p| p.post_date < new_summaries.last.post_date}.each do |p_s|
+    newly_tracked_posts << search_track.tracked_posts.build( :url => p_s.url, :created_at => p_s.post_date)
   end
   
   # Reject anything from this report which doesn't match the has/has_no :
@@ -361,7 +361,7 @@ craig_report.each_search do |search|
   if new_summaries.length > 0  
 
     # We'll use this in the cleanup at the bottom:
-    latest_post_date = new_summaries.last.date
+    latest_post_date = new_summaries.last.post_date
     
     new_summaries.reverse! if search.newest_first?
     

data/bin/report_mailer/craigslist_report.html.erb

@@ -4,8 +4,8 @@
    <h3><%=h summary[:search].name%></h3>
       <%summary[:postings].each do |post|%>
          <%='<p>%s <a href="%s">%s -</a>%s%s</p>' % [
-			h(post.date.strftime('%b %d')),
-			post.full_url,
+			h(post.post_date.strftime('%b %d')),
+			post.url,
 			h(post.label),
 			(post.location) ? '<font size="-1"> (%s)</font>' % h(post.location) : '',
 			(post.has_pic_or_img?) ? ' <span style="color: orange"> img</span>': ''

data/bin/report_mailer/craigslist_report.plain.erb

@@ -4,11 +4,11 @@ CRAIGSLIST REPORTER
    <%=summary[:search].name %>
    <% summary[:postings].collect do |post| -%>
       <%='%s : %s %s %s %s' % [
-			post.date.strftime('%b %d'),
+			post.post_date.strftime('%b %d'),
 			post.label,
 			(post.location) ? " (#{post.location})" : '',
 			(post.has_pic_or_img?) ? ' [img]': '',
-			post.full_url
+			post.url
       ] -%>
 
    <% end %>