libcraigscrape 0.9.1 → 1.0

data/CHANGELOG

@@ -1,5 +1,8 @@
 == Change Log
 
+=== Release 1.0
+- Replaced hpricot dependency with Nokogiri. Nokogiri should be faster and more reliable. Whoo-hoo!
+
 === Release 0.9.1
 - Added support for posting_has_expired? and expired post recognition
 - Fixed a weird bug in craigwatch that would cause a scrape to abort if a flagged_for_removal? was encountered when using certain (minimal) filtering 

data/Rakefile

@@ -11,7 +11,7 @@ include FileUtils
 RbConfig = Config unless defined? RbConfig
 
 NAME = "libcraigscrape"
-VERS = ENV['VERSION'] || "0.9.1"
+VERS = ENV['VERSION'] || "1.0"
 PKG = "#{NAME}-#{VERS}"
 
 RDOC_OPTS = ['--quiet', '--title', 'The libcraigscrape Reference', '--main', 'README', '--inline-source']
@@ -37,7 +37,7 @@ SPEC =
     s.files = PKG_FILES
     s.require_paths = ["lib"] 
     s.test_files = FileList['test/test_*.rb']
-    s.add_dependency 'hpricot',      '> 0.8'
+    s.add_dependency 'nokogiri',     '>= 1.4.4'
     s.add_dependency 'htmlentities', '>= 4.0.0'
     s.add_dependency 'activesupport','>= 2.3.0', '< 3'
     s.add_dependency 'activerecord', '>= 2.3.0', '< 3'

data/lib/geo_listings.rb

@@ -36,7 +36,7 @@ class CraigScrape
     def location
       unless @location
         cursor = html % 'h3 > b > a:first-of-type'
-        cursor = cursor.next_node if cursor       
+        cursor = cursor.next if cursor       
         @location = $1 if cursor and LOCATION_NAME.match he_decode(cursor.to_s)
       end
       

data/lib/libcraigscrape.rb

@@ -5,12 +5,13 @@
 require 'rubygems'
 
 gem 'activesupport', '~> 2.3'
-gem 'hpricot',       '~> 0.8'
+gem 'nokogiri',      '~> 1.4.4'
 gem 'htmlentities',  '~> 4.0.0'
 
+
 require 'net/http'
 require 'zlib'
-require 'hpricot'
+require 'nokogiri'
 require 'htmlentities'
 require 'active_support'
 

data/lib/listings.rb

@@ -21,7 +21,8 @@ class CraigScrape::Listings < CraigScrape::Scraper
       current_date = nil
       @posts = []
 
-      post_tags = html.get_elements_by_tag_name('p','h4')
+      # All we care about are p and h4 tags. This seemed to be the only way I could do this on Nokogiri:
+      post_tags = html.search('*').reject{|n| !/^(?:p|h4)$/i.match n.name } 
 
       # The last p in the list is sometimes a 'next XXX pages' link. We don't want to include this in our PostSummary output:
       post_tags.pop if (
@@ -82,7 +83,7 @@ class CraigScrape::Listings < CraigScrape::Scraper
 
         # If there's no 'a' in the next sibling, we'll have just performed a nil assignment, otherwise
         # We're looking good.
-        next_link = cursor.next_sibling if cursor and /^[\d]+$/.match cursor.inner_html
+        next_link = cursor.next_element if cursor and /^[\d]+$/.match cursor.inner_html
       end
       
       # We have an anchor tag - so - let's assign the href:

data/lib/posting.rb

@@ -79,7 +79,7 @@ class CraigScrape::Posting < CraigScrape::Scraper
   def reply_to
     unless @reply_to
       cursor = html_head.at 'hr' if html_head
-      cursor = cursor.next_sibling until cursor.nil? or cursor.name == 'a'
+      cursor = cursor.next until cursor.nil? or cursor.name == 'a'
       @reply_to = $1 if cursor and REPLY_TO.match he_decode(cursor.inner_html)
     end
     
@@ -90,7 +90,7 @@ class CraigScrape::Posting < CraigScrape::Scraper
   def post_time
     unless @post_time
       cursor = html_head.at 'hr' if html_head
-      cursor = cursor.next_node until cursor.nil? or POST_DATE.match cursor.to_s
+      cursor = cursor.next until cursor.nil? or POST_DATE.match cursor.to_s
       @post_time = Time.parse $1 if $1
     end
     
@@ -100,8 +100,8 @@ class CraigScrape::Posting < CraigScrape::Scraper
   # Integer, Craigslist's unique posting id
   def posting_id
     unless @posting_id     
-      cursor = Hpricot.parse html_footer if html_footer
-      cursor = cursor.next_node until cursor.nil? or POSTING_ID.match cursor.to_s
+      cursor = Nokogiri::HTML html_footer, nil, HTML_ENCODING if html_footer
+      cursor = cursor.next until cursor.nil? or POSTING_ID.match cursor.to_s
       @posting_id = $1.to_i if $1
     end
   
@@ -135,7 +135,7 @@ class CraigScrape::Posting < CraigScrape::Scraper
       # Real estate listings can work a little different for location:
       unless @location
         cursor = craigslist_body.at 'small'
-        cursor = cursor.previous_node until cursor.nil? or cursor.text?
+        cursor = cursor.previous until cursor.nil? or cursor.text?
         
         @location = he_decode(cursor.to_s.strip) if cursor
       end
@@ -295,7 +295,7 @@ class CraigScrape::Posting < CraigScrape::Scraper
   # I set apart from html to work around the SystemStackError bugs in test_bugs_found061710. Essentially we 
   # return everything above the user_body
   def html_head
-    @html_head = Hpricot.parse $1 if @html_head.nil? and HTML_HEADER.match html_source
+    @html_head = Nokogiri::HTML  $1, nil, HTML_ENCODING if @html_head.nil? and HTML_HEADER.match html_source
     # We return html itself if HTML_HEADER doesn't match, which would be case for a 404 page or something
     @html_head ||= html
      
@@ -316,9 +316,9 @@ class CraigScrape::Posting < CraigScrape::Scraper
   end
   
   # Read the notes on user_body. However,  unlike the user_body, the craigslist portion of this div can be relied upon to be valid html. 
-  # So - we'll return it as an Hpricot object.
+  # So - we'll return it as a Nokogiri object.
   def craigslist_body
-    Hpricot.parse $3 if USERBODY_PARTS.match html_source
+    Nokogiri::HTML $3, nil, HTML_ENCODING if USERBODY_PARTS.match html_source
   end
 
 end

data/lib/scraper.rb

@@ -33,6 +33,8 @@ class CraigScrape::Scraper
 
   URL_PARTS = /^(?:([^\:]+)\:\/\/([^\/]*))?(.*)$/
   HTML_TAG  = /<\/?[^>]*>/
+  # We have to specify this to nokogiri. Sometimes it tries to figure out encoding on its own, and craigslist users post crazy bytes sometimes  
+  HTML_ENCODING = "UTF-8"
   
   # Returns the full url that corresponds to this resource
   attr_reader :url
@@ -202,9 +204,9 @@ class CraigScrape::Scraper
     @html_source
   end
   
-  # Returns an hpricot parse, of the current URI
+  # Returns an Nokogiri parse, of the current URI
   def html
-    @html ||= Hpricot.parse html_source if html_source
+    @html ||= Nokogiri::HTML html_source, nil, HTML_ENCODING if html_source
     @html
   end
 end  

data/test/test_craigslist_listing.rb

@@ -210,47 +210,47 @@ class CraigslistListingTest < Test::Unit::TestCase
           
     mia_search_kitten031510 = CraigScrape::Listings.new relative_uri_for('listing_samples/mia_search_kitten.3.15.10.html')
     assert_equal "Adopt a 7 month on kitten- $75", mia_search_kitten031510.posts[0].label
-    assert_equal [0, 0, 0, 15, 3, 2010, 1, 74, true, "EDT"], mia_search_kitten031510.posts[0].post_date.to_a
+    assert_equal [15, 3], mia_search_kitten031510.posts[0].post_date.to_a[3..4]
     assert_equal "Adorable Kitten! Free!!!", mia_search_kitten031510.posts[1].label
-    assert_equal [0, 0, 0, 15, 3, 2010, 1, 74, true, "EDT"], mia_search_kitten031510.posts[1].post_date.to_a
+    assert_equal [15, 3], mia_search_kitten031510.posts[1].post_date.to_a[3..4]
     assert_equal "KITTENS,5 months, 1 Russian blue, 1 grey & white,vac spy/neu,$35fee ea", mia_search_kitten031510.posts[2].label
-    assert_equal [0, 0, 0, 13, 3, 2010, 6, 72, false, "EST"], mia_search_kitten031510.posts[2].post_date.to_a
+    assert_equal [13, 3], mia_search_kitten031510.posts[2].post_date.to_a[3..4]
     assert_equal "Kitties need a good home", mia_search_kitten031510.posts[3].label
-    assert_equal [0, 0, 0, 13, 3, 2010, 6, 72, false, "EST"], mia_search_kitten031510.posts[3].post_date.to_a
+    assert_equal [13, 3], mia_search_kitten031510.posts[3].post_date.to_a[3..4]
     assert_equal "7 week old kittens for adoption", mia_search_kitten031510.posts[4].label
-    assert_equal [0, 0, 0, 13, 3, 2010, 6, 72, false, "EST"], mia_search_kitten031510.posts[4].post_date.to_a
+    assert_equal [13, 3], mia_search_kitten031510.posts[4].post_date.to_a[3..4]
     assert_equal "Adorable Orange Kitten Free to Good Home", mia_search_kitten031510.posts[5].label
-    assert_equal [0, 0, 0, 12, 3, 2010, 5, 71, false, "EST"], mia_search_kitten031510.posts[5].post_date.to_a
+    assert_equal [12, 3], mia_search_kitten031510.posts[5].post_date.to_a[3..4]
     assert_equal "7 month old kitten free to good home", mia_search_kitten031510.posts[6].label
-    assert_equal [0, 0, 0, 12, 3, 2010, 5, 71, false, "EST"], mia_search_kitten031510.posts[6].post_date.to_a
+    assert_equal [12, 3], mia_search_kitten031510.posts[6].post_date.to_a[3..4]
     assert_equal "FEMALE KITTEN FOR GOOD HOME", mia_search_kitten031510.posts[7].label
-    assert_equal [0, 0, 0, 9, 3, 2010, 2, 68, false, "EST"], mia_search_kitten031510.posts[7].post_date.to_a
+    assert_equal [9, 3], mia_search_kitten031510.posts[7].post_date.to_a[3..4]
     assert_equal "Kitten", mia_search_kitten031510.posts[8].label
-    assert_equal [0, 0, 0, 4, 3, 2010, 4, 63, false, "EST"], mia_search_kitten031510.posts[8].post_date.to_a
+    assert_equal [4, 3], mia_search_kitten031510.posts[8].post_date.to_a[3..4]
     assert_equal "Kitties need a good home", mia_search_kitten031510.posts[9].label
-    assert_equal [0, 0, 0, 4, 3, 2010, 4, 63, false, "EST"], mia_search_kitten031510.posts[9].post_date.to_a
+    assert_equal [4, 3], mia_search_kitten031510.posts[9].post_date.to_a[3..4]
     assert_equal "Persain Cat And Tabby Cat", mia_search_kitten031510.posts[10].label
-    assert_equal [0, 0, 0, 1, 3, 2010, 1, 60, false, "EST"], mia_search_kitten031510.posts[10].post_date.to_a
+    assert_equal [1, 3], mia_search_kitten031510.posts[10].post_date.to_a[3..4]
     assert_equal "Tabby female kitten in a parking lot needs your help", mia_search_kitten031510.posts[11].label
-    assert_equal [0, 0, 0, 23, 2, 2010, 2, 54, false, "EST"], mia_search_kitten031510.posts[11].post_date.to_a
+    assert_equal [23, 2], mia_search_kitten031510.posts[11].post_date.to_a[3..4]
     assert_equal "Spring is almost officially here, grow your family, adopt a kitty!", mia_search_kitten031510.posts[12].label
-    assert_equal [0, 0, 0, 22, 2, 2010, 1, 53, false, "EST"], mia_search_kitten031510.posts[12].post_date.to_a
+    assert_equal [22, 2], mia_search_kitten031510.posts[12].post_date.to_a[3..4]
     assert_equal "Many adorable kittens for adoption!", mia_search_kitten031510.posts[13].label
-    assert_equal [0, 0, 0, 22, 2, 2010, 1, 53, false, "EST"], mia_search_kitten031510.posts[13].post_date.to_a
+    assert_equal [22, 2], mia_search_kitten031510.posts[13].post_date.to_a[3..4]
     assert_equal "2 free cats/kitten to good home", mia_search_kitten031510.posts[14].label
-    assert_equal [0, 0, 0, 19, 2, 2010, 5, 50, false, "EST"], mia_search_kitten031510.posts[14].post_date.to_a
+    assert_equal [19, 2], mia_search_kitten031510.posts[14].post_date.to_a[3..4]
     assert_equal "BEAUTIFUL KITTENS", mia_search_kitten031510.posts[15].label
-    assert_equal [0, 0, 0, 19, 2, 2010, 5, 50, false, "EST"], mia_search_kitten031510.posts[15].post_date.to_a
+    assert_equal [19, 2], mia_search_kitten031510.posts[15].post_date.to_a[3..4]
     assert_equal "MANY new adorable kittens for good homes!!!", mia_search_kitten031510.posts[16].label
-    assert_equal [0, 0, 0, 18, 2, 2010, 4, 49, false, "EST"], mia_search_kitten031510.posts[16].post_date.to_a
+    assert_equal [18, 2], mia_search_kitten031510.posts[16].post_date.to_a[3..4]
     assert_equal "Kitten living in a parking lot needs your help", mia_search_kitten031510.posts[17].label
-    assert_equal [0, 0, 0, 16, 2, 2010, 2, 47, false, "EST"], mia_search_kitten031510.posts[17].post_date.to_a
+    assert_equal [16, 2], mia_search_kitten031510.posts[17].post_date.to_a[3..4]
     assert_equal "BEAUTIFUL 8 WEEK KITTENS", mia_search_kitten031510.posts[18].label
-    assert_equal [0, 0, 0, 16, 2, 2010, 2, 47, false, "EST"], mia_search_kitten031510.posts[18].post_date.to_a
+    assert_equal [16, 2], mia_search_kitten031510.posts[18].post_date.to_a[3..4]
     assert_equal "ORANGE TABBY KITTEN", mia_search_kitten031510.posts[19].label
-    assert_equal [0, 0, 0, 13, 2, 2010, 6, 44, false, "EST"], mia_search_kitten031510.posts[19].post_date.to_a
+    assert_equal [13, 2], mia_search_kitten031510.posts[19].post_date.to_a[3..4]
     assert_equal "Lots of kittens to choose from! Pics!!", mia_search_kitten031510.posts[20].label
-    assert_equal [0, 0, 0, 13, 2, 2010, 6, 44, false, "EST"], mia_search_kitten031510.posts[20].post_date.to_a
+    assert_equal [13, 2], mia_search_kitten031510.posts[20].post_date.to_a[3..4]
 
   end
 

data/test/test_craigslist_posting.rb

@@ -37,7 +37,9 @@ EOD
 <p><a href="/mdc/jwl/1128691192.html">925 Sterling Silver Dragonfly Charm Bracelet - $25 -</a> <span class="p"> img</span></p>
 EOD
 
-    one = CraigScrape::Posting.new CraigScrape::Listings.parse_summary(Hpricot.parse(search_html_one).at('p'))
+    one = CraigScrape::Posting.new CraigScrape::Listings.parse_summary(
+      Nokogiri::HTML(search_html_one, nil, CraigScrape::Scraper::HTML_ENCODING).at('p')
+    )
     assert_equal true, one.has_img?
     assert_equal false, one.has_pic?
     assert_equal true, one.has_pic_or_img?
@@ -49,7 +51,9 @@ EOD
     assert_equal 18, one.post_date.day
     assert_equal nil, one.price
 
-    two = CraigScrape::Posting.new CraigScrape::Listings.parse_summary(Hpricot.parse(search_html_two).at('p'))
+    two = CraigScrape::Posting.new CraigScrape::Listings.parse_summary(
+      Nokogiri::HTML(search_html_two, nil, CraigScrape::Scraper::HTML_ENCODING).at('p')
+    )
     assert_equal true, two.has_img?
     assert_equal true, two.has_pic?
     assert_equal true, two.has_pic_or_img?
@@ -61,7 +65,9 @@ EOD
     assert_equal 4, two.post_date.day
     assert_equal 348000.0, two.price
 
-    three = CraigScrape::Posting.new CraigScrape::Listings.parse_summary(Hpricot.parse(search_html_three).at('p'))
+    three = CraigScrape::Posting.new CraigScrape::Listings.parse_summary(
+      Nokogiri::HTML(search_html_three, nil, CraigScrape::Scraper::HTML_ENCODING).at('p')
+    )
     assert_equal false, three.has_img?
     assert_equal true, three.has_pic?
     assert_equal true, three.has_pic_or_img?
@@ -73,7 +79,9 @@ EOD
     assert_equal 31, three.post_date.day
     assert_equal 22.0, three.price
 
-    four = CraigScrape::Posting.new CraigScrape::Listings.parse_summary(Hpricot.parse(search_html_four).at('p'))
+    four = CraigScrape::Posting.new CraigScrape::Listings.parse_summary(
+      Nokogiri::HTML(search_html_four, nil, CraigScrape::Scraper::HTML_ENCODING).at('p')
+    )
     assert_equal false, four.has_img?
     assert_equal false, four.has_pic?
     assert_equal false, four.has_pic_or_img?
@@ -85,7 +93,9 @@ EOD
     assert_equal 22, four.post_date.day
     assert_equal 325000.0, four.price
 
-    five = CraigScrape::Posting.new CraigScrape::Listings.parse_summary(Hpricot.parse(search_html_five).at('p'))
+    five = CraigScrape::Posting.new CraigScrape::Listings.parse_summary(
+      Nokogiri::HTML(search_html_five, nil, CraigScrape::Scraper::HTML_ENCODING).at('p')
+    )
     assert_equal false, five.has_img?
     assert_equal true, five.has_pic?
     assert_equal true, five.has_pic_or_img?
@@ -97,27 +107,31 @@ EOD
     assert_equal 9, five.post_date.day
     assert_equal 105000.0, five.price
 
-    five = CraigScrape::Posting.new CraigScrape::Listings.parse_summary(Hpricot.parse(category_listing_one).at('p'))
-    assert_equal false, five.has_img?
-    assert_equal true,  five.has_pic?
-    assert_equal true, five.has_pic_or_img?
-    assert_equal '/pbc/reb/1128661387.html', five.href
-    assert_equal "$2995000 / 5br - Downtown Boca New Home To Be Built", five.label
-    assert_equal "real estate - by broker", five.section
-    assert_equal "Boca Raton", five.location
-    assert_equal nil, five.post_date
-    assert_equal 2995000.0, five.price
-
-    six = CraigScrape::Posting.new CraigScrape::Listings.parse_summary(Hpricot.parse(category_listing_two).at('p'))
-    assert_equal true, six.has_img?
-    assert_equal false,  six.has_pic?
+    six = CraigScrape::Posting.new CraigScrape::Listings.parse_summary(
+      Nokogiri::HTML(category_listing_one, nil, CraigScrape::Scraper::HTML_ENCODING).at('p')
+    )
+    assert_equal false, six.has_img?
+    assert_equal true,  six.has_pic?
     assert_equal true, six.has_pic_or_img?
-    assert_equal '/mdc/jwl/1128691192.html', six.href
-    assert_equal "925 Sterling Silver Dragonfly Charm Bracelet - $25", six.label
-    assert_equal nil, six.section
-    assert_equal nil, six.location
+    assert_equal '/pbc/reb/1128661387.html', six.href
+    assert_equal "$2995000 / 5br - Downtown Boca New Home To Be Built", six.label
+    assert_equal "real estate - by broker", six.section
+    assert_equal "Boca Raton", six.location
     assert_equal nil, six.post_date
-    assert_equal 25.0, six.price
+    assert_equal 2995000.0, six.price
+
+    seven = CraigScrape::Posting.new CraigScrape::Listings.parse_summary(
+      Nokogiri::HTML(category_listing_two, nil, CraigScrape::Scraper::HTML_ENCODING).at('p')
+    )
+    assert_equal true, seven.has_img?
+    assert_equal false,  seven.has_pic?
+    assert_equal true, seven.has_pic_or_img?
+    assert_equal '/mdc/jwl/1128691192.html', seven.href
+    assert_equal "925 Sterling Silver Dragonfly Charm Bracelet - $25", seven.label
+    assert_equal nil, seven.section
+    assert_equal nil, seven.location
+    assert_equal nil, seven.post_date
+    assert_equal 25.0, seven.price
   end
   
 
@@ -302,7 +316,7 @@ EOD
     assert_equal [:pic], sfbay_art_1223614914.img_types
   end
   
-  # This is actually a 'bug' with hpricot itself when the ulimit is set too low. 
+  # This was actually a 'bug' with hpricot itself when the ulimit is set too low. 
   # the Easy fix is running "ulimit -s 16384" before the tests. But the better fix was
   # to remove the userbody sending these pages to be parsed by Hpricot
   def test_bugs_found061710

metadata

@@ -1,13 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: libcraigscrape
 version: !ruby/object:Gem::Version 
-  hash: 57
+  hash: 15
   prerelease: false
   segments: 
-  - 0
-  - 9
   - 1
-  version: 0.9.1
+  - 0
+  version: "1.0"
 platform: ruby
 authors: 
 - Chris DeRose, DeRose Technologies, Inc.
@@ -15,22 +14,23 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-01-05 00:00:00 -05:00
+date: 2011-02-13 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  name: hpricot
+  name: nokogiri
   prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
-    - - ">"
+    - - ">="
       - !ruby/object:Gem::Version 
-        hash: 27
+        hash: 15
         segments: 
-        - 0
-        - 8
-        version: "0.8"
+        - 1
+        - 4
+        - 4
+        version: 1.4.4
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency