olek-libcraigscrape 1.0.1

data/bin/report_mailer/craigslist_report.html.erb

@@ -0,0 +1,17 @@
+<h2><%=h @subject %></h2>
+<%@summaries.each do |summary| %>
+   <h3><%=h summary[:search].name%></h3>
+      <% if summary[:postings].length > 0 %>
+         <%summary[:postings].each do |post|%>
+            <%='<p>%s <a href="%s">%s -</a>%s%s</p>' % [
+            h(post.post_date.strftime('%b %d')),
+            post.url,
+            h(post.label),
+            (post.location) ? '<font size="-1"> (%s)</font>' % h(post.location) : '',
+            (post.has_pic_or_img?) ? ' <span style="color: orange"> img</span>': ''
+            ] -%>
+         <% end %>
+      <% else %>
+         <p><i>No new postings were found, which matched the search criteria.</i></p>
+      <% end %>
+<% end %>

data/bin/report_mailer/craigslist_report.plain.erb

@@ -0,0 +1,18 @@
+CRAIGSLIST REPORTER
+
+<%@summaries.each do |summary| -%>
+   <%=summary[:search].name %>
+   <% summary[:postings].collect do |post| -%>
+      <% if summary[:postings].length > 0 %>
+      <%='%s : %s %s %s %s' % [
+            post.post_date.strftime('%b %d'),
+            post.label,
+            (post.location) ? " (#{post.location})" : '',
+            (post.has_pic_or_img?) ? ' [img]': '',
+            post.url
+      ] -%>
+      <% else %>
+      No new postings were found, which matched the search criteria.
+      <% end %>
+   <% end %>
+<% end -%>

data/lib/geo_listings.rb

@@ -0,0 +1,144 @@
+# = About geo_listings.rb
+#
+# This file contains the parsing code, and logic relating to geographic site pages and paths. You
+# should never need to include this file directly, as all of libcraigscrape's objects and methods
+# are loaded when you use <tt>require 'libcraigscrape'</tt> in your code.
+#
+
+require 'scraper'
+
+class CraigScrape
+
+  # GeoListings represents a parsed Craigslist geo lisiting page. (i.e. {'http://geo.craigslist.org/iso/us'}[http://geo.craigslist.org/iso/us])
+  # These list all the craigslist sites in a given region.
+  class GeoListings < Scraper
+    GEOLISTING_BASE_URL = %{http://geo.craigslist.org/iso/}
+
+    LOCATION_NAME    = /[ ]*\>[ ](.+)[ ]*/
+    PATH_SCANNER     = /(?:\\\/|[^\/])+/
+    URL_HOST_PART    = /^[^\:]+\:\/\/([^\/]+)[\/]?$/
+    SITE_PREFIX      = /^([^\.]+)/
+    FIND_SITES_PARTS = /^[ ]*([\+|\-]?)[ ]*(.+)[ ]*/
+
+    class BadGeoListingPath < StandardError #:nodoc:
+    end
+
+    # The geolisting constructor works like all other Scraper objects, in that it accepts a string 'url'.
+    # See the Craigscrape.find_sites for a more powerful way to find craigslist sites.
+    def initialize(init_via = nil)
+      super(init_via)
+
+      # Validate that required fields are present, at least - if we've downloaded it from a url
+      parse_error! unless location
+    end
+
+    # Returns the GeoLocation's full name
+    def location
+      unless @location
+        cursor = html % 'h3 > b > a:first-of-type'
+        cursor = cursor.next if cursor
+        @location = $1 if cursor and LOCATION_NAME.match he_decode(cursor.to_s)
+      end
+
+      @location
+    end
+
+    # Returns a hash of site name to urls in the current listing
+    def sites
+      unless @sites
+        @sites = {}
+        (html / 'div#list > a').each do |el_a|
+          site_name = he_decode strip_html(el_a.inner_html)
+          @sites[site_name] = $1 if URL_HOST_PART.match el_a[:href]
+        end
+      end
+
+      @sites
+    end
+
+    # This method will return an array of all possible sites that match the specified location path.
+    # Sample location paths:
+    # - us/ca
+    # - us/fl/miami
+    # - jp/fukuoka
+    # - mx
+    # Here's how location paths work.
+    # - The components of the path are to be separated by '/' 's.
+    # - Up to (and optionally, not including) the last component, the path should correspond against a valid GeoLocation url with the prefix of 'http://geo.craigslist.org/iso/'
+    # - the last component can either be a site's 'prefix' on a GeoLocation page, or, the last component can just be a geolocation page itself, in which case all the sites on that page are selected.
+    # - the site prefix is the first dns record in a website listed on a GeoLocation page. (So, for the case of us/fl/miami , the last 'miami' corresponds to the 'south florida' link on {'http://geo.craigslist.org/iso/us/fl'}[http://geo.craigslist.org/iso/us/fl]
+    def self.sites_in_path(full_path, base_url = GEOLISTING_BASE_URL)
+      # the base_url parameter is mostly so we can test this method
+
+      # Unfortunately - the easiest way to understand much of this is to see how craigslist returns
+      # these geolocations. Watch what happens when you request us/fl/non-existant/page/here.
+      # I also made this a little forgiving in a couple ways not specified with official support, per
+      # the rules above.
+      full_path_parts = full_path.scan PATH_SCANNER
+
+      # We'll either find a single site in this loop andf return that, or, we'll find a whole listing
+      # and set the geo_listing object to reflect that
+      geo_listing = nil
+      full_path_parts.each_with_index do |part, i|
+
+        # Let's un-escape the path-part, if needed:
+        part.gsub! "\\/", "/"
+
+        # If they're specifying a single site, this will catch and return it immediately
+        site = geo_listing.sites.find{ |n,s|
+          (SITE_PREFIX.match s and $1 == part) or n == part
+        } if geo_listing
+
+        # This returns the site component of the found array
+        return [site.last] if site
+
+        begin
+          # The URI escape is mostly needed to translate the space characters
+          l = GeoListings.new base_url+full_path_parts[0...i+1].collect{|p| URI.escape p}.join('/')
+        rescue CraigScrape::Scraper::FetchError
+          bad_geo_path! full_path
+        end
+
+        # This probably tells us the first part of the path was 'correct', but not the rest:
+        bad_geo_path! full_path if geo_listing and geo_listing.location == l.location
+
+        geo_listing = l
+      end
+
+      # We have a valid listing page we found, and we can just return all the sites on it:
+      geo_listing.sites.collect{|n,s| s }
+    end
+
+    # find_sites takes a single array of strings as an argument. Each string is to be either a location path
+    # (see sites_in_path), or a full site (in canonical form - ie "memphis.craigslist.org"). Optionally,
+    # each of this may/should contain a '+' or '-' prefix to indicate whether the string is supposed to
+    # include sites from the master list, or remove them from the list. If no '+' or'-' is
+    # specified, the default assumption is '+'. Strings are processed from left to right, which gives
+    # a high degree of control over the selection set. Examples:
+    # - find_sites "us/fl", "- miami.craigslist.org"
+    # - find_sites "us", "- us/nm"
+    # - find_sites "us", "- us/ny", "+ newyork.craigslist.org"
+    # - find_sites "us/ny", "us/id", "caribbean.craigslist.org"
+    # There's a lot of flexibility here, you get the idea.
+    def self.find_sites(specs, base_url = GEOLISTING_BASE_URL)
+      ret = []
+
+      specs.each do |spec|
+        (op,spec = $1,$2) if FIND_SITES_PARTS.match spec
+
+        spec = (spec.include? '.')  ? [spec] : sites_in_path(spec, base_url)
+
+        (op == '-') ? ret -= spec : ret |= spec
+      end
+
+      ret
+    end
+
+    private
+
+    def self.bad_geo_path!(path)
+      raise BadGeoListingPath, "Unable to load path #{path.inspect}, either you're having problems connecting to Craiglist, or your path is invalid."
+    end
+
+  end
+end

data/lib/libcraigscrape.rb

@@ -0,0 +1,217 @@
+# = About libcraigscrape.rb
+#
+# All of libcraigscrape's objects and methods are loaded when you use <tt>require 'libcraigscrape'</tt> in your code.
+#
+require 'rubygems'
+
+gem 'activesupport', '~> 2.3'
+gem 'nokogiri',      '>= 1.4.4'
+gem 'htmlentities',  '>= 4.0.0'
+
+
+require 'net/http'
+require 'zlib'
+require 'nokogiri'
+require 'htmlentities'
+require 'active_support'
+
+
+# A base class encapsulating the various libcraigscrape objects, and providing most of the
+# craigslist interaction methods. Currently, we're supporting the old Class methods
+# in a legacy-compatibility mode, but these methods are marked for deprecation. Instead,
+# create an instance of the Craigslist object, and use its Public Instance methods.
+# See the README for easy to follow examples.
+
+class CraigScrape
+  cattr_accessor :time_now
+  cattr_accessor :site_to_url_prefix
+
+  #--
+  # NOTE:
+  # The only reason I took this out is b/c I might want to test with a file://
+  # prefix at some point
+  #++
+  self.site_to_url_prefix = 'http://'
+
+
+  # Takes a variable number of site/path specifiers (strings) as an argument.
+  # This list gets flattened and passed to CraigScrape::GeoListings.find_sites .
+  # See that method's rdoc for a complete set of rules on what arguments are allowed here.
+  def initialize(*args)
+    @sites_specs = args.flatten
+  end
+
+  # Returns which sites are included in any operations performed by this object. This is directly
+  # ascertained from the initial constructor's spec-list
+  def sites
+    @sites ||= GeoListings.find_sites @sites_specs
+    @sites
+  end
+
+  # Determines all listings which can be construed by combining the sites specified in the object
+  # constructor with the provided url-path fragments.
+  #
+  # Passes the <b>first page listing</b> of each of these urls to the provided block.
+  def each_listing(*fragments)
+    listing_urls_for(fragments).each{|url| yield Listings.new(url) }
+  end
+
+  # Determines all listings which can be construed by combining the sites specified in the object
+  # constructor with the provided url-path fragments.
+  #
+  # Passes <b>each page on every listing</b> for the passed URLs to the provided block.
+  def each_page_in_each_listing(*fragments)
+    each_listing(*fragments) do |listing|
+      while listing
+        yield listing
+        listing = listing.next_page
+      end
+    end
+  end
+
+  # Determines all listings which can be construed by combining the sites specified in the object
+  # constructor with the provided url-path fragments.
+  #
+  # Returns the <b>first page listing</b> of each of these urls to the provided block.
+  def listings(*fragments)
+    listing_urls_for(fragments).collect{|url| Listings.new url }
+  end
+
+  # Determines all listings which can be construed by combining the sites specified in the object
+  # constructor with the provided url-path fragments.
+  #
+  # Passes all posts from each of these urls to the provided block, in the order they're parsed
+  # (for each listing, newest posts are returned first).
+  def each_post(*fragments)
+    each_page_in_each_listing(*fragments){ |l| l.posts.each{|p| yield p} }
+  end
+
+  # Determines all listings which can be construed by combining the sites specified in the object
+  # constructor with the provided url-path fragments.
+  #
+  # Returns all posts from each of these urls, in the order they're parsed
+  # (newest posts first).
+  def posts(*fragments)
+    ret = []
+    each_page_in_each_listing(*fragments){ |l| ret += l.posts }
+    ret
+  end
+
+  # Determines all listings which can be construed by combining the sites specified in the object
+  # constructor with the provided url-path fragments.
+  #
+  # Returns all posts from each of these urls, which are newer than the provider 'newer_then' date.
+  # (Returns 'newest' posts first).
+  def posts_since(newer_then, *fragments)
+    ret = []
+    fragments.each do |frag|
+      each_post(frag) do |p|
+        break if p.post_date <= newer_then
+        ret << p
+      end
+    end
+
+    ret
+  end
+
+  class << self # Class methods
+
+    #--
+    # NOTE: These Class methods are all marked for deprecation as of
+    # version 0.8.0, and should not be used with any new project code
+    #++
+
+    # <b>This method is for legacy compatibility and is not recommended for use by new projects.</b>
+    # Instead, consider using CraigScrape::Listings.new
+    #
+    # Scrapes a single listing url and returns a Listings object representing the contents.
+    # Mostly here to preserve backwards-compatibility with the older api, CraigScrape::Listings.new "listing_url" does the same thing
+    def scrape_listing(listing_url)
+      CraigScrape::Listings.new listing_url
+    end
+
+    # <b>This method is for legacy compatibility and is not recommended for use by new projects.</b>
+    # Instead, consider using the CraigScrape::each_post method.
+    #
+    # Continually scrapes listings, using the supplied url as a starting point, until the supplied block returns true or
+    # until there's no more 'next page' links available to click on
+    def scrape_until(listing_url, &post_condition)
+      ret = []
+
+      listings = CraigScrape::Listings.new listing_url
+      catch "ScrapeBreak" do
+        while listings do
+          listings.posts.each do |post|
+            throw "ScrapeBreak" if post_condition.call(post)
+            ret << post
+          end
+
+          listings = listings.next_page
+        end
+      end
+
+      ret
+    end
+
+    # <b>This method is for legacy compatibility and is not recommended for use by new projects.</b>
+    # Instead, consider using CraigScrape::Posting.new
+    #
+    # Scrapes a single Post Url, and returns a Posting object representing its contents.
+    # Mostly here to preserve backwards-compatibility with the older api, CraigScrape::Listings.new "listing_url" does the same thing
+    def scrape_full_post(post_url)
+      CraigScrape::Posting.new post_url
+    end
+
+    # <b>This method is for legacy compatibility and is not recommended for use by new projects.</b>
+    # Instead,  consider using the CraigScrape::each_post method.
+    #
+    # Continually scrapes listings, using the supplied url as a starting point, until 'count' summaries have been retrieved
+    # or no more 'next page' links are avialable to be clicked on. Returns an array of PostSummary objects.
+    def scrape_posts(listing_url, count)
+      count_so_far = 0
+      self.scrape_until(listing_url) {|post| count_so_far+=1; count < count_so_far }
+    end
+
+    # <b>This method is for legacy compatibility and is not recommended for use by new projects.</b>
+    # Instead, consider using the CraigScrape::posts_since method.
+    #
+    # Continually scrapes listings, until the date newer_then has been reached, or no more 'next page' links are avialable to be clicked on.
+    # Returns an array of PostSummary objects. Dates are based on the Month/Day 'datestamps' reported in the listing summaries.
+    # As such, time-based cutoffs are not supported here. The scrape_until method, utilizing the SummaryPost.full_post method could achieve
+    # time-based cutoffs, at the expense of retrieving every post in full during enumerations.
+    #
+    # <b>Note:</b> The results will not include post summaries having the newer_then date themselves.
+    def scrape_posts_since(listing_url, newer_then)
+      self.scrape_until(listing_url) {|post| post.post_date <= newer_then}
+    end
+  end
+
+  private
+
+  # This  takes a fragments paramter, and turns it into actual urls
+  def listing_urls_for(listing_fragments)
+    listing_fragments.collect{ |lf|
+      # This removes any /'s from he beginning of the fragment
+      lf = $1 if /^\/(.*)/.match lf
+      # This adds a '/' to the end of a path, so long as its not a query we're dealing with...
+      lf += '/' unless lf.index '?'
+      sites.collect { |site| '%s%s/%s' % [site_to_url_prefix,site,lf] }
+    }.flatten
+  end
+
+  # Returns the most recentlt expired  time for the provided month and day
+  def self.most_recently_expired_time(month, day)  #:nodoc:
+    now = (time_now) ? time_now : Time.now
+
+    # This ensures we always generate a time in the past, by guessing the year and subtracting one if we guessed wrong
+    ret = Time.local now.year, month, day
+    ret = Time.local now.year-1, month, day if ret > now
+
+    ret
+  end
+
+end
+
+require 'listings'
+require 'posting'
+require 'geo_listings'

data/lib/listings.rb

@@ -0,0 +1,160 @@
+# = About listings.rb
+#
+# This file contains the parsing code, and logic relating to post-listing pages. You
+# should never need to include this file directly, as all of libcraigscrape's objects and methods
+# are loaded when you use <tt>require 'libcraigscrape'</tt> in your code.
+#
+require 'scraper'
+
+# Listings represents a parsed Craigslist listing page and is generally returned by CraigScrape.scrape_listing
+class CraigScrape::Listings < CraigScrape::Scraper
+  LABEL          = /^(.+?)[ ]*[\-]?$/
+  LOCATION       = /^[ ]*\((.*?)\)$/
+  IMG_TYPE       = /^[ ]*(.+)[ ]*$/
+  HEADER_DATE    = /^[ ]*(?:Sun|Mon|Tue|Wed|Thu|Fri|Sat)[ ]+(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Nov|Dec)[ ]+([0-9]{1,2})[ ]*$/i
+  SUMMARY_DATE   = /^[ ]([^ ]+)[ ]+([^ ]+)[ ]*[\-][ ]*$/
+  NEXT_PAGE_LINK = /^[ ]*next [\d]+ postings[ ]*$/
+
+  # Array, PostSummary objects found in the listing
+  def posts
+    unless @posts
+      current_date = nil
+      @posts = []
+
+      # All we care about are p and h4 tags. This seemed to be the only way I could do this on Nokogiri:
+      post_tags = html.search('*').reject{|n| !/^(?:p|h4)$/i.match n.name }
+
+      # The last p in the list is sometimes a 'next XXX pages' link. We don't want to include this in our PostSummary output:
+      post_tags.pop if (
+        post_tags.length > 0 and
+        post_tags.last.at('a') and
+        NEXT_PAGE_LINK.match post_tags.last.at('a').inner_html
+      )
+
+      # Now we iterate though the listings:
+      post_tags.each do |el|
+        case el.name
+          when 'p'
+           post_summary = self.class.parse_summary el, current_date
+
+           # Validate that required fields are present:
+           parse_error! unless [post_summary[:label],post_summary[:href]].all?{|f| f and f.length > 0}
+
+           post_summary[:url] = url_from_href post_summary[:href]
+
+           @posts << CraigScrape::Posting.new(post_summary)
+         when 'h4'
+          # Let's make sense of the h4 tag, and then read all the p tags below it
+          if HEADER_DATE.match he_decode(el.inner_html)
+            # Generally, the H4 tags contain valid dates. When they do - this is easy:
+            current_date = CraigScrape.most_recently_expired_time $1, $2
+          elsif html.at('h4:last-of-type') == el
+            # There's a specific bug in craigslist, where these nonsense h4's just appear without anything relevant inside them.
+            # They're safe to ignore if they're not the last h4 on the page. I fthey're the last h4 on the page,
+            # we need to pull up the full post in order to accurate tell the date.
+            # Setting this to nil will achieve the eager-load.
+            current_date = nil
+          end
+        end
+      end
+    end
+
+    @posts
+  end
+
+  # String, URL Path href-fragment of the next page link
+  def next_page_href
+    unless @next_page_href
+      cursor = html.at 'p:last-of-type'
+
+      cursor = cursor.at 'a' if cursor
+
+      # Category Listings have their 'next 100 postings' link at the end of the doc in a p tag
+      next_link = cursor if cursor and NEXT_PAGE_LINK.match cursor.inner_html
+
+      # Search listings put their next page in a link towards the top
+      next_link = (html / 'a').find{ |a| he_decode(a.inner_html) == '<b>Next>></b>' } unless next_link
+
+      # Some search pages have a bug, whereby a 'next page' link isn't displayed,
+      # even though we can see that theres another page listed in the page-number links block at the top
+      # and bottom of the listing page
+      unless next_link
+        cursor = html % 'div.sh:first-of-type > b:last-of-type'
+
+        # If there's no 'a' in the next sibling, we'll have just performed a nil assignment, otherwise
+        # We're looking good.
+        next_link = cursor.next_element if cursor and /^[\d]+$/.match cursor.inner_html
+      end
+
+      # We have an anchor tag - so - let's assign the href:
+      @next_page_href = next_link[:href] if next_link
+    end
+
+    @next_page_href
+  end
+
+  # String, Full URL Path of the 'next page' link
+  def next_page_url
+    (next_page_href) ? url_from_href(next_page_href) : nil
+  end
+
+  # Returns a Listings object of the next_page_url on the current listings object
+  def next_page
+    CraigScrape::Listings.new next_page_url if next_page_url
+  end
+
+  # Takes a paragraph element and returns a mostly-parsed Posting
+  # We separate this from the rest of the parsing both for readability and ease of testing
+  def self.parse_summary(p_element, date = nil)  #:nodoc:
+    ret = {}
+
+    title_anchor   = nil
+    section_anchor = nil
+
+    # This loop got a little more complicated after Craigslist start inserting weird <spans>'s in
+    # its list summary postings (See test_new_listing_span051710)
+    p_element.search('a').each do |a_el|
+      # We want the first a-tag that doesn't have spans in it to be the title anchor
+      if title_anchor.nil?
+        title_anchor = a_el if !a_el.at('span')
+      # We want the next a-tag after the title_anchor to be the section anchor
+      elsif section_anchor.nil?
+        section_anchor = a_el
+        # We have no need to tranverse these further:
+        break
+      end
+    end
+
+    location_tag = p_element.at 'font'
+    has_pic_tag = p_element.at 'span'
+
+    href = nil
+
+    location = he_decode p_element.at('font').inner_html if location_tag
+    ret[:location] = $1 if location and LOCATION.match location
+
+    ret[:img_types] = []
+    if has_pic_tag
+      img_type = he_decode has_pic_tag.inner_html
+      img_type = $1.tr('^a-zA-Z0-9',' ') if IMG_TYPE.match img_type
+
+      ret[:img_types] = img_type.split(' ').collect{|t| t.to_sym}
+    end
+
+    ret[:section] = he_decode(section_anchor.inner_html).split("\302\240").join(" ") if section_anchor
+
+    ret[:post_date] = date
+    if SUMMARY_DATE.match he_decode(p_element.children[0])
+      ret[:post_date] = CraigScrape.most_recently_expired_time $1, $2.to_i
+    end
+
+    if title_anchor
+      label = he_decode title_anchor.inner_html
+      ret[:label] = $1 if LABEL.match label
+
+      ret[:href] = title_anchor[:href]
+    end
+
+    ret
+  end
+end