quixoten-craigler 1.0.0 → 1.1.0

data/LICENSE

@@ -1,3 +1,5 @@
+(MIT License)
+
 Copyright (c) 2009 Devin Christensen
 
 Permission is hereby granted, free of charge, to any person obtaining

data/README.rdoc

@@ -1,21 +1,40 @@
-= craigler
+= Craigler
 
 Search API for craigslist
 
 == SYNOPSIS:
+I couldn't decide which API I liked best, so you have two choices.
   
-  Craigler.search(:motorcycles, :in => [:utah, :nevada, :arizona], :for => 'Boulevard M50) do |item|
+  Craigler.search(:motorcycles, :in => [:utah, :nevada, :arizona], :for => 'Boulevard M50') do |item|
     puts item.title
     puts item.url
     puts item.time
   end
+
+or
   
-  Craigler.find('Boulevard M50', :in => [:utah, :nevada, :arizona], :only => :motorcycles) do |item|
+  Craigler.find('Boulevard M50', :in => :california, :only => :motorcycles) do |item|
     puts item.title
     puts item.url
     puts item.time
   end
 
+You can also create a search object to fetch the results later. When no location or category is given Craigler searches <tt>:all_for_sale_or_wanted</tt> in <tt>:anywhere</tt>
+
+  search = Craigler::Search.new('Yamaha')
+  search.results(:page_limit => 1)
+  
+Note that additional calls to <tt>search.results()</tt> will always return the same result set unless refresh is forced
+  search.results(:refresh => true)
+
+=== Supported Categories
+<tt>:all_for_sale_or_wanted</tt>, <tt>:art_and_crafts</tt>, <tt>:auto_parts</tt>, <tt>:baby_and_kid_stuff</tt>, <tt>:barter</tt>, <tt>:bicycles</tt>, <tt>:boats</tt>, <tt>:books</tt>, <tt>:business</tt>, <tt>:cars_and_trucks</tt>, <tt>:clothing</tt>, <tt>:collectibles</tt>, <tt>:community</tt>, <tt>:computers_and_tech</tt>, <tt>:electronics</tt>, <tt>:event</tt>, <tt>:farm_and_garden</tt>, <tt>:free_stuff</tt>, <tt>:furniture</tt>, <tt>:games_and_toys</tt>, <tt>:garage_sales</tt>, <tt>:general</tt>, <tt>:gigs</tt>, <tt>:household</tt>, <tt>:housing</tt>, <tt>:items_wanted</tt>, <tt>:jewelry</tt>, <tt>:jobs</tt>, <tt>:materials</tt>, <tt>:media</tt>, <tt>:motorcycles</tt>, <tt>:musical_instruments</tt>, <tt>:personals</tt>, <tt>:photo_and_video</tt>, <tt>:recreational_vehicles</tt>, <tt>:resume</tt>, <tt>:services_offered</tt>, <tt>:sporting_goods</tt>, <tt>:tickets</tt>, <tt>:tools</tt>
+
+=== Supported Locations
+<tt>:alaska</tt>, <tt>:arizona</tt>, <tt>:arkansas</tt>, <tt>:california</tt>, <tt>:colorado</tt>, <tt>:connecticut</tt>, <tt>:delaware</tt>, <tt>:dc</tt>, <tt>:florida</tt>, <tt>:georgia</tt>, <tt>:hawaii</tt>, <tt>:idaho</tt>, <tt>:illinois</tt>, <tt>:indiana</tt>, <tt>:iowa</tt>, <tt>:kansas</tt>, <tt>:kentucky</tt>, <tt>:louisiana</tt>, <tt>:maine</tt>, <tt>:maryland</tt>, <tt>:mass</tt>, <tt>:michigan</tt>, <tt>:minnesota</tt>, <tt>:mississippi</tt>, <tt>:missouri</tt>, <tt>:montana</tt>, <tt>:nebraska</tt>, <tt>:nevada</tt>, <tt>:n_hampshire</tt>, <tt>:new_jersey</tt>, <tt>:new_mexico</tt>, <tt>:new_york</tt>, <tt>:n_carolina</tt>, <tt>:north_dakota</tt>, <tt>:ohio</tt>, <tt>:oklahoma</tt>, <tt>:oregon</tt>, <tt>:pennsylvania</tt>, <tt>:rhode_island</tt>, <tt>:s_carolina</tt>, <tt>:south_dakota</tt>, <tt>:tennessee</tt>, <tt>:texas</tt>, <tt>:utah</tt>, <tt>:vermont</tt>, <tt>:virginia</tt>, <tt>:washington</tt>, <tt>:west_virginia</tt> <tt>:wisconsin</tt>, <tt>:wyoming</tt>
+
+Or use <tt>:anywhere</tt> to search all supported locations.
+
 == REQUIREMENTS:
 
 * Hpricot

data/Rakefile

@@ -49,8 +49,8 @@ Rake::RDocTask.new do |rdoc|
   end
 
   rdoc.rdoc_dir = 'rdoc'
-  rdoc.title = "craigler #{version}"
-  rdoc.rdoc_files.include('README*')
+  rdoc.title = "Craigler #{version}"
+  rdoc.rdoc_files.include('README.rdoc', 'LICENSE')
   rdoc.rdoc_files.include('lib/**/*.rb')
 end
 

data/VERSION.yml

@@ -1,4 +1,4 @@
----
+--- 
 :major: 1
-:minor: 0
+:minor: 1
 :patch: 0

data/craigler.gemspec

@@ -1,12 +1,15 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE
+# Instead, edit Jeweler::Tasks in Rakefile, and run `rake gemspec`
 # -*- encoding: utf-8 -*-
 
 Gem::Specification.new do |s|
   s.name = %q{craigler}
-  s.version = "1.0.0"
+  s.version = "1.1.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Devin Christensen"]
-  s.date = %q{2009-07-06}
+  s.date = %q{2009-08-11}
   s.email = %q{quixoten@gmail.com}
   s.extra_rdoc_files = [
     "LICENSE",
@@ -30,7 +33,7 @@ Gem::Specification.new do |s|
   s.homepage = %q{http://github.com/quixoten/craigler}
   s.rdoc_options = ["--charset=UTF-8"]
   s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.4}
+  s.rubygems_version = %q{1.3.5}
   s.summary = %q{Search API for craigslist}
   s.test_files = [
     "test/craigler_search_test.rb",

data/lib/craigler.rb

@@ -5,20 +5,26 @@ require 'craigler/constants'
 require 'craigler/search'
 
 module Craigler
-  VERSION = '0.1.0'
-  
   class CraiglerError < StandardError; end
   class InvalidCategory < CraiglerError; end
   class InvalidSearchTerm < CraiglerError; end
   class InvalidLocation < CraiglerError; end
   
   class << self
+    # Interface to Search that may or may not be more readable
     def search(category, options = {})
-      search = Search.new(options[:for], :in => (options[:in] || :anywhere), :only => category)
+      results = Search.new(options[:for], :in => (options[:in] || :anywhere), :only => category).results()
+      results.each {|result| yield(result) } if block_given?
+      results
     end
     
+    # Interface to Search that somewhat mimics ActiveRecord#find
+    #
+    # Supports all the options of Search#new
     def find(search_term, options = {})
-      search = Search.new(search_term, options)
+      results = Search.new(search_term, options).results()
+      results.each {|result| yield(result) } if block_given?
+      results
     end
   end
 end

data/lib/craigler/constants.rb

@@ -1,4 +1,6 @@
 module Craigler
+  RESULTS_PER_PAGE = 25 # :nodoc:
+  
   LOCATIONS = {
     :alabama => ['http://auburn.craigslist.org/','http://bham.craigslist.org/','http://columbusga.craigslist.org/','http://dothan.craigslist.org/','http://shoals.craigslist.org/','http://gadsden.craigslist.org/','http://huntsville.craigslist.org/','http://mobile.craigslist.org/','http://montgomery.craigslist.org/','http://tuscaloosa.craigslist.org/'],
     :alaska => ['http://anchorage.craigslist.org/'],
@@ -51,7 +53,7 @@ module Craigler
     :west_virginia => ['http://charlestonwv.craigslist.org/','http://huntington.craigslist.org/','http://martinsburg.craigslist.org/','http://morgantown.craigslist.org/','http://parkersburg.craigslist.org/','http://wv.craigslist.org/','http://wheeling.craigslist.org/'],
     :wisconsin => ['http://appleton.craigslist.org/','http://duluth.craigslist.org/','http://eauclaire.craigslist.org/','http://greenbay.craigslist.org/','http://janesville.craigslist.org/','http://racine.craigslist.org/','http://lacrosse.craigslist.org/','http://madison.craigslist.org/','http://milwaukee.craigslist.org/','http://sheboygan.craigslist.org/','http://wausau.craigslist.org/'],
     :wyoming => ['http://wyoming.craigslist.org/']
-  }
+  } # :nodoc:
 
   CATEGORIES = {
     :community => 'ccc',
@@ -94,5 +96,5 @@ module Craigler
     :personals => 'ppp',
     :resume => 'res',
     :services_offered => 'bbb'
-  }
+  } # :nodoc:
 end

data/lib/craigler/search.rb

@@ -6,6 +6,13 @@ module Craigler
     
     attr_reader :search_term, :categories, :locations
     
+    # Creates a wrapper object for a craigslist search
+    #
+    # === Options
+    # [:in]
+    #   Specifies the location(s) to search in. Defaults to <tt>:anywhere</tt>.
+    # [:only]
+    #   Specifies the category or categories to search in. Defaults to <tt>:all_for_sale_or_wanted</tt>
     def initialize(search_term, options = {})
       raise InvalidSearchTerm if search_term.nil? || search_term == ''
       
@@ -14,15 +21,30 @@ module Craigler
       _parse_options(options)
     end
     
-    def results(refresh = false)
-      return @results unless @results.nil? || refresh
+    # Returns the results of the search. If this is the first time
+    # calling #results then they will be fetched over the internet and cached in the search object.
+    #
+    # === Options
+    # [:page_limit]
+    #   Maximum number of pages to fetch results from. Defaults to <tt>4</tt>.
+    #   <b>Note:</b> A location may, and often does, have more than one searchable
+    #   url assciated with it, e.g., {California}[http://geo.craigslist.org/iso/us/ca]. Because
+    #   <tt>:page_limit</tt> is applied seperately to each url within the location, searching <tt>:in => :california</tt>
+    #   with a <tt>:page_limit => 4</tt> could potentially make up to 100 page requests.</em>
+    # [:refresh]
+    #   Set to <tt>true</tt> to force an update across the internet.
+    def results(options = {})
+      options = { :page_limit => 4, :refresh => false }.merge(options)
+      return @results unless @results.nil? || options[:refresh]
+      
+      @results  = []
+      last_page = options[:page_limit] - 1 # pages start at 0
       
-      @results = []
       _for_each_locations_search_url() do |location, url|
-        (0..19).each do |page|
-          items = _extract_items_from_url(location, "#{url}&s=#{page*25}")
-          @results.push(*items)
-          break unless items.size == 25
+        (0..last_page).each do |page|
+          results = _extract_items_from_url(location, "#{url}&s=#{page*25}")
+          @results.push(*results)
+          break if results.size < RESULTS_PER_PAGE
         end
       end
       

data/test/craigler_search_test.rb

@@ -30,6 +30,12 @@ class CraiglerSearchTest < Test::Unit::TestCase
       end
     end
     
+    should "use a default category of :all_for_sale_or_wanted" do
+      search = Craigler::Search.new('Buell', :in => :utah)
+      assert(search.categories == [:all_for_sale_or_wanted],
+              "category was [:#{search.categories.join(", :")}] but should have been [:all_for_sale_or_wanted]")
+    end
+    
     should "require that the location is valid" do
       assert_raises(Craigler::InvalidLocation) do
         Craigler::Search.new('Honda Nighthawk', :in => :invalid)
@@ -45,7 +51,7 @@ class CraiglerSearchTest < Test::Unit::TestCase
   
   context "fetching search results" do
     setup do
-      @search = Craigler::Search.new('Honda Shadow', :in => :utah, :only => :motorcycles)
+      @search = Craigler::Search.new('Honda', :in => :utah, :only => :motorcycles, :page_limit => 1)
     end
     
     should "return an array of hashes" do
@@ -54,5 +60,11 @@ class CraiglerSearchTest < Test::Unit::TestCase
       assert(results.size > 0, "No results were returned")
       assert(results.inject(true) {|t,r| t && r.is_a?(Hash)})
     end
+    
+    should "allow us to limit the number of pages searched" do
+      one_page_count  = @search.results(:page_limit => 1).size
+      two_page_count  = @search.results(:page_limit => 2, :refresh => true).size
+      assert(one_page_count < two_page_count, "#{one_page_count} is not less than #{two_page_count}")
+    end
   end
 end

data/test/craigler_test.rb

@@ -1,11 +1,33 @@
 require 'test_helper'
 
 class CraiglerTest < Test::Unit::TestCase
-  should "respond to search" do
-    assert_respond_to(Craigler, :search)
+  context "Search interface" do
+    should "respond to search" do
+      assert_respond_to(Craigler, :search)
+    end
+    
+    should "yield results to a block if given" do
+      yielded = false 
+      Craigler::search(:motorcycles, :in => :utah, :for => 'Yamaha', :page_limit => 1) do
+        yielded = true; break;
+      end
+      
+      assert(yielded, "results were never yielded to the block")
+    end
   end
   
-  should "respond to find" do
-    assert_respond_to(Craigler, :find)
+  context "Find interface" do
+    should "respond to find" do
+      assert_respond_to(Craigler, :find)
+    end
+    
+    should "yield results to a block if given" do
+      yielded = false 
+      Craigler::find('Yamaha', :in => :utah, :only => :motorcycles, :page_limit => 1) do
+        yielded = true; break;
+      end
+      
+      assert(yielded, "results were never yielded to the block")
+    end
   end
 end

data/test/test_helper.rb

@@ -1,10 +1,17 @@
 require 'rubygems'
 require 'test/unit'
 require 'shoulda'
+require 'ruby-debug'
 
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
+
 require 'craigler'
 
-class Test::Unit::TestCase
+module Craigler
+  class Search
+    def open(*args)
+      @@results_page ||= Kernel::open("http://saltlakecity.craigslist.org/search/sss?query=Honda&format=rss&s=0").read()
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: quixoten-craigler
 version: !ruby/object:Gem::Version 
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors: 
 - Devin Christensen
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-07-06 00:00:00 -07:00
+date: 2009-08-11 00:00:00 -07:00
 default_executable: 
 dependencies: []
 
@@ -38,6 +38,7 @@ files:
 - test/test_helper.rb
 has_rdoc: false
 homepage: http://github.com/quixoten/craigler
+licenses: 
 post_install_message: 
 rdoc_options: 
 - --charset=UTF-8
@@ -58,7 +59,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.2.0
+rubygems_version: 1.3.5
 signing_key: 
 specification_version: 3
 summary: Search API for craigslist