crags 1.6.0 → 2.0.0

data/.gitignore

@@ -1,2 +1,4 @@
 pkg/
-.DS_Store
+.DS_Store
+.bundle
+coverage

data/Gemfile

@@ -0,0 +1,8 @@
+source "http://rubygems.org"
+
+gem "curb"
+gem "hpricot"
+
+group :test do
+  gem "rspec"
+end

data/README.rdoc

@@ -4,18 +4,55 @@ A library to help search across multiple craigslist locations.
 
 == Usage
 
-searches all available craigslist sites for a keyword in "for sale".
- search(keyword)
+There are two ways to search craigslist using crags. You can search by keyword
+in a single location scoped to a category. You can also search by keyword
+across a country scoped by category. Each search type also has a set of
+configurable options.
 
-search also takes a category
- search(keyword, 'sss')
+=== Single Location Search
 
-There are a bunch of helpful api methods, check the tests for more info.
+Options:
+  :keyword
+    default: bicycle (because bicycles are awesome)
+  :location
+    default: sfbay.craigslist.com (because that is where it all began)
+  :category
+    default: for sale (because buying another bike is always a good idea)
 
-== Requirements
+Locations and Categories are full-on Ruby objects. You can get all of the
+Categories using:
+  Category.all
 
-* hpricot
-* curb
+Search a single craigslist location (e.g. sfbay.craigslist.com)
+  s = Search::Location.new(:keyword => 'shoes')
+  s.keyword     # this will shoes instead of the default
+  s.location    # this will the sfbay Location
+  s.category    # the for sale Category
+  s.items       # will return a set of Items
+
+Each Item has a title, url (which is the link to the item on craigslist), and
+a date, which is the date the item was posted on craigslist.
+
+=== Country Search
+
+Options:
+  :keyword
+    default: bicycle (because bicycles are awesome)
+  :category
+    default: for sale (because buying another bike is always a good idea)
+  :country
+    default: us (because that is where I live)
+  :interval
+    default: 1 (because hammering the cl servers will make em' angry)
+
+Country searches are a combined set of individual location searches. Crags
+imposes a 1 second delay between each location search in order not to overload
+the Craigslist servers. You can change this at your own risk, but doing a lot
+of country wide searches with a 0s interval will probably get your IP banned.
+
+== Installation
+
+  gem install crags
 
 == Note on Patches/Pull Requests 
 

data/Rakefile

@@ -1,4 +1,3 @@
-require 'rubygems'
 require 'rake'
 
 begin
@@ -10,43 +9,34 @@ begin
     gem.email = "gotascii@gmail.com"
     gem.homepage = "http://github.com/gotascii/crags"
     gem.authors = ["Justin Marney"]
+    gem.add_dependency('curb')
+    gem.add_dependency('hpricot')
   end
   Jeweler::GemcutterTasks.new
 rescue LoadError
   puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
 end
 
-require 'rake/testtask'
-Rake::TestTask.new(:test) do |test|
-  test.libs << 'lib' << 'test'
-  test.pattern = 'test/**/*_test.rb'
-  test.verbose = true
+require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.spec_files = FileList['spec/**/*_spec.rb']
 end
 
-begin
-  require 'rcov/rcovtask'
-  Rcov::RcovTask.new do |test|
-    test.libs << 'test'
-    test.pattern = 'test/**/*_test.rb'
-    test.verbose = true
-  end
-rescue LoadError
-  task :rcov do
-    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
-  end
+Spec::Rake::SpecTask.new(:rcov) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+  spec.rcov_opts = ['--exclude', 'spec,gems,crags.rb']
 end
 
-task :test => :check_dependencies
+task :spec => :check_dependencies
 
-task :default => :test
+task :default => :spec
 
 require 'rake/rdoctask'
 Rake::RDocTask.new do |rdoc|
-  if File.exist?('VERSION')
-    version = File.read('VERSION')
-  else
-    version = ""
-  end
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
 
   rdoc.rdoc_dir = 'rdoc'
   rdoc.title = "crags #{version}"

data/VERSION

@@ -1 +1 @@
-1.6.0
+2.0.0

data/crags.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{crags}
-  s.version = "1.6.0"
+  s.version = "2.0.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Justin Marney"]
-  s.date = %q{2010-04-14}
+  s.date = %q{2010-07-26}
   s.description = %q{A library to help search across multiple craigslist locations.}
   s.email = %q{gotascii@gmail.com}
   s.extra_rdoc_files = [
@@ -18,21 +18,34 @@ Gem::Specification.new do |s|
   ]
   s.files = [
     ".gitignore",
+     "Gemfile",
      "LICENSE",
      "README.rdoc",
      "Rakefile",
      "VERSION",
      "crags.gemspec",
      "lib/crags.rb",
-     "lib/crags/fetch.rb",
-     "lib/crags/proxy.rb",
-     "lib/crags/runner.rb",
-     "lib/crags/searcher.rb",
-     "test/crags/fetch_test.rb",
-     "test/crags/proxy_test.rb",
-     "test/crags/runner_test.rb",
-     "test/crags/searcher_test.rb",
-     "test/test_helper.rb"
+     "lib/crags/category.rb",
+     "lib/crags/country.rb",
+     "lib/crags/fetcher.rb",
+     "lib/crags/item.rb",
+     "lib/crags/location.rb",
+     "lib/crags/search/country.rb",
+     "lib/crags/search/location.rb",
+     "lib/crags/search/search.rb",
+     "lib/ext/hpricot/elem.rb",
+     "lib/ext/string.rb",
+     "spec/crags/category_spec.rb",
+     "spec/crags/country_spec.rb",
+     "spec/crags/fetcher_spec.rb",
+     "spec/crags/item_spec.rb",
+     "spec/crags/location_spec.rb",
+     "spec/crags/search/country_spec.rb",
+     "spec/crags/search/location_spec.rb",
+     "spec/crags/search/search_spec.rb",
+     "spec/ext/hpricot/elem_spec.rb",
+     "spec/ext/string_spec.rb",
+     "spec/spec_helper.rb"
   ]
   s.homepage = %q{http://github.com/gotascii/crags}
   s.rdoc_options = ["--charset=UTF-8"]
@@ -40,11 +53,17 @@ Gem::Specification.new do |s|
   s.rubygems_version = %q{1.3.6}
   s.summary = %q{A library to help search across multiple craigslist locations.}
   s.test_files = [
-    "test/crags/fetch_test.rb",
-     "test/crags/proxy_test.rb",
-     "test/crags/runner_test.rb",
-     "test/crags/searcher_test.rb",
-     "test/test_helper.rb"
+    "spec/crags/category_spec.rb",
+     "spec/crags/country_spec.rb",
+     "spec/crags/fetcher_spec.rb",
+     "spec/crags/item_spec.rb",
+     "spec/crags/location_spec.rb",
+     "spec/crags/search/country_spec.rb",
+     "spec/crags/search/location_spec.rb",
+     "spec/crags/search/search_spec.rb",
+     "spec/ext/hpricot/elem_spec.rb",
+     "spec/ext/string_spec.rb",
+     "spec/spec_helper.rb"
   ]
 
   if s.respond_to? :specification_version then
@@ -52,9 +71,15 @@ Gem::Specification.new do |s|
     s.specification_version = 3
 
     if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<curb>, [">= 0"])
+      s.add_runtime_dependency(%q<hpricot>, [">= 0"])
     else
+      s.add_dependency(%q<curb>, [">= 0"])
+      s.add_dependency(%q<hpricot>, [">= 0"])
     end
   else
+    s.add_dependency(%q<curb>, [">= 0"])
+    s.add_dependency(%q<hpricot>, [">= 0"])
   end
 end
 

data/lib/crags/category.rb

@@ -0,0 +1,29 @@
+module Crags
+  class Category
+    extend Fetcher
+    attr_reader :name, :url
+
+    def initialize(name, abbr)
+      @name = name
+      @url = "/#{abbr}"
+    end
+
+    def self.doc
+      fetch_doc(Config.category_url)
+    end
+
+    def self.links
+      doc.search("div.col a").select do |link|
+        (link["href"] =~ /forum/).nil?
+      end
+    end
+
+    def self.all
+      links.collect do |link|
+        url = link["href"]
+        name = link.inner_html
+        Category.new(name, url)
+      end
+    end
+  end
+end

data/lib/crags/country.rb

@@ -0,0 +1,37 @@
+module Crags
+  class Country
+    include Fetcher
+    attr_reader :code
+
+    def initialize(code)
+      @code = code
+    end
+
+    def url
+      "#{Config.country_url}/#{code}"
+    end
+
+    def doc
+      fetch_doc(url)
+    end
+
+    def request
+      fetch_request(url)
+    end
+
+    def links
+      doc.search("#list a")
+    end
+
+    def locations
+      if links.empty?
+        str = request.last_effective_url
+        [Location.new(str.strip_http)]
+      else
+        links.collect do |link|
+          Location.new(link["href"].strip_http)
+        end
+      end
+    end
+  end
+end

data/lib/crags/{fetch.rb → fetcher.rb}

@@ -1,5 +1,5 @@
 module Crags
-  module Fetch
+  module Fetcher
     def fetch_doc(url)
       Hpricot.parse(fetch_html(url))
     end

data/lib/crags/item.rb

@@ -0,0 +1,11 @@
+module Crags
+  class Item
+    attr_reader :title, :url, :date
+
+    def initialize(elem)
+      @title = elem.title
+      @url = elem.url
+      @date = elem.date
+    end
+  end
+end

data/lib/crags/location.rb

@@ -0,0 +1,13 @@
+module Crags
+  class Location
+    attr_reader :domain
+
+    def initialize(domain)
+      @domain = domain
+    end
+
+    def url
+      "http://#{domain}"
+    end
+  end
+end

data/lib/crags/search/country.rb

@@ -0,0 +1,26 @@
+module Crags
+  module Search
+    class Country < Search
+      attr_reader :country, :interval
+
+      def initialize(opts = {})
+        super
+        @country = @opts[:country]
+        @interval = @opts[:interval]
+      end
+
+      def locations
+        country.locations
+      end
+
+      def items
+        combined_items = locations.collect do |loc|
+          sleep(interval)
+          search = Location.new(opts.merge(:location => loc))
+          search.items
+        end
+        combined_items.flatten
+      end
+    end
+  end
+end

data/lib/crags/search/location.rb

@@ -0,0 +1,28 @@
+module Crags
+  module Search
+    class Location < Search
+      include ERB::Util
+      include Fetcher
+      attr_reader :location
+
+      def initialize(opts = {})
+        super(opts)
+        @location = @opts[:location]
+      end
+
+      def url
+        "#{location.url}/search#{category.url}?query=#{url_encode(keyword)}"
+      end
+
+      def doc
+        fetch_doc("#{url}&format=rss")
+      end
+
+      def items
+        doc.search("item").collect do |elem|
+          Item.new(elem)
+        end
+      end
+    end
+  end
+end

data/lib/crags/search/search.rb

@@ -0,0 +1,13 @@
+module Crags
+  module Search
+    class Search
+      attr_reader :keyword, :category, :opts
+
+      def initialize(opts = {})
+        @opts = Config.defaults.merge(opts)
+        @keyword = @opts[:keyword]
+        @category = @opts[:category]
+      end
+    end
+  end
+end

data/lib/crags.rb

@@ -1,42 +1,29 @@
-require 'curb'
-require 'hpricot'
+require 'bundler'
+Bundler.require
 require 'erb'
+require 'ostruct'
+require 'ext/string'
+require 'ext/hpricot/elem'
+require 'crags/fetcher'
+require 'crags/country'
+require 'crags/location'
+require 'crags/category'
+require 'crags/item'
+require 'crags/search/search'
+require 'crags/search/location'
+require 'crags/search/country'
 
 module Crags
-  COUNTRIES = [
-    'jp',
-    'ar',
-    'bd',
-    'br',
-    'ca',
-    'cl',
-    'co',
-    'cr',
-    'cz',
-    'de',
-    'eg',
-    'gb',
-    'hu',
-    'id',
-    'ie',
-    'il',
-    'lb',
-    'my',
-    'nl',
-    'nz',
-    'no',
-    'pk',
-    'pa',
-    'ru',
-    'th',
-    'ae',
-    'us',
-    've',
-    'vn'
-  ]
-end
-
-require 'crags/fetch'
-require 'crags/proxy'
-require 'crags/searcher'
-require 'crags/runner'
+  Config = OpenStruct.new({
+    :defaults => {
+      :keyword => 'bicycle',
+      :category => Category.new('for sale', 'sss'),
+      :country => Country.new('us'),
+      :location => Location.new('sfbay.craigslist.org'),
+      :interval => 1
+    },
+    :category_url => "http://sfbay.craigslist.org/",
+    :country_url => "http://geo.craigslist.org/iso",
+    :country_codes => ['jp', 'ar', 'bd', 'br', 'ca', 'cl', 'co', 'cr', 'cz', 'de', 'eg', 'gb', 'hu', 'id', 'ie', 'il', 'lb', 'my', 'nl', 'nz', 'no', 'pk', 'pa', 'ru', 'th', 'ae', 'us', 've', 'vn']
+  })
+end

data/lib/ext/hpricot/elem.rb

@@ -0,0 +1,19 @@
+module Hpricot
+  class Elem
+    def title
+      at("title").inner_text
+    end
+
+    def url
+      self["rdf:about"].strip_http
+    end
+
+    def date_str
+      at("dc:date").inner_text
+    end
+
+    def date
+      DateTime.parse(date_str)
+    end
+  end
+end

data/lib/ext/string.rb

@@ -0,0 +1,5 @@
+class String
+  def strip_http
+    self.gsub(/^http\:\/\//,'').gsub(/\/$/,'')
+  end
+end

data/spec/crags/category_spec.rb

@@ -0,0 +1,54 @@
+require 'spec_helper'
+
+describe Category, "the class" do
+  it "fetches the doc at the configured category url" do
+    Crags::Config.stub!(:category_url).and_return("category_url")
+    Category.should_receive(:fetch_doc).with("category_url").and_return("doc")
+    Category.doc.should == 'doc'
+  end
+
+  describe "with a doc" do
+    before do
+      @links = [{'href' => 'whoa'}]
+      @doc = stub
+      Category.stub!(:doc).and_return(@doc)
+    end
+
+    it "gets all of the category links from the doc" do
+      @doc.should_receive(:search).with("div.col a").and_return(@links)
+      Category.links.should == @links
+    end
+
+    it "doesn't return links to forums" do
+      @doc.stub!(:search).and_return(@links)
+      @links.first['href'] = 'omg/forum/dude'
+      Category.links.should == []
+    end
+  end
+
+  it "creates an array of new Categories based on the links" do
+    links = []
+    2.times do
+      link = {'href' => 'link_href'}
+      link.stub!(:inner_html).and_return('link_inner_html')
+      links << link
+    end
+    Category.stub!(:links).and_return(links)
+    Category.should_receive(:new).with('link_inner_html', 'link_href').twice.and_return('category!')
+    Category.all.should == ['category!', 'category!']
+  end
+end
+
+describe Category do
+  before do
+    @cat = Category.new('name', 'abbr')
+  end
+
+  it "has a name reader" do
+    @cat.name.should == 'name'
+  end
+
+  it "has a url reader" do
+    @cat.url.should == '/abbr'
+  end
+end

data/spec/crags/country_spec.rb

@@ -0,0 +1,57 @@
+require 'spec_helper'
+
+describe Country do
+  before do
+    @country = Country.new("us")
+  end
+
+  it "generates the url for a country" do
+    Crags::Config.stub!(:country_url).and_return("country_url")
+    @country.url.should == "country_url/us"
+  end
+
+  it "is initialized with a country code" do
+    @country.code.should == "us"
+  end
+
+  it "fetches the doc for the country" do
+    @country.stub!(:url).and_return("url")
+    @country.should_receive(:fetch_doc).with("url").and_return("doc")
+    @country.doc.should == "doc"
+  end
+
+  it "fetches the request for the country" do
+    @country.stub!(:url).and_return("url")
+    @country.should_receive(:fetch_request).with("url").and_return("request")
+    @country.request.should == "request"
+  end
+
+  it "gets all a tags from div with id list" do
+    doc = mock
+    doc.should_receive(:search).with("#list a").and_return("links")
+    @country.stub!(:doc).and_return(doc)
+    @country.links.should == "links"
+  end
+
+  it "generates an array of urls using link hrefs" do
+    links = []
+    2.times do |i|
+      link = mock
+      link.should_receive(:[]).with("href").and_return("http://url#{i}/")
+      links << link
+    end
+    @country.stub!(:links).and_return(links)
+    Location.should_receive(:new).with("url0").and_return("url0")
+    Location.should_receive(:new).with("url1").and_return("url1")
+    @country.locations.should == ["url0", "url1"]
+  end
+
+  it "generates an array containing one url using location_urls last_effective_url when no links are present" do
+    @country.stub!(:links).and_return([])
+    req = mock
+    req.should_receive(:last_effective_url).and_return('http://url.org/')
+    @country.stub!(:request).and_return(req)
+    Location.should_receive(:new).with("url.org").and_return("url.org")
+    @country.locations.should == ["url.org"]
+  end
+end

data/spec/crags/fetcher_spec.rb

@@ -0,0 +1,36 @@
+require 'spec_helper'
+
+describe Fetcher do
+  before do
+    extend Fetcher
+  end
+
+  it "hpricots the fetched html" do
+    stub!(:fetch_html).with("url").and_return("html")
+    Hpricot.should_receive(:parse).with("html").and_return("doc")
+    fetch_doc("url").should == "doc"
+  end
+
+  it "fetch_requests a url" do
+    curb = mock("curb")
+    curb.stub!(:body_str).and_return("uhh")
+    should_receive(:fetch_request).with("url").and_return(curb)
+    fetch_html("url").should == "uhh"
+  end
+
+  it "creates a new request" do
+    req = mock("req")
+    req.stub!(:follow_location=)
+    req.stub!(:perform)
+    Curl::Easy.should_receive(:new).with("url").and_return(req)
+    fetch_request("url").should == req
+  end
+
+  it "follows redirects for fetched requests" do
+    req = mock("req")
+    req.should_receive(:follow_location=)
+    req.should_receive(:perform)
+    Curl::Easy.stub!(:new).and_return(req)
+    fetch_request("url")
+  end
+end