crags 1.6.0 → 2.0.0

data/spec/crags/item_spec.rb

@@ -0,0 +1,23 @@
+require 'spec_helper'
+
+describe Item do
+  before do
+    elem = stub
+    elem.stub!(:title).and_return("title!")
+    elem.stub!(:url).and_return("url!")
+    elem.stub!(:date).and_return("date!")
+    @item = Item.new(elem)
+  end
+
+  it "has a title reader" do
+    @item.title.should == "title!"
+  end
+
+  it "has a url reader" do
+    @item.url.should == "url!"
+  end
+
+  it "has a date reader" do
+    @item.date.should == "date!"
+  end
+end

data/spec/crags/location_spec.rb

@@ -0,0 +1,15 @@
+require 'spec_helper'
+
+describe Location do
+  before do
+    @loc = Location.new("domain.com")
+  end
+
+  it "has a domain reader" do
+    @loc.domain.should == "domain.com"
+  end
+
+  it "has a url generated form the domain" do
+    @loc.url.should == "http://domain.com"
+  end
+end

data/spec/crags/search/country_spec.rb

@@ -0,0 +1,67 @@
+require 'spec_helper'
+
+describe Search::Country do
+  before do
+    Crags::Config.stub!(:defaults).and_return({:country => 'heya', :interval => 200})
+    @default = Search::Country.new
+    @custom = Search::Country.new({:country => 'gabba', :interval => 10})
+  end
+
+  it "has config default country" do
+    @default.country.should == "heya"
+  end
+
+  it "has config default interval" do
+    @default.interval.should == 200
+  end
+
+  it "has custom country" do
+    @custom.country.should == "gabba"
+  end
+
+  it "has config default interval" do
+    @custom.interval.should == 10
+  end
+
+  it "delegates locations to country" do
+    @default.country.should_receive(:locations).and_return("locations")
+    @default.locations.should == "locations"
+  end
+
+  describe "with location searches" do
+    before do
+      @search = stub
+      @search.stub!(:items)
+      Search::Location.stub!(:new).and_return(@search)
+      @locations = ["loc1", "loc2"]
+      @custom.stub!(:sleep)
+    end
+
+    it "calls sleep with interval for each location" do
+      @custom.stub!(:locations).and_return(@locations)
+      @custom.should_receive(:sleep).with(10).twice
+      @custom.items
+    end
+
+    it "creates a new search for each location" do
+      Search::Location.should_receive(:new).twice.and_return(@search)
+      @custom.stub!(:locations).and_return(@locations)
+      @custom.items
+    end
+
+    it "performs the search for each location" do
+      @search.should_receive(:items).twice
+      @custom.stub!(:locations).and_return(@locations)
+      @custom.items
+    end
+
+    it "flattens the search results" do
+      collect = mock
+      collect.should_receive(:flatten).and_return("flat!")
+      locations = stub
+      locations.stub!(:collect).and_return(collect)
+      @custom.stub!(:locations).and_return(locations)
+      @custom.items.should == "flat!"
+    end
+  end
+end

data/spec/crags/search/location_spec.rb

@@ -0,0 +1,59 @@
+require 'spec_helper'
+
+describe Search::Location do
+  before do
+    @default_loc = Crags::Location.new("default.domain.com")
+    @default_cat = Crags::Category.new('for sale', 'sss/')
+    @custom_loc = Crags::Location.new("custom.domain.com")
+    Crags::Config.stub!(:defaults).and_return({
+      :keyword => 'bicycle',
+      :location => @default_loc,
+      :category => @default_cat
+    })
+    @default = Search::Location.new
+    @custom = Search::Location.new({:location => @custom_loc})
+  end
+
+  it "has config default location" do
+    @default.location.should == @default_loc
+  end
+
+  it "has custom country" do
+    @custom.location.should == @custom_loc
+  end
+
+  it "generate a url based on location, category and keyword" do
+    @default.stub!(:url_encode).with('bicycle').and_return('bicycle')
+    @default.url.should == "#{@default_loc.url}/search#{@default_cat.url}?query=bicycle"
+  end
+
+  it "fetch the doc at the url with an rss format" do
+    @default.stub!(:url).and_return("url")
+    @default.should_receive(:fetch_doc).with("url&format=rss").and_return('doc')
+    @default.doc.should == 'doc'
+  end
+
+  describe "with a doc" do
+    before do
+      @doc = mock
+      @items = ["item"]
+      @doc.stub!(:search).and_return(@items)
+      @default.stub!(:doc).and_return(@doc)
+      Item.stub!(:new).and_return("crags_item")
+    end
+
+    it "gets all items from the doc" do
+      @doc.should_receive(:search).with("item").and_return(@items)
+      @default.items
+    end
+
+    it "creates a new crags item for each doc item" do
+      Item.should_receive(:new).with("item").and_return("crags_item")
+      @default.items
+    end
+
+    it "collects all created items" do
+      @default.items.should == ["crags_item"]
+    end
+  end
+end

data/spec/crags/search/search_spec.rb

@@ -0,0 +1,25 @@
+require 'spec_helper'
+
+describe Search::Search do
+  before do
+    Crags::Config.stub!(:defaults).and_return({:keyword => 'cribz', :category => 'cribcat'})
+    @default_obj = Search::Search.new
+    @custom_obj = Search::Search.new(:keyword => 'zaks', :category => 'catz')
+  end
+
+  it "has a custom keyword" do
+    @custom_obj.keyword.should == "zaks"
+  end
+
+  it "has a custom category" do
+    @custom_obj.category.should == "catz"
+  end
+
+  it "has a custom keyword" do
+    @default_obj.keyword.should == "cribz"
+  end
+
+  it "has a custom category" do
+    @default_obj.category.should == "cribcat"
+  end
+end

data/spec/ext/hpricot/elem_spec.rb

@@ -0,0 +1,32 @@
+require 'spec_helper'
+
+describe Hpricot::Elem do
+  before do
+    @elem = Hpricot::Elem.new("<omg></omg>")
+  end
+
+  it "uses the inner_text of title element for title" do
+    title = stub
+    title.stub!(:inner_text).and_return("title!")
+    @elem.stub!(:at).with("title").and_return(title)
+    @elem.title.should == "title!"
+  end
+
+  it "rdf:about attribute for the url" do
+    @elem.stub!(:[]).with("rdf:about").and_return("url!")
+    @elem.url.should == "url!"
+  end
+
+  it "uses the inner_text of the dc:date element for date_str" do
+    dc_date = stub
+    dc_date.stub!(:inner_text).and_return("date_str!")
+    @elem.stub!(:at).with("dc:date").and_return(dc_date)
+    @elem.date_str.should == "date_str!"
+  end
+
+  it "parses date_str to create a real date" do
+    DateTime.should_receive(:parse).with("date_str!").and_return("date!")
+    @elem.stub!(:date_str).and_return("date_str!")
+    @elem.date.should == "date!"
+  end
+end

data/spec/ext/string_spec.rb

@@ -0,0 +1,13 @@
+require 'spec_helper'
+
+describe String do
+  it "removes http:// and trailing /" do
+    url = "http://omg/"
+    url.strip_http.should == "omg"
+  end
+
+  it "removes http:// when there is no trailing slash" do
+    url = "http://omg"
+    url.strip_http.should == "omg"
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,5 @@
+require 'rubygems'
+require 'bundler'
+Bundler.require(:default, :test)
+require 'crags'
+include Crags

metadata

@@ -3,10 +3,10 @@ name: crags
 version: !ruby/object:Gem::Version 
   prerelease: false
   segments: 
-  - 1
-  - 6
+  - 2
   - 0
-  version: 1.6.0
+  - 0
+  version: 2.0.0
 platform: ruby
 authors: 
 - Justin Marney
@@ -14,10 +14,33 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-04-14 00:00:00 -04:00
+date: 2010-07-26 00:00:00 -04:00
 default_executable: 
-dependencies: []
-
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: curb
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: hpricot
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
 description: A library to help search across multiple craigslist locations.
 email: gotascii@gmail.com
 executables: []
@@ -29,21 +52,34 @@ extra_rdoc_files:
 - README.rdoc
 files: 
 - .gitignore
+- Gemfile
 - LICENSE
 - README.rdoc
 - Rakefile
 - VERSION
 - crags.gemspec
 - lib/crags.rb
-- lib/crags/fetch.rb
-- lib/crags/proxy.rb
-- lib/crags/runner.rb
-- lib/crags/searcher.rb
-- test/crags/fetch_test.rb
-- test/crags/proxy_test.rb
-- test/crags/runner_test.rb
-- test/crags/searcher_test.rb
-- test/test_helper.rb
+- lib/crags/category.rb
+- lib/crags/country.rb
+- lib/crags/fetcher.rb
+- lib/crags/item.rb
+- lib/crags/location.rb
+- lib/crags/search/country.rb
+- lib/crags/search/location.rb
+- lib/crags/search/search.rb
+- lib/ext/hpricot/elem.rb
+- lib/ext/string.rb
+- spec/crags/category_spec.rb
+- spec/crags/country_spec.rb
+- spec/crags/fetcher_spec.rb
+- spec/crags/item_spec.rb
+- spec/crags/location_spec.rb
+- spec/crags/search/country_spec.rb
+- spec/crags/search/location_spec.rb
+- spec/crags/search/search_spec.rb
+- spec/ext/hpricot/elem_spec.rb
+- spec/ext/string_spec.rb
+- spec/spec_helper.rb
 has_rdoc: true
 homepage: http://github.com/gotascii/crags
 licenses: []
@@ -75,8 +111,14 @@ signing_key:
 specification_version: 3
 summary: A library to help search across multiple craigslist locations.
 test_files: 
-- test/crags/fetch_test.rb
-- test/crags/proxy_test.rb
-- test/crags/runner_test.rb
-- test/crags/searcher_test.rb
-- test/test_helper.rb
+- spec/crags/category_spec.rb
+- spec/crags/country_spec.rb
+- spec/crags/fetcher_spec.rb
+- spec/crags/item_spec.rb
+- spec/crags/location_spec.rb
+- spec/crags/search/country_spec.rb
+- spec/crags/search/location_spec.rb
+- spec/crags/search/search_spec.rb
+- spec/ext/hpricot/elem_spec.rb
+- spec/ext/string_spec.rb
+- spec/spec_helper.rb

data/lib/crags/proxy.rb

@@ -1,21 +0,0 @@
-module Crags
-  module Proxy
-    include Fetch
-
-    def lists
-      ["http://www.proxy4free.com/page1.html","http://www.proxy4free.com/page3.html"]
-    end
-
-    def fetch_lists
-      lists.collect {|url| fetch_html(url)}
-    end
-
-    def scan(text)
-      text.scan(/\d+\.\d+\.\d+\.\d+/)
-    end
-
-    def proxies
-      fetch_lists.collect{|html|scan(html)}.flatten.uniq
-    end
-  end
-end

data/lib/crags/runner.rb

@@ -1,10 +0,0 @@
-module Crags
-  class Runner
-    include Searcher
-
-    def search_location(keyword, loc, category = 'sss')
-      puts "Searching #{loc}..."
-      super
-    end
-  end
-end

data/lib/crags/searcher.rb

@@ -1,77 +0,0 @@
-module Crags
-  module Searcher
-    include Fetch
-    include ERB::Util
-
-    def strip_http(url)
-      url.gsub(/^http\:\/\//,'').gsub(/\/$/,'')
-    end
-
-    def location_link(country)
-      "http://geo.craigslist.org/iso/#{country}"
-    end
-
-    def location_doc(country)
-      fetch_doc(location_link(country))
-    end
-
-    def location_request(country)
-      fetch_request(location_link(country))
-    end
-
-    def location_links(country)
-      location_doc(country).search("#list a")
-    end
-
-    def locations(country)
-      linkz = location_links(country)
-      if linkz.empty?
-        [strip_http(location_request(country).last_effective_url)]
-      else
-        linkz.collect{|link| strip_http(link["href"]) }
-      end
-    end
-
-    def categories
-      doc = fetch_doc("http://sfbay.craigslist.org/")
-      links = doc.search("table[@summary=\"for sale\"] a")
-      categories = {}
-      links.each do |link|
-        categories[link.inner_html] = link["href"]
-      end
-      categories
-    end
-
-    def search(keyword, country = 'us', category = 'sss', &block)
-      locations(country).collect do |loc|
-        sleep(1 + rand(3))
-        search_location(keyword, loc, category, &block)
-      end.flatten
-    end
-
-    def items(doc)
-      doc.search("item").collect do |item|
-        hashify(item)
-      end
-    end
-
-    def hashify(item)
-      title = item.at("title").inner_text
-      url = strip_http(item["rdf:about"])
-      date = DateTime.parse(item.at("dc:date").inner_text)
-      {:title => title, :url => url, :date => date}
-    end
-
-    def search_location_link(keyword, loc, category = 'sss')
-      "http://#{loc}/search/#{category}?query=#{url_encode(keyword)}"
-    end
-
-    def search_location(keyword, loc, category = 'sss', &block)
-      doc = fetch_doc("#{search_location_link(keyword, loc, category)}&format=rss")
-      items(doc).collect do |item|
-        yield item if block_given?
-        item
-      end
-    end
-  end
-end

data/test/crags/fetch_test.rb

@@ -1,33 +0,0 @@
-require File.dirname(__FILE__) + '/../test_helper'
-
-class Crags::FetchTest < Test::Unit::TestCase
-  context "Fetch" do
-    setup do
-      extend Crags::Fetch
-    end
-
-    should "fetch doc should hpricot fetched html" do
-      stubs(:fetch_html).with("url").returns("html")
-      Hpricot.expects(:parse).with("html").returns("doc")
-      fetch_doc("url").should == "doc"
-    end
-
-    should "fetch html should fetch_request a url" do
-      curb = stub(:body_str => "uhh")
-      expects(:fetch_request).with("url").returns(curb)
-      fetch_html("url").should == "uhh"
-    end
-
-    should "create a new request" do
-      req = stub(:follow_location= => nil, :perform => nil)
-      Curl::Easy.expects(:new).with("url").returns(req)
-      fetch_request("url").should == req
-    end
-    
-    should "follow redirects for fetched requests" do
-      req = mock(:follow_location= => nil, :perform => nil)
-      Curl::Easy.stubs(:new).returns(req)
-      fetch_request("url")
-    end
-  end
-end

data/test/crags/proxy_test.rb

@@ -1,25 +0,0 @@
-require File.dirname(__FILE__) + '/../test_helper'
-
-class Crags::ProxyTest < Test::Unit::TestCase
-  context "Proxy" do
-    setup do
-      extend Crags::Proxy
-    end
-
-    should "lists should return a list of proxy list websites" do
-      lists.should == ["http://www.proxy4free.com/page1.html", "http://www.proxy4free.com/page3.html"]
-    end
-
-    should "fetch lists should fetch html for each site in lists" do
-      stubs(:lists).returns(["1", "2"])
-      expects(:fetch_html).with("1").returns("html_1")
-      expects(:fetch_html).with("2").returns("html_2")
-      fetch_lists.should == ["html_1", "html_2"]
-    end
-
-    should "scan should return all ips in a text blizoc" do
-      text = "192.168.1.2 omg dude!! wtf.f.f.asdasd9.8.9 78.900.42.32"
-      scan(text).should == ["192.168.1.2", "78.900.42.32"]
-    end
-  end
-end

data/test/crags/runner_test.rb

@@ -1,28 +0,0 @@
-require File.dirname(__FILE__) + '/../test_helper'
-
-class Crags::RunnerTest < Test::Unit::TestCase
-  context "instance of Runner" do
-    setup do
-      @runner = Crags::Runner.new
-      @runner.stubs(:fetch_doc)
-      @runner.stubs(:items).returns([])
-    end
-
-    should "runner should include searcher" do
-      Crags::Runner.ancestors.include?(Crags::Searcher).should == true
-    end
-
-    should "search location should puts message with loc" do
-      @runner.expects(:puts).with { |val| val =~ /location/ }
-      @runner.search_location("", "location", "category")
-    end
-
-    should "search location should take a category" do
-      @runner.search_location("", "location", "category")
-    end
-
-    should "search location should have default category sss" do
-      @runner.search_location("", "location")
-    end
-  end
-end