crags 1.6.0 → 2.0.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +3 -1
- data/Gemfile +8 -0
- data/README.rdoc +45 -8
- data/Rakefile +14 -24
- data/VERSION +1 -1
- data/crags.gemspec +41 -16
- data/lib/crags/category.rb +29 -0
- data/lib/crags/country.rb +37 -0
- data/lib/crags/{fetch.rb → fetcher.rb} +1 -1
- data/lib/crags/item.rb +11 -0
- data/lib/crags/location.rb +13 -0
- data/lib/crags/search/country.rb +26 -0
- data/lib/crags/search/location.rb +28 -0
- data/lib/crags/search/search.rb +13 -0
- data/lib/crags.rb +26 -39
- data/lib/ext/hpricot/elem.rb +19 -0
- data/lib/ext/string.rb +5 -0
- data/spec/crags/category_spec.rb +54 -0
- data/spec/crags/country_spec.rb +57 -0
- data/spec/crags/fetcher_spec.rb +36 -0
- data/spec/crags/item_spec.rb +23 -0
- data/spec/crags/location_spec.rb +15 -0
- data/spec/crags/search/country_spec.rb +67 -0
- data/spec/crags/search/location_spec.rb +59 -0
- data/spec/crags/search/search_spec.rb +25 -0
- data/spec/ext/hpricot/elem_spec.rb +32 -0
- data/spec/ext/string_spec.rb +13 -0
- data/spec/spec_helper.rb +5 -0
- metadata +62 -20
- data/lib/crags/proxy.rb +0 -21
- data/lib/crags/runner.rb +0 -10
- data/lib/crags/searcher.rb +0 -77
- data/test/crags/fetch_test.rb +0 -33
- data/test/crags/proxy_test.rb +0 -25
- data/test/crags/runner_test.rb +0 -28
- data/test/crags/searcher_test.rb +0 -139
- data/test/test_helper.rb +0 -5
data/test/crags/searcher_test.rb
DELETED
@@ -1,139 +0,0 @@
|
|
1
|
-
require File.dirname(__FILE__) + '/../test_helper'
|
2
|
-
|
3
|
-
class Crags::SearcherTest < Test::Unit::TestCase
|
4
|
-
|
5
|
-
context "Searcher with stubbed fetch doc" do
|
6
|
-
setup do
|
7
|
-
extend Crags::Searcher
|
8
|
-
stubs(:sleep)
|
9
|
-
stubs(:fetch_doc)
|
10
|
-
end
|
11
|
-
|
12
|
-
should "strip_http should remove http:// and trailing /" do
|
13
|
-
url = "http://omg/"
|
14
|
-
strip_http(url).should == "omg"
|
15
|
-
end
|
16
|
-
|
17
|
-
should "strip_http should remove http:// when there is no trailing slash" do
|
18
|
-
url = "http://omg"
|
19
|
-
strip_http(url).should == "omg"
|
20
|
-
end
|
21
|
-
|
22
|
-
should "location doc should fetch doc at location url" do
|
23
|
-
expects(:fetch_doc).with("http://geo.craigslist.org/iso/us").returns("doc")
|
24
|
-
location_doc('us').should == "doc"
|
25
|
-
end
|
26
|
-
|
27
|
-
should "location links should get all a tags from div with id list" do
|
28
|
-
doc = mock { expects(:search).with("#list a").returns("links") }
|
29
|
-
stubs(:location_doc).returns(doc)
|
30
|
-
location_links('us').should == "links"
|
31
|
-
end
|
32
|
-
|
33
|
-
should "locations should return array of urls using a location link's href" do
|
34
|
-
links = []
|
35
|
-
2.times do |i|
|
36
|
-
links << mock {|m| m.expects(:[]).with("href").returns("http://url#{i}/") }
|
37
|
-
end
|
38
|
-
stubs(:location_links).returns(links)
|
39
|
-
locations('us').should == ["url0", "url1"]
|
40
|
-
end
|
41
|
-
|
42
|
-
should "locations should return array of one url using location_urls last_effective_url when no links are present on location_url page" do
|
43
|
-
stubs(:location_links).returns([])
|
44
|
-
req = mock(:last_effective_url => 'http://url.org/')
|
45
|
-
stubs(:location_request).with('us').returns(req)
|
46
|
-
locations('us').should == ["url.org"]
|
47
|
-
end
|
48
|
-
|
49
|
-
should "search should search location for each location with keyword and return list" do
|
50
|
-
locations = ["url0", "url1"]
|
51
|
-
|
52
|
-
locations.each do |loc|
|
53
|
-
expects(:search_location).with("omg", loc, 'sss').returns(["1#{loc}", "2#{loc}"])
|
54
|
-
end
|
55
|
-
|
56
|
-
stubs(:locations).returns(locations)
|
57
|
-
search("omg").should == ["1url0", "2url0", "1url1", "2url1"]
|
58
|
-
end
|
59
|
-
|
60
|
-
should "search should call sleep for each location" do
|
61
|
-
expects(:sleep).times(2)
|
62
|
-
stubs(:locations).returns([1,2])
|
63
|
-
stubs(:search_location)
|
64
|
-
search("")
|
65
|
-
end
|
66
|
-
|
67
|
-
should "search location should fetch doc for search url" do
|
68
|
-
expects(:fetch_doc).with("http://url/search/sss?query=keyword&format=rss")
|
69
|
-
stubs(:items).returns([])
|
70
|
-
search_location("keyword", "url")
|
71
|
-
end
|
72
|
-
|
73
|
-
should "search location should create return items" do
|
74
|
-
items = [1,2,3]
|
75
|
-
expects(:items).returns(items)
|
76
|
-
search_location("keyword", "url").should == items
|
77
|
-
end
|
78
|
-
|
79
|
-
should "items should get all item elements from doc" do
|
80
|
-
item = stub
|
81
|
-
stubs(:hashify).with(item).returns(1)
|
82
|
-
doc = mock { expects(:search).with("item").returns([item]) }
|
83
|
-
items(doc).should == [1]
|
84
|
-
end
|
85
|
-
|
86
|
-
should "items should hashify all item elements from doc" do
|
87
|
-
item = stub
|
88
|
-
expects(:hashify).with(item).returns(1)
|
89
|
-
doc = stub { stubs(:search).returns([item]) }
|
90
|
-
items(doc).should == [1]
|
91
|
-
end
|
92
|
-
|
93
|
-
should "categories should fetch doc the main sfbay page" do
|
94
|
-
doc = stub(:search => [])
|
95
|
-
expects(:fetch_doc).with("http://sfbay.craigslist.org/").returns(doc)
|
96
|
-
categories
|
97
|
-
end
|
98
|
-
|
99
|
-
should "categories should search for all links in the table with property summary equal to for sale" do
|
100
|
-
doc = mock { expects(:search).with("table[@summary=\"for sale\"] a").returns([]) }
|
101
|
-
stubs(:fetch_doc).returns(doc)
|
102
|
-
categories
|
103
|
-
end
|
104
|
-
|
105
|
-
should "categories should return a hash with link inner html keys and link href values" do
|
106
|
-
link = stub(:inner_html => "inner_html") do
|
107
|
-
stubs(:[]).with("href").returns("href")
|
108
|
-
end
|
109
|
-
|
110
|
-
doc = stub(:search => [link, link])
|
111
|
-
stubs(:fetch_doc).returns(doc)
|
112
|
-
categories.should == {'inner_html' => 'href', 'inner_html' => 'href'}
|
113
|
-
end
|
114
|
-
|
115
|
-
should "search location should accept a category parameter" do
|
116
|
-
expects(:fetch_doc).with("http://loc/search/scram?query=keyword&format=rss")
|
117
|
-
stubs(:items).returns([])
|
118
|
-
search_location('keyword', 'loc', 'scram')
|
119
|
-
end
|
120
|
-
|
121
|
-
should "search location default category is sss" do
|
122
|
-
expects(:fetch_doc).with("http://loc/search/sss?query=keyword&format=rss")
|
123
|
-
stubs(:items).returns([])
|
124
|
-
search_location('keyword', 'loc')
|
125
|
-
end
|
126
|
-
|
127
|
-
should "search should pass parameter to search location" do
|
128
|
-
stubs(:locations).returns([0])
|
129
|
-
expects(:search_location).with('keyword', 0, 'chum')
|
130
|
-
search('keyword', 'us', 'chum')
|
131
|
-
end
|
132
|
-
|
133
|
-
should "search should have default category of sss" do
|
134
|
-
stubs(:locations).returns([0])
|
135
|
-
expects(:search_location).with('keyword', 0, 'sss')
|
136
|
-
search('keyword')
|
137
|
-
end
|
138
|
-
end
|
139
|
-
end
|