crags 1.6.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +3 -1
- data/Gemfile +8 -0
- data/README.rdoc +45 -8
- data/Rakefile +14 -24
- data/VERSION +1 -1
- data/crags.gemspec +41 -16
- data/lib/crags/category.rb +29 -0
- data/lib/crags/country.rb +37 -0
- data/lib/crags/{fetch.rb → fetcher.rb} +1 -1
- data/lib/crags/item.rb +11 -0
- data/lib/crags/location.rb +13 -0
- data/lib/crags/search/country.rb +26 -0
- data/lib/crags/search/location.rb +28 -0
- data/lib/crags/search/search.rb +13 -0
- data/lib/crags.rb +26 -39
- data/lib/ext/hpricot/elem.rb +19 -0
- data/lib/ext/string.rb +5 -0
- data/spec/crags/category_spec.rb +54 -0
- data/spec/crags/country_spec.rb +57 -0
- data/spec/crags/fetcher_spec.rb +36 -0
- data/spec/crags/item_spec.rb +23 -0
- data/spec/crags/location_spec.rb +15 -0
- data/spec/crags/search/country_spec.rb +67 -0
- data/spec/crags/search/location_spec.rb +59 -0
- data/spec/crags/search/search_spec.rb +25 -0
- data/spec/ext/hpricot/elem_spec.rb +32 -0
- data/spec/ext/string_spec.rb +13 -0
- data/spec/spec_helper.rb +5 -0
- metadata +62 -20
- data/lib/crags/proxy.rb +0 -21
- data/lib/crags/runner.rb +0 -10
- data/lib/crags/searcher.rb +0 -77
- data/test/crags/fetch_test.rb +0 -33
- data/test/crags/proxy_test.rb +0 -25
- data/test/crags/runner_test.rb +0 -28
- data/test/crags/searcher_test.rb +0 -139
- data/test/test_helper.rb +0 -5
data/test/crags/searcher_test.rb
DELETED
@@ -1,139 +0,0 @@
|
|
1
|
-
require File.dirname(__FILE__) + '/../test_helper'
|
2
|
-
|
3
|
-
class Crags::SearcherTest < Test::Unit::TestCase
|
4
|
-
|
5
|
-
context "Searcher with stubbed fetch doc" do
|
6
|
-
setup do
|
7
|
-
extend Crags::Searcher
|
8
|
-
stubs(:sleep)
|
9
|
-
stubs(:fetch_doc)
|
10
|
-
end
|
11
|
-
|
12
|
-
should "strip_http should remove http:// and trailing /" do
|
13
|
-
url = "http://omg/"
|
14
|
-
strip_http(url).should == "omg"
|
15
|
-
end
|
16
|
-
|
17
|
-
should "strip_http should remove http:// when there is no trailing slash" do
|
18
|
-
url = "http://omg"
|
19
|
-
strip_http(url).should == "omg"
|
20
|
-
end
|
21
|
-
|
22
|
-
should "location doc should fetch doc at location url" do
|
23
|
-
expects(:fetch_doc).with("http://geo.craigslist.org/iso/us").returns("doc")
|
24
|
-
location_doc('us').should == "doc"
|
25
|
-
end
|
26
|
-
|
27
|
-
should "location links should get all a tags from div with id list" do
|
28
|
-
doc = mock { expects(:search).with("#list a").returns("links") }
|
29
|
-
stubs(:location_doc).returns(doc)
|
30
|
-
location_links('us').should == "links"
|
31
|
-
end
|
32
|
-
|
33
|
-
should "locations should return array of urls using a location link's href" do
|
34
|
-
links = []
|
35
|
-
2.times do |i|
|
36
|
-
links << mock {|m| m.expects(:[]).with("href").returns("http://url#{i}/") }
|
37
|
-
end
|
38
|
-
stubs(:location_links).returns(links)
|
39
|
-
locations('us').should == ["url0", "url1"]
|
40
|
-
end
|
41
|
-
|
42
|
-
should "locations should return array of one url using location_urls last_effective_url when no links are present on location_url page" do
|
43
|
-
stubs(:location_links).returns([])
|
44
|
-
req = mock(:last_effective_url => 'http://url.org/')
|
45
|
-
stubs(:location_request).with('us').returns(req)
|
46
|
-
locations('us').should == ["url.org"]
|
47
|
-
end
|
48
|
-
|
49
|
-
should "search should search location for each location with keyword and return list" do
|
50
|
-
locations = ["url0", "url1"]
|
51
|
-
|
52
|
-
locations.each do |loc|
|
53
|
-
expects(:search_location).with("omg", loc, 'sss').returns(["1#{loc}", "2#{loc}"])
|
54
|
-
end
|
55
|
-
|
56
|
-
stubs(:locations).returns(locations)
|
57
|
-
search("omg").should == ["1url0", "2url0", "1url1", "2url1"]
|
58
|
-
end
|
59
|
-
|
60
|
-
should "search should call sleep for each location" do
|
61
|
-
expects(:sleep).times(2)
|
62
|
-
stubs(:locations).returns([1,2])
|
63
|
-
stubs(:search_location)
|
64
|
-
search("")
|
65
|
-
end
|
66
|
-
|
67
|
-
should "search location should fetch doc for search url" do
|
68
|
-
expects(:fetch_doc).with("http://url/search/sss?query=keyword&format=rss")
|
69
|
-
stubs(:items).returns([])
|
70
|
-
search_location("keyword", "url")
|
71
|
-
end
|
72
|
-
|
73
|
-
should "search location should create return items" do
|
74
|
-
items = [1,2,3]
|
75
|
-
expects(:items).returns(items)
|
76
|
-
search_location("keyword", "url").should == items
|
77
|
-
end
|
78
|
-
|
79
|
-
should "items should get all item elements from doc" do
|
80
|
-
item = stub
|
81
|
-
stubs(:hashify).with(item).returns(1)
|
82
|
-
doc = mock { expects(:search).with("item").returns([item]) }
|
83
|
-
items(doc).should == [1]
|
84
|
-
end
|
85
|
-
|
86
|
-
should "items should hashify all item elements from doc" do
|
87
|
-
item = stub
|
88
|
-
expects(:hashify).with(item).returns(1)
|
89
|
-
doc = stub { stubs(:search).returns([item]) }
|
90
|
-
items(doc).should == [1]
|
91
|
-
end
|
92
|
-
|
93
|
-
should "categories should fetch doc the main sfbay page" do
|
94
|
-
doc = stub(:search => [])
|
95
|
-
expects(:fetch_doc).with("http://sfbay.craigslist.org/").returns(doc)
|
96
|
-
categories
|
97
|
-
end
|
98
|
-
|
99
|
-
should "categories should search for all links in the table with property summary equal to for sale" do
|
100
|
-
doc = mock { expects(:search).with("table[@summary=\"for sale\"] a").returns([]) }
|
101
|
-
stubs(:fetch_doc).returns(doc)
|
102
|
-
categories
|
103
|
-
end
|
104
|
-
|
105
|
-
should "categories should return a hash with link inner html keys and link href values" do
|
106
|
-
link = stub(:inner_html => "inner_html") do
|
107
|
-
stubs(:[]).with("href").returns("href")
|
108
|
-
end
|
109
|
-
|
110
|
-
doc = stub(:search => [link, link])
|
111
|
-
stubs(:fetch_doc).returns(doc)
|
112
|
-
categories.should == {'inner_html' => 'href', 'inner_html' => 'href'}
|
113
|
-
end
|
114
|
-
|
115
|
-
should "search location should accept a category parameter" do
|
116
|
-
expects(:fetch_doc).with("http://loc/search/scram?query=keyword&format=rss")
|
117
|
-
stubs(:items).returns([])
|
118
|
-
search_location('keyword', 'loc', 'scram')
|
119
|
-
end
|
120
|
-
|
121
|
-
should "search location default category is sss" do
|
122
|
-
expects(:fetch_doc).with("http://loc/search/sss?query=keyword&format=rss")
|
123
|
-
stubs(:items).returns([])
|
124
|
-
search_location('keyword', 'loc')
|
125
|
-
end
|
126
|
-
|
127
|
-
should "search should pass parameter to search location" do
|
128
|
-
stubs(:locations).returns([0])
|
129
|
-
expects(:search_location).with('keyword', 0, 'chum')
|
130
|
-
search('keyword', 'us', 'chum')
|
131
|
-
end
|
132
|
-
|
133
|
-
should "search should have default category of sss" do
|
134
|
-
stubs(:locations).returns([0])
|
135
|
-
expects(:search_location).with('keyword', 0, 'sss')
|
136
|
-
search('keyword')
|
137
|
-
end
|
138
|
-
end
|
139
|
-
end
|