gscraper 0.1.7 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- data/COPYING.txt +339 -0
- data/History.txt +21 -0
- data/Manifest.txt +23 -10
- data/README.txt +17 -21
- data/Rakefile +3 -6
- data/lib/gscraper.rb +22 -0
- data/lib/gscraper/extensions.rb +22 -0
- data/lib/gscraper/extensions/uri.rb +22 -0
- data/lib/gscraper/extensions/uri/http.rb +25 -71
- data/lib/gscraper/extensions/uri/query_params.rb +96 -0
- data/lib/gscraper/gscraper.rb +30 -0
- data/lib/gscraper/has_pages.rb +114 -0
- data/lib/gscraper/licenses.rb +22 -0
- data/lib/gscraper/page.rb +64 -0
- data/lib/gscraper/search.rb +24 -0
- data/lib/gscraper/search/ajax_query.rb +176 -0
- data/lib/gscraper/search/page.rb +27 -72
- data/lib/gscraper/search/query.rb +46 -457
- data/lib/gscraper/search/result.rb +32 -29
- data/lib/gscraper/search/search.rb +44 -3
- data/lib/gscraper/search/web_query.rb +472 -0
- data/lib/gscraper/sponsored_ad.rb +26 -2
- data/lib/gscraper/sponsored_links.rb +77 -8
- data/lib/gscraper/version.rb +23 -1
- data/spec/extensions/uri/http_spec.rb +9 -0
- data/spec/extensions/uri/query_params_spec.rb +38 -0
- data/spec/gscraper_spec.rb +29 -0
- data/spec/has_pages_examples.rb +19 -0
- data/spec/has_sponsored_links_examples.rb +57 -0
- data/spec/helpers/query.rb +1 -0
- data/spec/helpers/uri.rb +8 -0
- data/spec/page_has_results_examples.rb +13 -0
- data/spec/search/ajax_query_spec.rb +124 -0
- data/spec/search/page_has_results_examples.rb +51 -0
- data/spec/search/query_spec.rb +103 -0
- data/spec/search/web_query_spec.rb +74 -0
- data/spec/spec_helper.rb +6 -0
- data/tasks/spec.rb +7 -0
- metadata +34 -20
- data/LICENSE.txt +0 -23
- data/lib/gscraper/web_agent.rb +0 -38
- data/test/search/page_results.rb +0 -103
- data/test/search/query_from_url.rb +0 -50
- data/test/search/query_pages.rb +0 -32
- data/test/search/query_result.rb +0 -30
- data/test/test_gscraper.rb +0 -4
|
@@ -1,50 +0,0 @@
|
|
|
1
|
-
require 'test/unit'
|
|
2
|
-
require 'gscraper/search/query'
|
|
3
|
-
|
|
4
|
-
class QueryFromURL < Test::Unit::TestCase
|
|
5
|
-
|
|
6
|
-
include GScraper
|
|
7
|
-
|
|
8
|
-
QUERY_URL = 'http://www.google.com/search?as_q=test&hl=en&num=20&btnG=Google+Search&as_epq=what+if&as_oq=dog&as_eq=haha&lr=&cr=&as_ft=i&as_filetype=&as_qdr=w&as_nlo=&as_nhi=&as_occt=body&as_dt=i&as_sitesearch=&as_rights=&safe=images'
|
|
9
|
-
|
|
10
|
-
def setup
|
|
11
|
-
@query = Search::Query.from_url(QUERY_URL)
|
|
12
|
-
end
|
|
13
|
-
|
|
14
|
-
def test_query
|
|
15
|
-
assert_equal @query.query, 'test'
|
|
16
|
-
end
|
|
17
|
-
|
|
18
|
-
def test_exact_phrase
|
|
19
|
-
assert_equal @query.exact_phrase, 'what+if'
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
def test_with_words
|
|
23
|
-
assert_equal @query.with_words, 'dog'
|
|
24
|
-
end
|
|
25
|
-
|
|
26
|
-
def test_without_words
|
|
27
|
-
assert_equal @query.without_words, 'haha'
|
|
28
|
-
end
|
|
29
|
-
|
|
30
|
-
def test_within_past_week
|
|
31
|
-
assert_equal @query.within_past_week, true
|
|
32
|
-
end
|
|
33
|
-
|
|
34
|
-
def test_occurrs_within
|
|
35
|
-
assert_equal @query.occurrs_within, :body
|
|
36
|
-
end
|
|
37
|
-
|
|
38
|
-
def test_similar_to
|
|
39
|
-
assert_nil @query.similar_to
|
|
40
|
-
end
|
|
41
|
-
|
|
42
|
-
def test_links_to
|
|
43
|
-
assert_nil @query.links_to
|
|
44
|
-
end
|
|
45
|
-
|
|
46
|
-
def teardown
|
|
47
|
-
@query = nil
|
|
48
|
-
end
|
|
49
|
-
|
|
50
|
-
end
|
data/test/search/query_pages.rb
DELETED
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
require 'test/unit'
|
|
2
|
-
require 'gscraper/search/query'
|
|
3
|
-
|
|
4
|
-
class QueryPages < Test::Unit::TestCase
|
|
5
|
-
|
|
6
|
-
include GScraper
|
|
7
|
-
|
|
8
|
-
def setup
|
|
9
|
-
@query = Search::Query.new(:query => 'ruby')
|
|
10
|
-
end
|
|
11
|
-
|
|
12
|
-
def test_first_page
|
|
13
|
-
page = @query.first_page
|
|
14
|
-
|
|
15
|
-
assert_not_nil page
|
|
16
|
-
assert_equal page.empty?, false, "Query of 'ruby' has zero results"
|
|
17
|
-
assert_equal page.length, @query.results_per_page
|
|
18
|
-
end
|
|
19
|
-
|
|
20
|
-
def test_second_page
|
|
21
|
-
page = @query.page(2)
|
|
22
|
-
|
|
23
|
-
assert_not_nil page
|
|
24
|
-
assert_equal page.empty?, false, "Query of 'ruby' has zero results"
|
|
25
|
-
assert_equal page.length, @query.results_per_page
|
|
26
|
-
end
|
|
27
|
-
|
|
28
|
-
def teardown
|
|
29
|
-
@query = nil
|
|
30
|
-
end
|
|
31
|
-
|
|
32
|
-
end
|
data/test/search/query_result.rb
DELETED
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
require 'test/unit'
|
|
2
|
-
require 'gscraper/search/query'
|
|
3
|
-
|
|
4
|
-
class QueryResult < Test::Unit::TestCase
|
|
5
|
-
|
|
6
|
-
include GScraper
|
|
7
|
-
|
|
8
|
-
def setup
|
|
9
|
-
@query = Search::Query.new(:query => 'ruby')
|
|
10
|
-
end
|
|
11
|
-
|
|
12
|
-
def test_first_result
|
|
13
|
-
result = @query.top_result
|
|
14
|
-
|
|
15
|
-
assert_not_nil result, "The Query for 'ruby' has no first-result"
|
|
16
|
-
assert_equal result.rank, 1, "The first result for the Query 'ruby' does not have the rank of 1"
|
|
17
|
-
end
|
|
18
|
-
|
|
19
|
-
def test_second_result
|
|
20
|
-
result = @query.result_at(2)
|
|
21
|
-
|
|
22
|
-
assert_not_nil result, "The Query for 'ruby' has no second-result"
|
|
23
|
-
assert_equal result.rank, 2, "The second result for the Query 'ruby' does not have the rank of 2"
|
|
24
|
-
end
|
|
25
|
-
|
|
26
|
-
def teardown
|
|
27
|
-
@query = nil
|
|
28
|
-
end
|
|
29
|
-
|
|
30
|
-
end
|
data/test/test_gscraper.rb
DELETED