gscraper 0.1.7 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/COPYING.txt +339 -0
- data/History.txt +21 -0
- data/Manifest.txt +23 -10
- data/README.txt +17 -21
- data/Rakefile +3 -6
- data/lib/gscraper.rb +22 -0
- data/lib/gscraper/extensions.rb +22 -0
- data/lib/gscraper/extensions/uri.rb +22 -0
- data/lib/gscraper/extensions/uri/http.rb +25 -71
- data/lib/gscraper/extensions/uri/query_params.rb +96 -0
- data/lib/gscraper/gscraper.rb +30 -0
- data/lib/gscraper/has_pages.rb +114 -0
- data/lib/gscraper/licenses.rb +22 -0
- data/lib/gscraper/page.rb +64 -0
- data/lib/gscraper/search.rb +24 -0
- data/lib/gscraper/search/ajax_query.rb +176 -0
- data/lib/gscraper/search/page.rb +27 -72
- data/lib/gscraper/search/query.rb +46 -457
- data/lib/gscraper/search/result.rb +32 -29
- data/lib/gscraper/search/search.rb +44 -3
- data/lib/gscraper/search/web_query.rb +472 -0
- data/lib/gscraper/sponsored_ad.rb +26 -2
- data/lib/gscraper/sponsored_links.rb +77 -8
- data/lib/gscraper/version.rb +23 -1
- data/spec/extensions/uri/http_spec.rb +9 -0
- data/spec/extensions/uri/query_params_spec.rb +38 -0
- data/spec/gscraper_spec.rb +29 -0
- data/spec/has_pages_examples.rb +19 -0
- data/spec/has_sponsored_links_examples.rb +57 -0
- data/spec/helpers/query.rb +1 -0
- data/spec/helpers/uri.rb +8 -0
- data/spec/page_has_results_examples.rb +13 -0
- data/spec/search/ajax_query_spec.rb +124 -0
- data/spec/search/page_has_results_examples.rb +51 -0
- data/spec/search/query_spec.rb +103 -0
- data/spec/search/web_query_spec.rb +74 -0
- data/spec/spec_helper.rb +6 -0
- data/tasks/spec.rb +7 -0
- metadata +34 -20
- data/LICENSE.txt +0 -23
- data/lib/gscraper/web_agent.rb +0 -38
- data/test/search/page_results.rb +0 -103
- data/test/search/query_from_url.rb +0 -50
- data/test/search/query_pages.rb +0 -32
- data/test/search/query_result.rb +0 -30
- data/test/test_gscraper.rb +0 -4
|
@@ -1,50 +0,0 @@
|
|
|
1
|
-
require 'test/unit'
|
|
2
|
-
require 'gscraper/search/query'
|
|
3
|
-
|
|
4
|
-
class QueryFromURL < Test::Unit::TestCase
|
|
5
|
-
|
|
6
|
-
include GScraper
|
|
7
|
-
|
|
8
|
-
QUERY_URL = 'http://www.google.com/search?as_q=test&hl=en&num=20&btnG=Google+Search&as_epq=what+if&as_oq=dog&as_eq=haha&lr=&cr=&as_ft=i&as_filetype=&as_qdr=w&as_nlo=&as_nhi=&as_occt=body&as_dt=i&as_sitesearch=&as_rights=&safe=images'
|
|
9
|
-
|
|
10
|
-
def setup
|
|
11
|
-
@query = Search::Query.from_url(QUERY_URL)
|
|
12
|
-
end
|
|
13
|
-
|
|
14
|
-
def test_query
|
|
15
|
-
assert_equal @query.query, 'test'
|
|
16
|
-
end
|
|
17
|
-
|
|
18
|
-
def test_exact_phrase
|
|
19
|
-
assert_equal @query.exact_phrase, 'what+if'
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
def test_with_words
|
|
23
|
-
assert_equal @query.with_words, 'dog'
|
|
24
|
-
end
|
|
25
|
-
|
|
26
|
-
def test_without_words
|
|
27
|
-
assert_equal @query.without_words, 'haha'
|
|
28
|
-
end
|
|
29
|
-
|
|
30
|
-
def test_within_past_week
|
|
31
|
-
assert_equal @query.within_past_week, true
|
|
32
|
-
end
|
|
33
|
-
|
|
34
|
-
def test_occurrs_within
|
|
35
|
-
assert_equal @query.occurrs_within, :body
|
|
36
|
-
end
|
|
37
|
-
|
|
38
|
-
def test_similar_to
|
|
39
|
-
assert_nil @query.similar_to
|
|
40
|
-
end
|
|
41
|
-
|
|
42
|
-
def test_links_to
|
|
43
|
-
assert_nil @query.links_to
|
|
44
|
-
end
|
|
45
|
-
|
|
46
|
-
def teardown
|
|
47
|
-
@query = nil
|
|
48
|
-
end
|
|
49
|
-
|
|
50
|
-
end
|
data/test/search/query_pages.rb
DELETED
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
require 'test/unit'
|
|
2
|
-
require 'gscraper/search/query'
|
|
3
|
-
|
|
4
|
-
class QueryPages < Test::Unit::TestCase
|
|
5
|
-
|
|
6
|
-
include GScraper
|
|
7
|
-
|
|
8
|
-
def setup
|
|
9
|
-
@query = Search::Query.new(:query => 'ruby')
|
|
10
|
-
end
|
|
11
|
-
|
|
12
|
-
def test_first_page
|
|
13
|
-
page = @query.first_page
|
|
14
|
-
|
|
15
|
-
assert_not_nil page
|
|
16
|
-
assert_equal page.empty?, false, "Query of 'ruby' has zero results"
|
|
17
|
-
assert_equal page.length, @query.results_per_page
|
|
18
|
-
end
|
|
19
|
-
|
|
20
|
-
def test_second_page
|
|
21
|
-
page = @query.page(2)
|
|
22
|
-
|
|
23
|
-
assert_not_nil page
|
|
24
|
-
assert_equal page.empty?, false, "Query of 'ruby' has zero results"
|
|
25
|
-
assert_equal page.length, @query.results_per_page
|
|
26
|
-
end
|
|
27
|
-
|
|
28
|
-
def teardown
|
|
29
|
-
@query = nil
|
|
30
|
-
end
|
|
31
|
-
|
|
32
|
-
end
|
data/test/search/query_result.rb
DELETED
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
require 'test/unit'
|
|
2
|
-
require 'gscraper/search/query'
|
|
3
|
-
|
|
4
|
-
class QueryResult < Test::Unit::TestCase
|
|
5
|
-
|
|
6
|
-
include GScraper
|
|
7
|
-
|
|
8
|
-
def setup
|
|
9
|
-
@query = Search::Query.new(:query => 'ruby')
|
|
10
|
-
end
|
|
11
|
-
|
|
12
|
-
def test_first_result
|
|
13
|
-
result = @query.top_result
|
|
14
|
-
|
|
15
|
-
assert_not_nil result, "The Query for 'ruby' has no first-result"
|
|
16
|
-
assert_equal result.rank, 1, "The first result for the Query 'ruby' does not have the rank of 1"
|
|
17
|
-
end
|
|
18
|
-
|
|
19
|
-
def test_second_result
|
|
20
|
-
result = @query.result_at(2)
|
|
21
|
-
|
|
22
|
-
assert_not_nil result, "The Query for 'ruby' has no second-result"
|
|
23
|
-
assert_equal result.rank, 2, "The second result for the Query 'ruby' does not have the rank of 2"
|
|
24
|
-
end
|
|
25
|
-
|
|
26
|
-
def teardown
|
|
27
|
-
@query = nil
|
|
28
|
-
end
|
|
29
|
-
|
|
30
|
-
end
|
data/test/test_gscraper.rb
DELETED