gscraper 0.2.4 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +8 -0
- data/.specopts +1 -0
- data/.yardopts +1 -0
- data/ChangeLog.md +122 -0
- data/Gemfile +25 -0
- data/{README.txt → README.md} +25 -24
- data/Rakefile +32 -10
- data/gscraper.gemspec +112 -0
- data/lib/gscraper.rb +0 -2
- data/lib/gscraper/extensions.rb +0 -2
- data/lib/gscraper/extensions/uri.rb +0 -2
- data/lib/gscraper/extensions/uri/http.rb +0 -2
- data/lib/gscraper/extensions/uri/query_params.rb +18 -5
- data/lib/gscraper/gscraper.rb +61 -70
- data/lib/gscraper/has_pages.rb +76 -20
- data/lib/gscraper/licenses.rb +0 -2
- data/lib/gscraper/page.rb +45 -16
- data/lib/gscraper/search.rb +0 -2
- data/lib/gscraper/search/ajax_query.rb +75 -22
- data/lib/gscraper/search/page.rb +328 -122
- data/lib/gscraper/search/query.rb +100 -7
- data/lib/gscraper/search/result.rb +27 -6
- data/lib/gscraper/search/search.rb +59 -9
- data/lib/gscraper/search/web_query.rb +120 -37
- data/lib/gscraper/sponsored_ad.rb +19 -6
- data/lib/gscraper/sponsored_links.rb +260 -92
- data/lib/gscraper/version.rb +2 -3
- data/spec/extensions/uri/query_params_spec.rb +8 -0
- data/spec/gscraper_spec.rb +9 -4
- data/spec/has_pages_examples.rb +0 -2
- data/spec/has_sponsored_links_examples.rb +2 -1
- data/spec/helpers/query.rb +3 -1
- data/spec/helpers/uri.rb +6 -4
- data/spec/page_has_results_examples.rb +0 -2
- data/spec/search/ajax_query_spec.rb +6 -11
- data/spec/search/page_has_results_examples.rb +0 -2
- data/spec/search/web_query_spec.rb +6 -11
- data/spec/spec_helper.rb +10 -4
- metadata +147 -54
- data/History.txt +0 -101
- data/Manifest.txt +0 -38
- data/tasks/spec.rb +0 -9
data/History.txt
DELETED
@@ -1,101 +0,0 @@
|
|
1
|
-
=== 0.2.4 / 2009-03-18
|
2
|
-
|
3
|
-
* Added SponsoredAd#direct_link.
|
4
|
-
* Fixed a bug in SponsoredAd#direct_url where direct links we're not
|
5
|
-
being URI escaped.
|
6
|
-
* Removed last references to Hpricot, replacing them with Nokogiri.
|
7
|
-
|
8
|
-
=== 0.2.3 / 2009-01-27
|
9
|
-
|
10
|
-
* Fixed a bug in GScraper::Search::WebQuery#page, when the search query
|
11
|
-
returned less results than the expected results-per-page.
|
12
|
-
|
13
|
-
=== 0.2.2 / 2009-01-14
|
14
|
-
|
15
|
-
* Updated GScraper::Search::WebQuery to use Nokogiri properly.
|
16
|
-
|
17
|
-
=== 0.2.1 / 2008-08-27
|
18
|
-
|
19
|
-
* Updated XPath queries in GScraper::Search::WebQuery for new Google (tm)
|
20
|
-
Search Result HTML schema.
|
21
|
-
|
22
|
-
=== 0.2.0 / 2008-05-10
|
23
|
-
|
24
|
-
* Removed GScraper::WebAgent.
|
25
|
-
* Added GScraper::Page and GScraper::HasPages.
|
26
|
-
* GScraper::Search::Result#page and GScraper::Search::Result#cached_page
|
27
|
-
no longer receives blocks.
|
28
|
-
* GScraper::SponsoredLinks.new and GScraper::Search::Page.new now take
|
29
|
-
blocks.
|
30
|
-
* Renamed GScraper::Search::Query to GScraper::Search::WebQuery.
|
31
|
-
* Added GScraper::Search::Query which supports building query expressions.
|
32
|
-
* GScraper::Search::WebQuery#page and GScraper::Search::WebQuery#sponsored_links
|
33
|
-
no longer take blocks.
|
34
|
-
* Added GScraper::Search::AJAXQuery.
|
35
|
-
* Replaced Unit Tests with Rspec specifications.
|
36
|
-
|
37
|
-
=== 0.1.8 / 2008-04-30
|
38
|
-
|
39
|
-
* Added the GScraper.user_agent_alias=(name) method.
|
40
|
-
* Added URI::HTTP::QueryParams module.
|
41
|
-
* Changed license from MIT to GPL-2.
|
42
|
-
|
43
|
-
=== 0.1.7 / 2008-04-28
|
44
|
-
|
45
|
-
* Added support for specifing Search modifiers.
|
46
|
-
|
47
|
-
Search.query(:filetype => :xls)
|
48
|
-
|
49
|
-
* Added the Search::Result#page method.
|
50
|
-
|
51
|
-
=== 0.1.6 / 2008-03-15
|
52
|
-
|
53
|
-
* Renamed GScraper.http_agent to GScraper.web_agent.
|
54
|
-
* Added GScraper.proxy for global proxy configuration.
|
55
|
-
* Added the WebAgent module.
|
56
|
-
* Renamed Search::Query#first_result to Search::Query#top_result.
|
57
|
-
* Updated Search::Query#page logic for the new DOM layout being used.
|
58
|
-
* Added support for Sponsored Ad scraping.
|
59
|
-
* Added the methods Query#sponsored_links and Query#top_sponsored_link.
|
60
|
-
* Added examples to README.txt.
|
61
|
-
|
62
|
-
=== 0.1.5 / 2007-12-29
|
63
|
-
|
64
|
-
* Fixed class inheritance in gscraper/extensions/uri/http.rb, found by
|
65
|
-
sanitybit.
|
66
|
-
|
67
|
-
=== 0.1.4 / 2007-12-23
|
68
|
-
|
69
|
-
* Added Search::Query#result_at for easier access of a single result at
|
70
|
-
a given index.
|
71
|
-
* Adding scraping of the "Cached" and "Similar Pages" URLs of Search
|
72
|
-
Results.
|
73
|
-
* Added methods to Search::Page for accessing cached URLs, cached pages,
|
74
|
-
similar query URLs and similar Queries in mass.
|
75
|
-
* Search::Query#page and Search::Query#first_page now can receive blocks.
|
76
|
-
* Improved the formating of URL query parameters.
|
77
|
-
* Added more unit-tests.
|
78
|
-
* Fixed scraping of Search Result summaries.
|
79
|
-
* Fixed various bugs in Search::Query uncovered during unit-testing.
|
80
|
-
* Fixed typos in Search::Page's documentation.
|
81
|
-
|
82
|
-
=== 0.1.3 / 2007-12-22
|
83
|
-
|
84
|
-
* Added the Search::Page class, which contains many of convenance methods
|
85
|
-
for searching through the results within a Page.
|
86
|
-
|
87
|
-
=== 0.1.2 / 2007-12-22
|
88
|
-
|
89
|
-
* Fixed a bug related to extracting the correct content-rights from search
|
90
|
-
query URLs.
|
91
|
-
* Added GScraper.user_agent_aliases.
|
92
|
-
|
93
|
-
=== 0.1.1 / 2007-12-21
|
94
|
-
|
95
|
-
* Forgot to include lib/gscraper/version.rb.
|
96
|
-
|
97
|
-
=== 0.1.0 / 2007-12-20
|
98
|
-
|
99
|
-
* Initial release.
|
100
|
-
* Supports the Google Search service.
|
101
|
-
|
data/Manifest.txt
DELETED
@@ -1,38 +0,0 @@
|
|
1
|
-
History.txt
|
2
|
-
COPYING.txt
|
3
|
-
Manifest.txt
|
4
|
-
README.txt
|
5
|
-
Rakefile
|
6
|
-
lib/gscraper/extensions/uri/query_params.rb
|
7
|
-
lib/gscraper/extensions/uri/http.rb
|
8
|
-
lib/gscraper/extensions/uri.rb
|
9
|
-
lib/gscraper/extensions.rb
|
10
|
-
lib/gscraper/licenses.rb
|
11
|
-
lib/gscraper/page.rb
|
12
|
-
lib/gscraper/has_pages.rb
|
13
|
-
lib/gscraper/sponsored_ad.rb
|
14
|
-
lib/gscraper/sponsored_links.rb
|
15
|
-
lib/gscraper/search/result.rb
|
16
|
-
lib/gscraper/search/page.rb
|
17
|
-
lib/gscraper/search/query.rb
|
18
|
-
lib/gscraper/search/web_query.rb
|
19
|
-
lib/gscraper/search/ajax_query.rb
|
20
|
-
lib/gscraper/search/search.rb
|
21
|
-
lib/gscraper/search.rb
|
22
|
-
lib/gscraper/gscraper.rb
|
23
|
-
lib/gscraper/version.rb
|
24
|
-
lib/gscraper.rb
|
25
|
-
tasks/spec.rb
|
26
|
-
spec/spec_helper.rb
|
27
|
-
spec/helpers/uri.rb
|
28
|
-
spec/helpers/query.rb
|
29
|
-
spec/extensions/uri/query_params_spec.rb
|
30
|
-
spec/extensions/uri/http_spec.rb
|
31
|
-
spec/has_pages_examples.rb
|
32
|
-
spec/page_has_results_examples.rb
|
33
|
-
spec/has_sponsored_links_examples.rb
|
34
|
-
spec/search/page_has_results_examples.rb
|
35
|
-
spec/search/query_spec.rb
|
36
|
-
spec/search/ajax_query_spec.rb
|
37
|
-
spec/search/web_query_spec.rb
|
38
|
-
spec/gscraper_spec.rb
|