gscraper 0.2.4 → 0.3.0

data/.gitignore

@@ -0,0 +1,8 @@
+doc
+pkg
+tmp/*
+.DS_Store
+.bundle
+.yardoc
+*.swp
+*~

data/.specopts

@@ -0,0 +1 @@
+--colour --format specdoc

data/.yardopts

@@ -0,0 +1 @@
+--markup markdown --title 'GScraper Documentation' --protected --files ChangeLog.md,COPYING.txt

data/ChangeLog.md

@@ -0,0 +1,122 @@
+### 0.3.0 / 2010-07-01
+
+* Upgraded to mechanize ~> 1.0.0.
+* Upgraded from json to json_pure ~> 1.4.0.
+* Switched from Hoe to Jeweler for building RubyGems.
+* Switched to Markdown documentation syntax with full YARD tags.
+* Added support for `:allinanchor` and `:inanchor` options to
+  {GScraper::Search::Query}.
+* Added support for the `:define` option in {GScraper::Search::Query}.
+* Aliased {GScraper::Search::WebQuery#similar_to} to `related`.
+* Aliased {GScraper::Search::WebQuery#similar_to=} to `related=`.
+* Aliased {GScraper::Search::WebQuery#links_to} to `link`.
+* Aliased {GScraper::Search::WebQuery#links_to=} to `link=`.
+* Removed `GScraper.open_uri`.
+* Removed `GScraper.open_page`.
+* Fixed the escaping/unescaping of URL query params in {URI::QueryParams}.
+* Use `yield` instead of `block.call`, when possible.
+* All enumerable methods now return an `Enumerator` object, if no block was
+  given.
+
+### 0.2.4 / 2009-03-18
+
+* Added {GScraper::SponsoredAd#direct_link}.
+* Fixed a bug in {GScraper::SponsoredAd#direct_url} where direct links
+  were not being URI escaped.
+* Removed last references to Hpricot, replacing them with Nokogiri.
+
+### 0.2.3 / 2009-01-27
+
+* Fixed a bug in {GScraper::Search::WebQuery#page}, when the search query
+  returned less results than the expected results-per-page.
+
+### 0.2.2 / 2009-01-14
+
+* Updated {GScraper::Search::WebQuery} to use Nokogiri properly.
+
+### 0.2.1 / 2008-08-27
+
+* Updated XPath queries in {GScraper::Search::WebQuery} for new Google (tm)
+  Search Result HTML schema.
+
+### 0.2.0 / 2008-05-10
+
+* Removed `GScraper::WebAgent`.
+* Added {GScraper::Page} and {GScraper::HasPages}.
+* {GScraper::Search::Result#page} and {GScraper::Search::Result#cached_page}
+  no longer receives blocks.
+* Added `GScraper::Search::Query` which supports building query expressions.
+* {GScraper::SponsoredLinks#initialize} and {GScraper::Page#initialize}
+  now take blocks.
+* Renamed `GScraper::Search::Query` to {GScraper::Search::WebQuery}.
+* {GScraper::Search::WebQuery#page} and
+  {GScraper::Search::WebQuery#sponsored_links} no longer take blocks.
+* Added {GScraper::Search::AJAXQuery}.
+* Replaced Unit Tests with Rspec specifications.
+
+### 0.1.8 / 2008-04-30
+
+* Added the {GScraper.user_agent_alias=} method.
+* Added {URI::HTTP::QueryParams} module.
+* Changed license from MIT to GPL-2.
+
+### 0.1.7 / 2008-04-28
+
+* Added support for specifing Search modifiers.
+
+        Search.query(:filetype => :xls)
+
+* Added the {GScraper::Search::Result#page} method.
+
+### 0.1.6 / 2008-03-15
+
+* Renamed `GScraper.http_agent` to {GScraper.web_agent}.
+* Added {GScraper.proxy} for global proxy configuration.
+* Added the `WebAgent` module.
+* Renamed `Search::Query#first_result` to `Search::Query#top_result`.
+* Updated `Search::Query#page` logic for the new DOM layout being used.
+* Added support for Sponsored Ad scraping.
+  * Added the methods `Query#sponsored_links` and
+    `Query#top_sponsored_link`.
+* Added examples to README.txt.
+
+### 0.1.5 / 2007-12-29
+
+* Fixed class inheritance in `gscraper/extensions/uri/http.rb`, found by
+  sanitybit.
+
+### 0.1.4 / 2007-12-23
+
+* Added `Search::Query#result_at` for easier access of a single result at
+  a given index.
+* Adding scraping of the Cached and Similar Pages URLs of Search
+  Results.
+* Added methods to `Search::Page` for accessing cached URLs, cached pages,
+  similar query URLs and similar Queries in mass.
+* Search::Query#page and `Search::Query#first_page` now can receive blocks.
+* Improved the formating of URL query parameters.
+* Added more unit-tests.
+* Fixed scraping of Search Result summaries.
+* Fixed various bugs in `Search::Query` uncovered during unit-testing.
+* Fixed typos in the documentation for `Search::Page`.
+
+### 0.1.3 / 2007-12-22
+
+* Added the `Search::Page` class, which contains many of convenance methods
+  for searching through the results within a Page.
+
+### 0.1.2 / 2007-12-22
+
+* Fixed a bug related to extracting the correct content-rights from search
+  query URLs.
+* Added {GScraper.user_agent_aliases}.
+
+### 0.1.1 / 2007-12-21
+
+* Forgot to include `lib/gscraper/version.rb`.
+
+### 0.1.0 / 2007-12-20
+
+* Initial release.
+* Supports the Google Search service.
+

data/Gemfile

@@ -0,0 +1,25 @@
+source 'https://rubygems.org'
+
+group(:runtime) do
+  gem 'json_pure',	'~> 1.4.0'
+  gem 'mechanize',	'~> 1.0.0'
+end
+
+group(:development) do
+  gem 'bundler',		'~> 0.9.19'
+  gem 'rake',			'~> 0.8.7'
+  gem 'jeweler',		'~> 1.4.0', :git => 'git://github.com/technicalpickles/jeweler.git'
+end
+
+group(:doc) do
+  case RUBY_PLATFORM
+  when 'java'
+    gem 'maruku',	'~> 0.6.0'
+  else
+    gem 'rdiscount',	'~> 1.6.3'
+  end
+
+  gem 'yard',		'~> 0.5.3'
+end
+
+gem 'rspec',	'~> 1.3.0', :group => [:development, :test]

data/{README.txt → README.md}

@@ -1,14 +1,14 @@
-= GScraper
+# GScraper
 
-* http://gscraper.rubyforge.org/
-* http://github.com/postmodern/gscraper/
+* [github.com/postmodern/gscraper](http://github.com/postmodern/gscraper/)
+* [github.com/postmodern/gscraper/issues](http://github.com/postmodern/gscraper/issues)
 * Postmodern (postmodern.mod3 at gmail.com)
 
-== DESCRIPTION:
+## Description
   
 GScraper is a web-scraping interface to various Google Services.
 
-== FEATURES/PROBLEMS:
+## Features
   
 * Supports the Google Search service.
   * Provides access to search results and ranks.
@@ -16,21 +16,21 @@ GScraper is a web-scraping interface to various Google Services.
 * Provides HTTP access with custom User-Agent strings.
 * Provides proxy settings for HTTP access.
 
-== REQUIREMENTS:
+## Requirements
 
-* mechanize >= 0.9.0
+* [mechanize](http://mechanize.rubyforge.org/mechanize/) ~> 1.0.0
 
-== INSTALL:
+## Install
 
-  $ sudo gem install gscraper
+    $ sudo gem install gscraper
 
-== EXAMPLES:
+## Examples
 
-* Basic query:
+Basic query:
 
     q = GScraper::Search.query(:query => 'ruby')
 
-* Advanced query:
+Advanced query:
 
     q = GScraper::Search.query(:query => 'ruby') do |q|
       q.without_words = 'is'
@@ -38,7 +38,7 @@ GScraper is a web-scraping interface to various Google Services.
       q.numeric_range = 2..10
     end
 
-* Queries from URLs:
+Queries from URLs:
 
     q = GScraper::Search.query_from_url('http://www.google.com/search?as_q=ruby&as_epq=&as_oq=rails&as_ft=i&as_qdr=all&as_occt=body&as_rights=%28cc_publicdomain%7Ccc_attribute%7Ccc_sharealike%7Ccc_noncommercial%29.-%28cc_nonderived%29')
 
@@ -47,7 +47,7 @@ GScraper is a web-scraping interface to various Google Services.
     q.occurrs_within # => :title
     q.rights # => :cc_by_nc
 
-* Getting the search results:
+Getting the search results:
 
     q.first_page.select do |result|
       result.title =~ /Blog/
@@ -61,8 +61,8 @@ GScraper is a web-scraping interface to various Google Services.
 
     q.top_result # => Result
 
-* A Result object contains the rank, title, summary, cahced URL, similiar
-  query URL and link URL of the search result.
+A Result object contains the rank, title, summary, cahced URL, similiar
+query URL and link URL of the search result.
 
     page = q.page(2)
 
@@ -73,7 +73,7 @@ GScraper is a web-scraping interface to various Google Services.
     page.cached_pages # => [...]
     page.similar_queries # => [...]
 
-* Iterating over the search results:
+Iterating over the search results:
 
     q.each_on_page(2) do |result|
       puts result.title
@@ -83,7 +83,7 @@ GScraper is a web-scraping interface to various Google Services.
       puts result.url
     end
 
-* Iterating over the data within the search results:
+Iterating over the data within the search results:
 
     page.each_title do |title|
       puts title
@@ -93,7 +93,7 @@ GScraper is a web-scraping interface to various Google Services.
       puts text
     end
 
-* Selecting search results:
+Selecting search results:
 
     page.results_with do |result|
       ((result.rank > 2) && (result.rank < 10))
@@ -101,30 +101,30 @@ GScraper is a web-scraping interface to various Google Services.
 
     page.results_with_title(/Ruby/i) # => [...]
 
-* Selecting data within the search results:
+Selecting data within the search results:
 
     page.titles # => [...]
 
     page.summaries # => [...]
 
-* Selecting the data of search results based on the search result:
+Selecting the data of search results based on the search result:
 
     page.urls_of do |result|
       result.description.length > 10
     end
 
-* Selecting the Sponsored Links of a Query:
+Selecting the Sponsored Links of a Query:
 
     q.sponsored_links # => [...]
 
     q.top_sponsored_link # => SponsoredAd
 
-* Setting the User-Agent globally:
+Setting the User-Agent globally:
 
     GScraper.user_agent # => nil
     GScraper.user_agent = 'Awesome Browser v1.2'
 
-== LICENSE:
+## License
 
 GScraper - A web-scraping interface to various Google Services.
 
@@ -143,3 +143,4 @@ GNU General Public License for more details.
 You should have received a copy of the GNU General Public License
 along with this program; if not, write to the Free Software
 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+

data/Rakefile

@@ -1,16 +1,38 @@
-# -*- ruby -*-
-
 require 'rubygems'
-require 'hoe'
+require 'bundler'
+
+begin
+  Bundler.setup(:development, :doc)
+rescue Bundler::BundlerError => e
+  STDERR.puts e.message
+  STDERR.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
 
-require './tasks/spec.rb'
+require 'rake'
+require 'jeweler'
 require './lib/gscraper/version.rb'
 
-Hoe.new('gscraper', GScraper::VERSION) do |p|
-  p.rubyforge_name = 'gscraper'
-  p.developer('Postmodern', 'postmodern.mod3@gmail.com')
-  p.remote_rdoc_dir = ''
-  p.extra_deps = [['mechanize', '>=0.9.0']]
+Jeweler::Tasks.new do |gem|
+  gem.name = 'gscraper'
+  gem.version = GScraper::VERSION
+  gem.license = 'GPL-2'
+  gem.summary = %Q{GScraper is a web-scraping interface to various Google Services.}
+  gem.description = %Q{GScraper is a web-scraping interface to various Google Services.}
+  gem.email = 'postmodern.mod3@gmail.com'
+  gem.homepage = 'http://github.com/postmodern/gscraper'
+  gem.authors = ['Postmodern']
+  gem.has_rdoc = 'yard'
 end
 
-# vim: syntax=Ruby
+require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs += ['lib', 'spec']
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+  spec.spec_opts = ['--options', '.specopts']
+end
+
+task :default => :spec
+
+require 'yard'
+YARD::Rake::YardocTask.new

data/gscraper.gemspec

@@ -0,0 +1,112 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{gscraper}
+  s.version = "0.3.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Postmodern"]
+  s.date = %q{2010-07-02}
+  s.description = %q{GScraper is a web-scraping interface to various Google Services.}
+  s.email = %q{postmodern.mod3@gmail.com}
+  s.extra_rdoc_files = [
+    "ChangeLog.md",
+    "README.md"
+  ]
+  s.files = [
+    ".gitignore",
+    ".specopts",
+    ".yardopts",
+    "COPYING.txt",
+    "ChangeLog.md",
+    "Gemfile",
+    "README.md",
+    "Rakefile",
+    "gscraper.gemspec",
+    "lib/gscraper.rb",
+    "lib/gscraper/extensions.rb",
+    "lib/gscraper/extensions/uri.rb",
+    "lib/gscraper/extensions/uri/http.rb",
+    "lib/gscraper/extensions/uri/query_params.rb",
+    "lib/gscraper/gscraper.rb",
+    "lib/gscraper/has_pages.rb",
+    "lib/gscraper/licenses.rb",
+    "lib/gscraper/page.rb",
+    "lib/gscraper/search.rb",
+    "lib/gscraper/search/ajax_query.rb",
+    "lib/gscraper/search/page.rb",
+    "lib/gscraper/search/query.rb",
+    "lib/gscraper/search/result.rb",
+    "lib/gscraper/search/search.rb",
+    "lib/gscraper/search/web_query.rb",
+    "lib/gscraper/sponsored_ad.rb",
+    "lib/gscraper/sponsored_links.rb",
+    "lib/gscraper/version.rb",
+    "spec/extensions/uri/http_spec.rb",
+    "spec/extensions/uri/query_params_spec.rb",
+    "spec/gscraper_spec.rb",
+    "spec/has_pages_examples.rb",
+    "spec/has_sponsored_links_examples.rb",
+    "spec/helpers/query.rb",
+    "spec/helpers/uri.rb",
+    "spec/page_has_results_examples.rb",
+    "spec/search/ajax_query_spec.rb",
+    "spec/search/page_has_results_examples.rb",
+    "spec/search/query_spec.rb",
+    "spec/search/web_query_spec.rb",
+    "spec/spec_helper.rb"
+  ]
+  s.has_rdoc = %q{yard}
+  s.homepage = %q{http://github.com/postmodern/gscraper}
+  s.licenses = ["GPL-2"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.7}
+  s.summary = %q{GScraper is a web-scraping interface to various Google Services.}
+  s.test_files = [
+    "spec/extensions/uri/http_spec.rb",
+    "spec/extensions/uri/query_params_spec.rb",
+    "spec/gscraper_spec.rb",
+    "spec/has_pages_examples.rb",
+    "spec/has_sponsored_links_examples.rb",
+    "spec/helpers/query.rb",
+    "spec/helpers/uri.rb",
+    "spec/page_has_results_examples.rb",
+    "spec/search/ajax_query_spec.rb",
+    "spec/search/page_has_results_examples.rb",
+    "spec/search/query_spec.rb",
+    "spec/search/web_query_spec.rb",
+    "spec/spec_helper.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<json_pure>, ["~> 1.4.0"])
+      s.add_runtime_dependency(%q<mechanize>, ["~> 1.0.0"])
+      s.add_development_dependency(%q<bundler>, ["~> 0.9.19"])
+      s.add_development_dependency(%q<rake>, ["~> 0.8.7"])
+      s.add_development_dependency(%q<jeweler>, ["~> 1.4.0"])
+      s.add_development_dependency(%q<rspec>, ["~> 1.3.0"])
+    else
+      s.add_dependency(%q<json_pure>, ["~> 1.4.0"])
+      s.add_dependency(%q<mechanize>, ["~> 1.0.0"])
+      s.add_dependency(%q<bundler>, ["~> 0.9.19"])
+      s.add_dependency(%q<rake>, ["~> 0.8.7"])
+      s.add_dependency(%q<jeweler>, ["~> 1.4.0"])
+      s.add_dependency(%q<rspec>, ["~> 1.3.0"])
+    end
+  else
+    s.add_dependency(%q<json_pure>, ["~> 1.4.0"])
+    s.add_dependency(%q<mechanize>, ["~> 1.0.0"])
+    s.add_dependency(%q<bundler>, ["~> 0.9.19"])
+    s.add_dependency(%q<rake>, ["~> 0.8.7"])
+    s.add_dependency(%q<jeweler>, ["~> 1.4.0"])
+    s.add_dependency(%q<rspec>, ["~> 1.3.0"])
+  end
+end
+