gscraper 0.3.0 → 0.4.0

data/.rspec

@@ -0,0 +1 @@
+--colour --format documentation

data/ChangeLog.md

@@ -1,3 +1,25 @@
+### 0.4.0 / 2012-04-26
+
+* Switched from Bundler to rubygems-tasks ~> 0.1.
+* Switched from json_pure to json ~> 1.6.
+* Require uri-query_params ~> 0.5.
+* Require mechanize ~> 2.0.
+* Added {GScraper::Search::Blocked}.
+* Added {GScraper::Hosts}.
+* Added {GScraper::Languages}.
+* Added {GScraper::Search::Query#define}.
+* Added `:load_balance` option to {GScraper::Search::Query#initialize}, which
+  will randomize {GScraper::Search::Query#search_host}.
+* Allow `:all*` / `:with*` search options to accept a String or Array values.
+* Allow {GScraper::Search::WebQuery} and {GScraper::Search::AJAXQuery} to
+  submit queries to alternate domains via the `:search_host` option.
+* Renamed `#occurrs_within`, `:occurrs_within` to `#occurs_within`,
+  `:occurs_within`, respectively in {GScraper::Search::WebQuery}.
+* Prefer XPath over CSS-path expressions.
+* Fixed XPath expressions in {GScraper::Search::WebQuery#page}
+  (thanks Jake Auswick and Ezekiel Templin).
+* Fixed spelling errors.
+
 ### 0.3.0 / 2010-07-01
 
 * Upgraded to mechanize ~> 1.0.0.
@@ -13,7 +35,7 @@
 * Aliased {GScraper::Search::WebQuery#links_to=} to `link=`.
 * Removed `GScraper.open_uri`.
 * Removed `GScraper.open_page`.
-* Fixed the escaping/unescaping of URL query params in {URI::QueryParams}.
+* Fixed the escaping/unescaping of URL query params in `URI::QueryParams`.
 * Use `yield` instead of `block.call`, when possible.
 * All enumerable methods now return an `Enumerator` object, if no block was
   given.
@@ -57,7 +79,7 @@
 ### 0.1.8 / 2008-04-30
 
 * Added the {GScraper.user_agent_alias=} method.
-* Added {URI::HTTP::QueryParams} module.
+* Added `URI::HTTP::QueryParams` module.
 * Changed license from MIT to GPL-2.
 
 ### 0.1.7 / 2008-04-28

data/README.md

@@ -1,8 +1,8 @@
 # GScraper
 
-* [github.com/postmodern/gscraper](http://github.com/postmodern/gscraper/)
-* [github.com/postmodern/gscraper/issues](http://github.com/postmodern/gscraper/issues)
-* Postmodern (postmodern.mod3 at gmail.com)
+* [Source](https://github.com/postmodern/gscraper/)
+* [Issues](https://github.com/postmodern/gscraper/issues)
+* [Email](mailto:postmodern.mod3 at gmail.com)
 
 ## Description
   
@@ -18,11 +18,16 @@ GScraper is a web-scraping interface to various Google Services.
 
 ## Requirements
 
-* [mechanize](http://mechanize.rubyforge.org/mechanize/) ~> 1.0.0
+* [json](http://flori.github.com/json/)
+  ~> 1.6
+* [uri-query_params](https://github.com/postmodern/uri-query_params#readme)
+  ~> 0.5
+* [mechanize](http://mechanize.rubyforge.org/mechanize/)
+  ~> 2.0
 
 ## Install
 
-    $ sudo gem install gscraper
+    $ gem install gscraper
 
 ## Examples
 
@@ -44,7 +49,7 @@ Queries from URLs:
 
     q.query # => "ruby"
     q.with_words # => "rails"
-    q.occurrs_within # => :title
+    q.occurs_within # => :title
     q.rights # => :cc_by_nc
 
 Getting the search results:
@@ -128,7 +133,7 @@ Setting the User-Agent globally:
 
 GScraper - A web-scraping interface to various Google Services.
 
-Copyright (c) 2007-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
+Copyright (c) 2007-2012 Hal Brodigan (postmodern.mod3 at gmail.com)
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

data/Rakefile

@@ -1,38 +1,35 @@
 require 'rubygems'
-require 'bundler'
+require 'rake'
 
 begin
-  Bundler.setup(:development, :doc)
-rescue Bundler::BundlerError => e
-  STDERR.puts e.message
-  STDERR.puts "Run `bundle install` to install missing gems"
-  exit e.status_code
-end
+  gem 'rubygems-tasks', '~> 0.1'
+  require 'rubygems/tasks'
 
-require 'rake'
-require 'jeweler'
-require './lib/gscraper/version.rb'
-
-Jeweler::Tasks.new do |gem|
-  gem.name = 'gscraper'
-  gem.version = GScraper::VERSION
-  gem.license = 'GPL-2'
-  gem.summary = %Q{GScraper is a web-scraping interface to various Google Services.}
-  gem.description = %Q{GScraper is a web-scraping interface to various Google Services.}
-  gem.email = 'postmodern.mod3@gmail.com'
-  gem.homepage = 'http://github.com/postmodern/gscraper'
-  gem.authors = ['Postmodern']
-  gem.has_rdoc = 'yard'
+  Gem::Tasks.new
+rescue LoadError => e
+  warn e.message
+  warn "Run `gem install rubygems-tasks` to install 'rubygems/tasks'."
 end
 
-require 'spec/rake/spectask'
-Spec::Rake::SpecTask.new(:spec) do |spec|
-  spec.libs += ['lib', 'spec']
-  spec.spec_files = FileList['spec/**/*_spec.rb']
-  spec.spec_opts = ['--options', '.specopts']
-end
+begin
+  gem 'rspec', '~> 2.4'
+  require 'rspec/core/rake_task'
 
+  RSpec::Core::RakeTask.new
+rescue LoadError => e
+  task :spec do
+    abort "Please run `gem install rspec` to install RSpec."
+  end
+end
 task :default => :spec
 
-require 'yard'
-YARD::Rake::YardocTask.new
+begin
+  gem 'yard', '~> 0.6.0'
+  require 'yard'
+
+  YARD::Rake::YardocTask.new  
+rescue LoadError => e
+  task :yard do
+    abort "Please run `gem install yard` to install YARD."
+  end
+end

data/gemspec.yml

@@ -0,0 +1,20 @@
+name: gscraper
+summary: Web-scraping interface to various Google Services.
+description:
+  GScraper is a web-scraping interface to various Google Services.
+
+license: GPL-2
+authors: Postmodern
+email: postmodern.mod3@gmail.com
+homepage: https://github.com/postmodern/gscraper
+has_yard: true
+
+dependencies:
+  json: ~> 1.6
+  uri-query_params: ~> 0.5
+  mechanize: ~> 2.0
+
+development_dependencies:
+  rubygems-tasks: ~> 0.1
+  rspec: ~> 2.4
+  yard: ~> 0.6

data/gscraper.gemspec

@@ -1,112 +1,127 @@
-# Generated by jeweler
-# DO NOT EDIT THIS FILE DIRECTLY
-# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
-# -*- encoding: utf-8 -*-
-
-Gem::Specification.new do |s|
-  s.name = %q{gscraper}
-  s.version = "0.3.0"
-
-  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = ["Postmodern"]
-  s.date = %q{2010-07-02}
-  s.description = %q{GScraper is a web-scraping interface to various Google Services.}
-  s.email = %q{postmodern.mod3@gmail.com}
-  s.extra_rdoc_files = [
-    "ChangeLog.md",
-    "README.md"
-  ]
-  s.files = [
-    ".gitignore",
-    ".specopts",
-    ".yardopts",
-    "COPYING.txt",
-    "ChangeLog.md",
-    "Gemfile",
-    "README.md",
-    "Rakefile",
-    "gscraper.gemspec",
-    "lib/gscraper.rb",
-    "lib/gscraper/extensions.rb",
-    "lib/gscraper/extensions/uri.rb",
-    "lib/gscraper/extensions/uri/http.rb",
-    "lib/gscraper/extensions/uri/query_params.rb",
-    "lib/gscraper/gscraper.rb",
-    "lib/gscraper/has_pages.rb",
-    "lib/gscraper/licenses.rb",
-    "lib/gscraper/page.rb",
-    "lib/gscraper/search.rb",
-    "lib/gscraper/search/ajax_query.rb",
-    "lib/gscraper/search/page.rb",
-    "lib/gscraper/search/query.rb",
-    "lib/gscraper/search/result.rb",
-    "lib/gscraper/search/search.rb",
-    "lib/gscraper/search/web_query.rb",
-    "lib/gscraper/sponsored_ad.rb",
-    "lib/gscraper/sponsored_links.rb",
-    "lib/gscraper/version.rb",
-    "spec/extensions/uri/http_spec.rb",
-    "spec/extensions/uri/query_params_spec.rb",
-    "spec/gscraper_spec.rb",
-    "spec/has_pages_examples.rb",
-    "spec/has_sponsored_links_examples.rb",
-    "spec/helpers/query.rb",
-    "spec/helpers/uri.rb",
-    "spec/page_has_results_examples.rb",
-    "spec/search/ajax_query_spec.rb",
-    "spec/search/page_has_results_examples.rb",
-    "spec/search/query_spec.rb",
-    "spec/search/web_query_spec.rb",
-    "spec/spec_helper.rb"
-  ]
-  s.has_rdoc = %q{yard}
-  s.homepage = %q{http://github.com/postmodern/gscraper}
-  s.licenses = ["GPL-2"]
-  s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.7}
-  s.summary = %q{GScraper is a web-scraping interface to various Google Services.}
-  s.test_files = [
-    "spec/extensions/uri/http_spec.rb",
-    "spec/extensions/uri/query_params_spec.rb",
-    "spec/gscraper_spec.rb",
-    "spec/has_pages_examples.rb",
-    "spec/has_sponsored_links_examples.rb",
-    "spec/helpers/query.rb",
-    "spec/helpers/uri.rb",
-    "spec/page_has_results_examples.rb",
-    "spec/search/ajax_query_spec.rb",
-    "spec/search/page_has_results_examples.rb",
-    "spec/search/query_spec.rb",
-    "spec/search/web_query_spec.rb",
-    "spec/spec_helper.rb"
-  ]
-
-  if s.respond_to? :specification_version then
-    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
-    s.specification_version = 3
-
-    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<json_pure>, ["~> 1.4.0"])
-      s.add_runtime_dependency(%q<mechanize>, ["~> 1.0.0"])
-      s.add_development_dependency(%q<bundler>, ["~> 0.9.19"])
-      s.add_development_dependency(%q<rake>, ["~> 0.8.7"])
-      s.add_development_dependency(%q<jeweler>, ["~> 1.4.0"])
-      s.add_development_dependency(%q<rspec>, ["~> 1.3.0"])
-    else
-      s.add_dependency(%q<json_pure>, ["~> 1.4.0"])
-      s.add_dependency(%q<mechanize>, ["~> 1.0.0"])
-      s.add_dependency(%q<bundler>, ["~> 0.9.19"])
-      s.add_dependency(%q<rake>, ["~> 0.8.7"])
-      s.add_dependency(%q<jeweler>, ["~> 1.4.0"])
-      s.add_dependency(%q<rspec>, ["~> 1.3.0"])
+# encoding: utf-8
+
+require 'yaml'
+
+Gem::Specification.new do |gemspec|
+  files = if File.directory?('.git')
+            `git ls-files`.split($/)
+          elsif File.directory?('.hg')
+            `hg manifest`.split($/)
+          elsif File.directory?('.svn')
+            `svn ls -R`.split($/).select { |path| File.file?(path) }
+          else
+            Dir['{**/}{.*,*}'].select { |path| File.file?(path) }
+          end
+
+  filter_files = lambda { |paths|
+    case paths
+    when Array
+      (files & paths)
+    when String
+      (files & Dir[paths])
     end
-  else
-    s.add_dependency(%q<json_pure>, ["~> 1.4.0"])
-    s.add_dependency(%q<mechanize>, ["~> 1.0.0"])
-    s.add_dependency(%q<bundler>, ["~> 0.9.19"])
-    s.add_dependency(%q<rake>, ["~> 0.8.7"])
-    s.add_dependency(%q<jeweler>, ["~> 1.4.0"])
-    s.add_dependency(%q<rspec>, ["~> 1.3.0"])
+  }
+
+  version = {
+    :file => 'lib/gscraper/version.rb',
+    :constant => 'GScraper::VERSION'
+  }
+
+  defaults = {
+    'name' => File.basename(File.dirname(__FILE__)),
+    'files' => files,
+    'executables' => filter_files['bin/*'].map { |path| File.basename(path) },
+    'test_files' => filter_files['{test/{**/}*_test.rb,spec/{**/}*_spec.rb}'],
+    'extra_doc_files' => filter_files['*.{txt,rdoc,md,markdown,tt,textile}'],
+  }
+
+  metadata = defaults.merge(YAML.load_file('gemspec.yml'))
+
+  gemspec.name = metadata.fetch('name',defaults[:name])
+  gemspec.version = if metadata['version']
+                      metadata['version']
+                    elsif File.file?(version[:file])
+                      require File.join('.',version[:file])
+                      eval(version[:constant])
+                    end
+
+  gemspec.summary = metadata.fetch('summary',metadata['description'])
+  gemspec.description = metadata.fetch('description',metadata['summary'])
+
+  case metadata['license']
+  when Array
+    gemspec.licenses = metadata['license']
+  when String
+    gemspec.license = metadata['license']
+  end
+
+  case metadata['authors']
+  when Array
+    gemspec.authors = metadata['authors']
+  when String
+    gemspec.author = metadata['authors']
+  end
+
+  gemspec.email = metadata['email']
+  gemspec.homepage = metadata['homepage']
+
+  case metadata['require_paths']
+  when Array
+    gemspec.require_paths = metadata['require_paths']
+  when String
+    gemspec.require_path = metadata['require_paths']
+  end
+
+  gemspec.files = filter_files[metadata['files']]
+
+  gemspec.executables = metadata['executables']
+  gemspec.extensions = metadata['extensions']
+
+  if Gem::VERSION < '1.7.'
+    gemspec.default_executable = gemspec.executables.first
   end
-end
 
+  gemspec.test_files = filter_files[metadata['test_files']]
+
+  unless gemspec.files.include?('.document')
+    gemspec.extra_rdoc_files = metadata['extra_doc_files']
+  end
+
+  gemspec.post_install_message = metadata['post_install_message']
+  gemspec.requirements = metadata['requirements']
+
+  if gemspec.respond_to?(:required_ruby_version=)
+    gemspec.required_ruby_version = metadata['required_ruby_version']
+  end
+
+  if gemspec.respond_to?(:required_rubygems_version=)
+    gemspec.required_rubygems_version = metadata['required_ruby_version']
+  end
+
+  parse_versions = lambda { |versions|
+    case versions
+    when Array
+      versions.map { |v| v.to_s }
+    when String
+      versions.split(/,\s*/)
+    end
+  }
+
+  if metadata['dependencies']
+    metadata['dependencies'].each do |name,versions|
+      gemspec.add_dependency(name,parse_versions[versions])  
+    end
+  end
+
+  if metadata['runtime_dependencies']
+    metadata['runtime_dependencies'].each do |name,versions|
+      gemspec.add_runtime_dependency(name,parse_versions[versions])  
+    end
+  end
+
+  if metadata['development_dependencies']
+    metadata['development_dependencies'].each do |name,versions|
+      gemspec.add_development_dependency(name,parse_versions[versions])  
+    end
+  end
+end

data/lib/gscraper.rb

@@ -1,7 +1,7 @@
 #
 # GScraper - A web-scraping interface to various Google Services.
 #
-# Copyright (c) 2007-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
+# Copyright (c) 2007-2012 Hal Brodigan (postmodern.mod3 at gmail.com)
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

data/lib/gscraper/gscraper.rb

@@ -1,7 +1,7 @@
 #
 # GScraper - A web-scraping interface to various Google Services.
 #
-# Copyright (c) 2007-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
+# Copyright (c) 2007-2012 Hal Brodigan (postmodern.mod3 at gmail.com)
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -21,7 +21,6 @@
 require 'uri/http'
 require 'mechanize'
 require 'nokogiri'
-require 'open-uri'
 
 module GScraper
   # Common proxy port.
@@ -32,8 +31,13 @@ module GScraper
   #
   # @return [Hash]
   #
-  def GScraper.proxy
-    @@gscraper_proxy ||= {:host => nil, :port => COMMON_PROXY_PORT, :user => nil, :password => nil}
+  def self.proxy
+    @@gscraper_proxy ||= {
+      :host     => nil,
+      :port     => COMMON_PROXY_PORT,
+      :user     => nil,
+      :password => nil
+    }
   end
 
   #
@@ -54,13 +58,13 @@ module GScraper
   # @option proxy_info [String] :password
   #   The password to login with.
   #
-  def GScraper.proxy_uri(proxy_info=GScraper.proxy)
-    if GScraper.proxy[:host]
+  def self.proxy_uri(proxy=self.proxy)
+    if proxy[:host]
       return URI::HTTP.build(
-        :host => GScraper.proxy[:host],
-        :port => GScraper.proxy[:port],
-        :userinfo => "#{GScraper.proxy[:user]}:#{GScraper.proxy[:password]}",
-        :path => '/'
+        :host     => proxy[:host],
+        :port     => proxy[:port],
+        :userinfo => "#{proxy[:user]}:#{proxy[:password]}",
+        :path     => '/'
       )
     end
   end
@@ -70,7 +74,7 @@ module GScraper
   #
   # @return [Array<String>]
   #
-  def GScraper.user_agent_aliases
+  def self.user_agent_aliases
     Mechanize::AGENT_ALIASES
   end
 
@@ -79,8 +83,8 @@ module GScraper
   #
   # @return [String]
   #
-  def GScraper.user_agent
-    @@gscraper_user_agent ||= GScraper.user_agent_aliases['Windows IE 6']
+  def self.user_agent
+    @@gscraper_user_agent ||= self.user_agent_aliases['Windows IE 6']
   end
 
   #
@@ -92,7 +96,7 @@ module GScraper
   # @return [String]
   #   The new User-Agent string.
   #
-  def GScraper.user_agent=(agent)
+  def self.user_agent=(agent)
     @@gscraper_user_agent = agent
   end
 
@@ -105,8 +109,8 @@ module GScraper
   # @return [String]
   #   The new User-Agent string.
   # 
-  def GScraper.user_agent_alias=(name)
-    @@gscraper_user_agent = GScraper.user_agent_aliases[name.to_s]
+  def self.user_agent_alias=(name)
+    @@gscraper_user_agent = self.user_agent_aliases[name.to_s]
   end
 
   #
@@ -143,18 +147,18 @@ module GScraper
   #   GScraper.web_agent(:user_agent_alias => 'Linux Mozilla')
   #   GScraper.web_agent(:user_agent => 'Google Bot')
   #
-  def GScraper.web_agent(options={})
+  def self.web_agent(options={})
     agent = Mechanize.new
 
     if options[:user_agent_alias]
       agent.user_agent_alias = options[:user_agent_alias]
     elsif options[:user_agent]
       agent.user_agent = options[:user_agent]
-    elsif GScraper.user_agent
-      agent.user_agent = GScraper.user_agent
+    elsif user_agent
+      agent.user_agent = self.user_agent
     end
 
-    proxy = (options[:proxy] || GScraper.proxy)
+    proxy = (options[:proxy] || self.proxy)
     if proxy[:host]
       agent.set_proxy(proxy[:host],proxy[:port],proxy[:user],proxy[:password])
     end