gscraper 0.3.0 → 0.4.0

data/lib/gscraper/version.rb

@@ -1,7 +1,7 @@
 #
 # GScraper - A web-scraping interface to various Google Services.
 #
-# Copyright (c) 2007-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
+# Copyright (c) 2007-2012 Hal Brodigan (postmodern.mod3 at gmail.com)
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -20,5 +20,5 @@
 
 module GScraper
   # The version of GScraper
-  VERSION = '0.3.0'
+  VERSION = '0.4.0'
 end

data/spec/languages_spec.rb

@@ -0,0 +1,28 @@
+require 'spec_helper'
+require 'gscraper/languages'
+
+describe GScraper::Languages do
+  it "should lookup the language for a locale" do
+    GScraper::Languages.find('es').should == 'es'
+  end
+
+  it "should lookup the language for locale_country" do
+    GScraper::Languages.find('es_AR').should == 'es'
+  end
+
+  it "should lookup the language for a locale@alias" do
+    GScraper::Languages.find('en@quot').should == 'en'
+  end
+
+  it "should map zh_CN* to zh-CN" do
+    GScraper::Languages.find('zh_CN').should == 'zh-CN'
+  end
+
+  it "should map zh_TW* to zh-TW" do
+    GScraper::Languages.find('zh_TW').should == 'zh-TW'
+  end
+
+  it "should auto-detect the native language" do
+    GScraper::Languages.native.should_not be_nil
+  end
+end

data/spec/search/ajax_query_spec.rb

@@ -50,7 +50,8 @@ describe GScraper::Search::AJAXQuery do
 
     it "should have a default 'hl' query-param" do
       hl = @uri.query_params['hl']
-      hl.should == GScraper::Search::AJAXQuery::DEFAULT_LANGUAGE
+
+      hl.should_not be_empty
     end
 
     it "should have a default 'gss' query-param of '.com'" do

data/spec/search/query_spec.rb

@@ -3,6 +3,35 @@ require 'spec_helper'
 require 'gscraper/search/query'
 
 describe GScraper::Search::Query do
+  it "should have a default host of www.google.com" do
+    query = GScraper::Search::Query.new
+
+    query.search_host.should == 'www.google.com'
+  end
+
+  it "should allow using alternate hosts" do
+    alternate_host = 'www.google.com.ar'
+    query = GScraper::Search::Query.new(
+      :search_host => alternate_host
+    )
+
+    query.search_host.should == alternate_host
+  end
+
+  it "should use random hosts if load balancing is enabled" do
+    query = GScraper::Search::Query.new(:load_balance => true)
+
+    host1 = query.search_host
+    host2 = query.search_host
+
+    host1.should_not == host2
+  end
+
+  it "should have a default language" do
+    query = GScraper::Search::Query.new
+
+    query.language.should_not be_nil
+  end
 
   it "should support basic queries" do
     expr = 'ruby -blog'

data/spec/search/web_query_spec.rb

@@ -11,7 +11,9 @@ describe GScraper::Search::WebQuery do
   include Helpers
 
   before(:all) do
-    @query = GScraper::Search::WebQuery.new(:query => Helpers::DEFAULT_QUERY)
+    @query = GScraper::Search::WebQuery.new(
+      :query => Helpers::DEFAULT_QUERY
+    )
     @page = @query.first_page
     @links = @query.sponsored_links
   end
@@ -30,6 +32,24 @@ describe GScraper::Search::WebQuery do
       @uri.class.should == URI::HTTP
     end
 
+    it "should have a default host of www.google.com" do
+      @uri.host.should == 'www.google.com'
+    end
+
+    it "should allow using alternate hosts" do
+      other_host = 'www.google.com.ar'
+      other_query = GScraper::Search::WebQuery.new(
+        :search_host => other_host,
+        :query => Helpers::DEFAULT_QUERY
+      )
+
+      other_query.search_url.host.should == other_host
+    end
+
+    it "should have a path of /search" do
+      @uri.path.should == '/search'
+    end
+
     it "should have a 'q' query-param" do
       @uri.query_params['q'].should == Helpers::DEFAULT_QUERY
     end

data/spec/spec_helper.rb

@@ -1,12 +1,2 @@
-require 'rubygems'
-require 'bundler'
-
-begin
-  Bundler.setup(:runtime, :test)
-rescue Bundler::BundlerError => e
-  STDERR.puts e.message
-  STDERR.puts "Run `bundle install` to install missing gems"
-  exit e.status_code
-end
-
-require 'spec'
+gem 'rspec', '~> 2.4'
+require 'rspec'

metadata

@@ -1,142 +1,141 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: gscraper
-version: !ruby/object:Gem::Version 
-  prerelease: false
-  segments: 
-  - 0
-  - 3
-  - 0
-  version: 0.3.0
+version: !ruby/object:Gem::Version
+  version: 0.4.0
+  prerelease: 
 platform: ruby
-authors: 
+authors:
 - Postmodern
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2010-07-02 00:00:00 -07:00
-default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  name: json_pure
-  requirement: &id001 !ruby/object:Gem::Requirement 
+date: 2012-04-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 1
-        - 4
-        - 0
-        version: 1.4.0
+      - !ruby/object:Gem::Version
+        version: '1.6'
   type: :runtime
   prerelease: false
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: mechanize
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: uri-query_params
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 1
-        - 0
-        - 0
-        version: 1.0.0
+      - !ruby/object:Gem::Version
+        version: '0.5'
   type: :runtime
   prerelease: false
-  version_requirements: *id002
-- !ruby/object:Gem::Dependency 
-  name: bundler
-  requirement: &id003 !ruby/object:Gem::Requirement 
+  version_requirements: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 0
-        - 9
-        - 19
-        version: 0.9.19
-  type: :development
+      - !ruby/object:Gem::Version
+        version: '0.5'
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
   prerelease: false
-  version_requirements: *id003
-- !ruby/object:Gem::Dependency 
-  name: rake
-  requirement: &id004 !ruby/object:Gem::Requirement 
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rubygems-tasks
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 0
-        - 8
-        - 7
-        version: 0.8.7
+      - !ruby/object:Gem::Version
+        version: '0.1'
   type: :development
   prerelease: false
-  version_requirements: *id004
-- !ruby/object:Gem::Dependency 
-  name: jeweler
-  requirement: &id005 !ruby/object:Gem::Requirement 
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 1
-        - 4
-        - 0
-        version: 1.4.0
+      - !ruby/object:Gem::Version
+        version: '2.4'
   type: :development
   prerelease: false
-  version_requirements: *id005
-- !ruby/object:Gem::Dependency 
-  name: rspec
-  requirement: &id006 !ruby/object:Gem::Requirement 
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.4'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 1
-        - 3
-        - 0
-        version: 1.3.0
+      - !ruby/object:Gem::Version
+        version: '0.6'
   type: :development
   prerelease: false
-  version_requirements: *id006
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.6'
 description: GScraper is a web-scraping interface to various Google Services.
 email: postmodern.mod3@gmail.com
 executables: []
-
 extensions: []
-
-extra_rdoc_files: 
+extra_rdoc_files:
+- COPYING.txt
 - ChangeLog.md
 - README.md
-files: 
+files:
 - .gitignore
-- .specopts
+- .rspec
 - .yardopts
 - COPYING.txt
 - ChangeLog.md
-- Gemfile
 - README.md
 - Rakefile
+- gemspec.yml
 - gscraper.gemspec
 - lib/gscraper.rb
-- lib/gscraper/extensions.rb
-- lib/gscraper/extensions/uri.rb
-- lib/gscraper/extensions/uri/http.rb
-- lib/gscraper/extensions/uri/query_params.rb
 - lib/gscraper/gscraper.rb
 - lib/gscraper/has_pages.rb
+- lib/gscraper/hosts.rb
+- lib/gscraper/languages.rb
 - lib/gscraper/licenses.rb
 - lib/gscraper/page.rb
 - lib/gscraper/search.rb
 - lib/gscraper/search/ajax_query.rb
+- lib/gscraper/search/exceptions.rb
+- lib/gscraper/search/exceptions/blocked.rb
 - lib/gscraper/search/page.rb
 - lib/gscraper/search/query.rb
 - lib/gscraper/search/result.rb
@@ -145,63 +144,46 @@ files:
 - lib/gscraper/sponsored_ad.rb
 - lib/gscraper/sponsored_links.rb
 - lib/gscraper/version.rb
-- spec/extensions/uri/http_spec.rb
-- spec/extensions/uri/query_params_spec.rb
 - spec/gscraper_spec.rb
 - spec/has_pages_examples.rb
 - spec/has_sponsored_links_examples.rb
 - spec/helpers/query.rb
 - spec/helpers/uri.rb
+- spec/languages_spec.rb
 - spec/page_has_results_examples.rb
 - spec/search/ajax_query_spec.rb
 - spec/search/page_has_results_examples.rb
 - spec/search/query_spec.rb
 - spec/search/web_query_spec.rb
 - spec/spec_helper.rb
-has_rdoc: yard
-homepage: http://github.com/postmodern/gscraper
-licenses: 
+homepage: https://github.com/postmodern/gscraper
+licenses:
 - GPL-2
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 734300353
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      segments: 
-      - 0
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.3.7
+rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
-summary: GScraper is a web-scraping interface to various Google Services.
-test_files: 
-- spec/extensions/uri/http_spec.rb
-- spec/extensions/uri/query_params_spec.rb
+summary: Web-scraping interface to various Google Services.
+test_files:
 - spec/gscraper_spec.rb
-- spec/has_pages_examples.rb
-- spec/has_sponsored_links_examples.rb
-- spec/helpers/query.rb
-- spec/helpers/uri.rb
-- spec/page_has_results_examples.rb
+- spec/languages_spec.rb
 - spec/search/ajax_query_spec.rb
-- spec/search/page_has_results_examples.rb
 - spec/search/query_spec.rb
 - spec/search/web_query_spec.rb
-- spec/spec_helper.rb