gscraper 0.3.0 → 0.4.0

data/lib/gscraper/{extensions.rb → search/exceptions.rb}

@@ -1,7 +1,7 @@
 #
 # GScraper - A web-scraping interface to various Google Services.
 #
-# Copyright (c) 2007-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
+# Copyright (c) 2007-2012 Hal Brodigan (postmodern.mod3 at gmail.com)
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -18,4 +18,4 @@
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 #
 
-require 'gscraper/extensions/uri'
+require 'gscraper/search/exceptions/blocked'

data/lib/gscraper/{extensions/uri.rb → search/exceptions/blocked.rb}

@@ -1,7 +1,7 @@
 #
 # GScraper - A web-scraping interface to various Google Services.
 #
-# Copyright (c) 2007-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
+# Copyright (c) 2007-2012 Hal Brodigan (postmodern.mod3 at gmail.com)
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -18,4 +18,12 @@
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 #
 
-require 'gscraper/extensions/uri/http'
+module GScraper
+  module Search
+    #
+    # @since 0.4.0
+    #
+    class Blocked < RuntimeError
+    end
+  end
+end

data/lib/gscraper/search/page.rb

@@ -1,7 +1,7 @@
 #
 # GScraper - A web-scraping interface to various Google Services.
 #
-# Copyright (c) 2007-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
+# Copyright (c) 2007-2012 Hal Brodigan (postmodern.mod3 at gmail.com)
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -51,15 +51,13 @@ module GScraper
       #   end
       #
       def results_with_title(title)
-        unless block_given?
-          enum_for(:results_with_title,title)
-        else
-          results_with do |result|
-            if result.title.match(title)
-              yield result
-
-              true
-            end
+        return enum_for(:results_with_title,title) unless block_given?
+
+        results_with do |result|
+          if result.title.match(title)
+            yield result
+
+            true
           end
         end
       end
@@ -88,15 +86,13 @@ module GScraper
       #   end
       #
       def results_with_url(url)
-        unless block_given?
-          enum_for(:results_with_url,url)
-        else
-          results_with do |result|
-            if result.url.match(url)
-              yield result
-
-              true
-            end
+        return enum_for(:results_with_url,url) unless block_given?
+
+        results_with do |result|
+          if result.url.match(url)
+            yield result
+
+            true
           end
         end
       end
@@ -125,15 +121,13 @@ module GScraper
       #   end
       #
       def results_with_summary(summary)
-        unless block_given?
-          enum_for(:results_with_summary,summary)
-        else
-          results_with do |result|
-            if result.summary.match(summary)
-              yield result
-
-              true
-            end
+        return enum_for(:results_with_summary,summary) unless block_given?
+
+        results_with do |result|
+          if result.summary.match(summary)
+            yield result
+
+            true
           end
         end
       end
@@ -155,11 +149,9 @@ module GScraper
       #   each_rank { |rank| puts rank }
       #
       def each_rank
-        unless block_given?
-          enum_for(:each_rank)
-        else
-          each { |result| yield result.rank }
-        end
+        return enum_for(:each_rank) unless block_given?
+
+        each { |result| yield result.rank }
       end
 
       #
@@ -179,11 +171,9 @@ module GScraper
       #   each_title { |title| puts title }
       #
       def each_title
-        unless block_given?
-          enum_for(:each_title)
-        else
-          each { |result| yield result.title }
-        end
+        return enum_for(:each_title) unless block_given?
+
+        each { |result| yield result.title }
       end
 
       #
@@ -203,11 +193,9 @@ module GScraper
       #   each_url { |url| puts url }
       #
       def each_url
-        unless block_given?
-          enum_for(:each_url)
-        else
-          each { |result| yield result.url }
-        end
+        return enum_for(:each_url) unless block_given?
+
+        each { |result| yield result.url }
       end
 
       #
@@ -227,11 +215,9 @@ module GScraper
       #   each_summary { |summary| puts summary }
       #
       def each_summary
-        unless block_given?
-          enum_for(:each_summary)
-        else
-          each { |result| yield result.summary }
-        end
+        return enum_for(:each_summary) unless block_given?
+
+        each { |result| yield result.summary }
       end
 
       #
@@ -251,12 +237,10 @@ module GScraper
       #   each_cached_url { |cached_url| puts cached_url }
       #
       def each_cached_url
-        unless block_given?
-          enum_for(:each_cached_url)
-        else
-          each do |result|
-            yield result.cached_url if result.cached_url
-          end
+        return enum_for(:each_cached_url) unless block_given?
+
+        each do |result|
+          yield result.cached_url if result.cached_url
         end
       end
 
@@ -277,12 +261,10 @@ module GScraper
       #   each_cached_page { |page| puts page.readlines }
       #
       def each_cached_page
-        unless block_given?
-          enum_for(:each_cached_page)
-        else
-          each do |result|
-            yield result.cached_page if result.cached_page
-          end
+        return enum_for(:each_cached_page) unless block_given?
+
+        each do |result|
+          yield result.cached_page if result.cached_page
         end
       end
 
@@ -303,12 +285,10 @@ module GScraper
       #   each_similar_url { |similar_url| puts similar_url }
       #
       def each_similar_url
-        unless block_given?
-          enum_for(:each_similar_url)
-        else
-          each do |result|
-            yield result.similar_url if result.similar_url
-          end
+        return enum_for(:each_similar_url) unless block_given?
+
+        each do |result|
+          yield result.similar_url if result.similar_url
         end
       end
 
@@ -421,7 +401,7 @@ module GScraper
       end
 
       #
-      # Returns the urls of the results that match the given block.
+      # Returns the URLs of the results that match the given block.
       #
       # @yield [result]
       #   The given block will be used to filter the results in the page.

data/lib/gscraper/search/query.rb

@@ -1,7 +1,7 @@
 #
 # GScraper - A web-scraping interface to various Google Services.
 #
-# Copyright (c) 2007-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
+# Copyright (c) 2007-2012 Hal Brodigan (postmodern.mod3 at gmail.com)
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -18,12 +18,8 @@
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 #
 
-require 'gscraper/search/result'
-require 'gscraper/search/page'
-require 'gscraper/sponsored_ad'
-require 'gscraper/sponsored_links'
-require 'gscraper/extensions/uri'
-require 'gscraper/has_pages'
+require 'gscraper/hosts'
+require 'gscraper/languages'
 require 'gscraper/licenses'
 require 'gscraper/gscraper'
 
@@ -31,9 +27,21 @@ module GScraper
   module Search
     class Query
 
+      # Web Search sub-domain
+      SUB_DOMAIN = 'www'
+
+      # Default host to submit queries to
+      DEFAULT_HOST = "#{SUB_DOMAIN}.#{Hosts::PRIMARY_DOMAIN}"
+
+      # The host to submit queries to
+      attr_writer :search_host
+
       # Search query
       attr_accessor :query
 
+      # The search language
+      attr_accessor :language
+
       # Search 'link' modifier
       attr_accessor :link
 
@@ -79,15 +87,24 @@ module GScraper
       # Search for results containing numbers between the range
       attr_accessor :numeric_range
 
+      # Search for results containing the definitions of the keywords
+      attr_accessor :define
+
       #
       # Creates a new query.
       #
       # @param [Hash] options
       #   Additional options.
       #
+      # @option options [String] :search_host (www.google.com)
+      #   The host to submit queries to.
+      #
       # @option options [String] :query
       #   The search query.
       #
+      # @option options [Symbol, String] :language (Languages.native)
+      #   The search language.
+      #
       # @option options [String] :link
       #   Search for results which link to the specified URI.
       #
@@ -103,20 +120,20 @@ module GScraper
       # @option options [String] :filetype
       #   Limit results to those with the specified file-type.
       #
-      # @option options [String, Array] :allintitle
+      # @option options [Array, String] :allintitle
       #   Search for results with all of the keywords appearing in the
       #   title.
       #
       # @option options [String] :intitle
       #   Search for results with the keyword appearing in the title.
       #
-      # @option options [String, Array] :allintext
+      # @option options [Array, String] :allintext
       #   Search for results with all of the keywords appearing in the text.
       #
       # @option options [String] :intext
       #   Search for results with the keyword appearing in the text.
       #
-      # @option options [String, Array] :allinanchor
+      # @option options [Array, String] :allinanchor
       #   Search for results with all of the keywords appearing in the
       #   text of links.
       #
@@ -127,13 +144,13 @@ module GScraper
       # @option options [String] :exact_phrase
       #   Search for results containing the specified exact phrase.
       #
-      # @option options [String, Array] :with_words
+      # @option options [Array, String] :with_words
       #   Search for results containing all of the specified words.
       #
-      # @option options [String, Array] :without_words
+      # @option options [Array, String] :without_words
       #   Search for results not containing any of the specified words.
       #
-      # @option options [Range] :numeric_range
+      # @option options [Range, Array, String] :numeric_range
       #   Search for results contain numbers that fall within the
       #   specified Range.
       #
@@ -141,6 +158,10 @@ module GScraper
       #   Search for results containing the definition of the specified
       #   keyword.
       #
+      # @option options [Boolean] :load_balance (false)
+      #   Specifies whether to distribute queries accross multiple Google
+      #   domains.
+      #
       # @yield [query]
       #   If a block is given, it will be passed the new query.
       #
@@ -151,33 +172,54 @@ module GScraper
       #   The new query.
       #
       def initialize(options={})
-        @query = options[:query]
+        @search_host = options.fetch(:search_host,DEFAULT_HOST)
+
+        @query    = options[:query]
+        @language = options.fetch(:language,Languages.native)
 
-        @link = options[:link]
-        @related = options[:related]
-        @info = options[:info]
-        @site = options[:site]
+        @link     = options[:link]
+        @related  = options[:related]
+        @info     = options[:info]
+        @site     = options[:site]
         @filetype = options[:filetype]
 
-        @allintitle = options[:allintitle]
-        @intitle = options[:intitle]
-        @allinurl = options[:allinurl]
-        @inurl = options[:inurl]
-        @allintext = options[:allintext]
-        @intext = options[:intext]
+        @allintitle  = options[:allintitle]
+        @intitle     = options[:intitle]
+        @allinurl    = options[:allinurl]
+        @inurl       = options[:inurl]
+        @allintext   = options[:allintext]
+        @intext      = options[:intext]
         @allinanchor = options[:allinanchor]
-        @inanchor = options[:inanchor]
+        @inanchor    = options[:inanchor]
 
-        @exact_phrase = options[:exact_phrase]
-        @with_words = options[:with_words]
+        @exact_phrase  = options[:exact_phrase]
+        @with_words    = options[:with_words]
         @without_words = options[:without_words]
 
         @numeric_range = options[:numeric_range]
-        @define = options[:define]
+        @define        = options[:define]
+
+        @load_balance = options.fetch(:load_balance,false)
 
         yield self if block_given?
       end
 
+      #
+      # The host to submit queries to.
+      #
+      # @return [String]
+      #   The host to submit queries to.
+      #
+      # @since 0.4.0
+      #
+      def search_host
+        if @load_balance
+          Hosts::DOMAINS[rand(Hosts::DOMAINS.length)]
+        else
+          @search_host
+        end
+      end
+
       #
       # The query expression.
       #
@@ -222,16 +264,25 @@ module GScraper
           expr << "\"#{@exact_phrase}\""
         end
 
-        if @with_words.kind_of?(Array)
+        case @with_words
+        when String
+          expr << @with_words
+        when Enumerable
           expr << @with_words.join(' OR ')
         end
-        
-        if @without_words.kind_of?(Array)
+
+        case @without_words
+        when String
+          expr << @without_words
+        when Enumerable
           expr << @without_words.map { |word| "-#{word}" }.join(' ')
         end
 
-        if @numeric_range.kind_of?(Range)
-          expr << "#{@numeric_range.begin}..#{@numeric_range.end}"
+        case @numeric_range
+        when String
+          expr << @numeric_range
+        when Range, Array
+          expr << "#{@numeric_range.first}..#{@numeric_range.last}"
         end
 
         return expr.join(' ')
@@ -240,7 +291,7 @@ module GScraper
       protected
 
       #
-      # Formats the value for a search modifer.
+      # Formats the value for a search modifier.
       #
       # @param [Regexp, String]
       #   The value for the search modifier.
@@ -249,10 +300,11 @@ module GScraper
       #   The formatted value.
       #
       def format_modifier(value)
-        if value.kind_of?(Regexp)
-          return value.source
+        case value
+        when Range
+          value.source
         else
-          return value.to_s
+          value.to_s
         end
       end
 
@@ -266,13 +318,7 @@ module GScraper
       #   The formatted value.
       #
       def format_options(value)
-        if value.kind_of?(Array)
-          return value.map { |element|
-            format_modifier(element)
-          }.join(' ')
-        else
-          return format_modifier(value)
-        end
+        Array(value).map(&method(:format_modifier)).join(' ')
       end
 
     end