gscraper 0.2.4 → 0.3.0

data/lib/gscraper/search/query.rb

@@ -1,5 +1,4 @@
 #
-#--
 # GScraper - A web-scraping interface to various Google Services.
 #
 # Copyright (c) 2007-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
@@ -17,7 +16,6 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
-#++
 #
 
 require 'gscraper/search/result'
@@ -82,10 +80,77 @@ module GScraper
       attr_accessor :numeric_range
 
       #
-      # Creates a new Query object from the given search options. If a
-      # block is given, it will be passed the newly created Query object.
+      # Creates a new query.
       #
-      def initialize(options={},&block)
+      # @param [Hash] options
+      #   Additional options.
+      #
+      # @option options [String] :query
+      #   The search query.
+      #
+      # @option options [String] :link
+      #   Search for results which link to the specified URI.
+      #
+      # @option options [String] :related
+      #   Search for results which relate to the specified URI.
+      #
+      # @option options [String] :info
+      #   Return information about the specified URI.
+      #
+      # @option options [String] :site
+      #   Limit results to the specified site.
+      #
+      # @option options [String] :filetype
+      #   Limit results to those with the specified file-type.
+      #
+      # @option options [String, Array] :allintitle
+      #   Search for results with all of the keywords appearing in the
+      #   title.
+      #
+      # @option options [String] :intitle
+      #   Search for results with the keyword appearing in the title.
+      #
+      # @option options [String, Array] :allintext
+      #   Search for results with all of the keywords appearing in the text.
+      #
+      # @option options [String] :intext
+      #   Search for results with the keyword appearing in the text.
+      #
+      # @option options [String, Array] :allinanchor
+      #   Search for results with all of the keywords appearing in the
+      #   text of links.
+      #
+      # @option options [String] :inanchor
+      #   Search for results with the keyword appearing in the text of
+      #   links.
+      #
+      # @option options [String] :exact_phrase
+      #   Search for results containing the specified exact phrase.
+      #
+      # @option options [String, Array] :with_words
+      #   Search for results containing all of the specified words.
+      #
+      # @option options [String, Array] :without_words
+      #   Search for results not containing any of the specified words.
+      #
+      # @option options [Range] :numeric_range
+      #   Search for results contain numbers that fall within the
+      #   specified Range.
+      #
+      # @option options [String] :define
+      #   Search for results containing the definition of the specified
+      #   keyword.
+      #
+      # @yield [query]
+      #   If a block is given, it will be passed the new query.
+      #
+      # @yieldparam [Query] query
+      #   The new query.
+      #
+      # @return [Query]
+      #   The new query.
+      #
+      def initialize(options={})
         @query = options[:query]
 
         @link = options[:link]
@@ -100,18 +165,24 @@ module GScraper
         @inurl = options[:inurl]
         @allintext = options[:allintext]
         @intext = options[:intext]
+        @allinanchor = options[:allinanchor]
+        @inanchor = options[:inanchor]
 
         @exact_phrase = options[:exact_phrase]
         @with_words = options[:with_words]
         @without_words = options[:without_words]
 
         @numeric_range = options[:numeric_range]
+        @define = options[:define]
 
-        block.call(self) if block
+        yield self if block_given?
       end
 
       #
-      # Returns the query expression.
+      # The query expression.
+      #
+      # @return [String]
+      #   The expression representing the query.
       #
       def expression
         expr = []
@@ -142,6 +213,10 @@ module GScraper
         append_modifier.call(:inurl)
         append_options.call(:allintext)
         append_modifier.call(:intext)
+        append_options.call(:allinanchor)
+        append_modifier.call(:inanchor)
+
+        append_modifier.call(:define)
 
         if @exact_phrase
           expr << "\"#{@exact_phrase}\""
@@ -164,6 +239,15 @@ module GScraper
 
       protected
 
+      #
+      # Formats the value for a search modifer.
+      #
+      # @param [Regexp, String]
+      #   The value for the search modifier.
+      #
+      # @return [String]
+      #   The formatted value.
+      #
       def format_modifier(value)
         if value.kind_of?(Regexp)
           return value.source
@@ -172,6 +256,15 @@ module GScraper
         end
       end
 
+      #
+      # Formats the value(s) for a search option.
+      #
+      # @param [Array, Regexp, String]
+      #   The value(s) for the search modifier.
+      #
+      # @return [String]
+      #   The formatted value.
+      #
       def format_options(value)
         if value.kind_of?(Array)
           return value.map { |element|

data/lib/gscraper/search/result.rb

@@ -1,5 +1,4 @@
 #
-#--
 # GScraper - A web-scraping interface to various Google Services.
 #
 # Copyright (c) 2007-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
@@ -17,7 +16,6 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
-#++
 #
 
 require 'gscraper/search/query'
@@ -46,8 +44,22 @@ module GScraper
       attr_reader :similar_url
 
       #
-      # Creates a new Result object with the given _rank_, _title_
-      # _summary_, _url_, _size_, _cache_url_ and _similar_url_.
+      # Creates a new {Result} object.
+      #
+      # @param [Integer] rank
+      #   The rank of the result.
+      #
+      # @param [String] title
+      #   The title of the result.
+      #
+      # @param [String] summary
+      #   The summary of the result.
+      #
+      # @param [URI::HTTP] cached_url
+      #   The Cached URL for the result.
+      #
+      # @param [URI::HTTP] similar_url
+      #   The Similar Query URL for the result.
       #
       def initialize(rank,title,url,summary,cached_url=nil,similar_url=nil)
         @agent = GScraper.web_agent
@@ -63,12 +75,18 @@ module GScraper
       #
       # Fetches the page of the result.
       #
+      # @return [Mechanize::Page]
+      #   The page the result represents.
+      #
       def page
         @agent.get(@url)
       end
 
       #
-      # Fetches the cached page of the result.
+      # Fetches the Cached Page of the result.
+      #
+      # @return [Mechanize::Page]
+      #   The Cached Page for the result.
       #
       def cached_page
         if @cached_url
@@ -77,7 +95,10 @@ module GScraper
       end
 
       #
-      # Returns a string containing the result's title.
+      # The result's title.
+      #
+      # @return [String]
+      #   The title of the result.
       #
       def to_s
         @title.to_s

data/lib/gscraper/search/search.rb

@@ -1,5 +1,4 @@
 #
-#--
 # GScraper - A web-scraping interface to various Google Services.
 #
 # Copyright (c) 2007-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
@@ -17,7 +16,6 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
-#++
 #
 
 require 'gscraper/search/web_query'
@@ -26,46 +24,98 @@ require 'gscraper/search/ajax_query'
 module GScraper
   module Search
     #
-    # Returns a new Query object with the given _options_. See Query.new.
+    # Creates a new web-query.
     #
+    # @param [Hash] options
+    #   Additional options.
+    #
+    # @yield [query]
+    #   If a block is given, it will be passed the new web-query.
+    #
+    # @yieldparam [WebQuery] query
+    #   The new web query.
+    #
+    # @return [WebQuery]
+    #   The new web-query.
+    #
+    # @example
     #   Search.query(:query => 'ruby', :with_words => 'sow rspec')
     #
+    # @example
     #   Search.query(:exact_phrase => 'fluent interfaces') do |q|
     #     q.within_past_week = true
     #   end
     #
+    # @see WebQuery#initialize
+    #
     def Search.query(options={},&block)
       WebQuery.new(options,&block)
     end
 
     #
-    # Returns the Query object that represents the specified _url_.
-    # See Query.from_url.
+    # Creates a web-query from a search URL.
+    #
+    # @param [String] url
+    #   The search URL.
+    #
+    # @yield [query]
+    #   If a block is given, it will be passed the new web-query.
+    #
+    # @yieldparam [WebQuery] query
+    #   The new web query.
+    #
+    # @return [WebQuery]
+    #   The new web-query.
     #
+    # @example
     #   Search.query_from_url('http://www.google.com/search?q=ruby+zen)
     #
+    # @example
     #   Search.query_from_url('http://www.google.com/search?q=ruby') do |q|
     #     q.within_last_month = true
     #     q.occurrs_within = :title
     #   end
     #
+    # @see WebQuery.from_url.
+    #
     def Search.query_from_url(url,&block)
       WebQuery.from_url(url,&block)
     end
 
     #
-    # Returns a new AJAXQuery object with the given _options_. See
-    # AJAXQuery.new.
+    # Creates a new AJAX query.
+    #
+    # @param [Hash] options
+    #   Additional options.
+    #
+    # @yield [query]
+    #   If a block is given, the new AJAX query will be passed to it.
     #
+    # @yieldparam [AJAXQuery] query
+    #   The new AJAX query.
+    #
+    # @example
     #   Search.ajax_query(:query => 'ruby')
     #
+    # @see AJAXQuery#initialize
+    #
     def Search.ajax_query(options={},&block)
       AJAXQuery.new(options,&block)
     end
 
     #
-    # Returns the AJAXQuery object that represents the specified _url_.
-    # See AJAXQuery.from_url.
+    # Creates a AJAX query from a given search URL.
+    #
+    # @param [URI::HTTP] url
+    #   The search URL.
+    #
+    # @yield [query]
+    #   If a block is given, the new AJAX query will be passed to it.
+    #
+    # @yieldparam [AJAXQuery] query
+    #   The new AJAX query.
+    #
+    # @see AJAXQuery.from_url.
     #
     def Search.ajax_query_from_url(url,&block)
       AJAXQuery.from_url(url,&block)

data/lib/gscraper/search/web_query.rb

@@ -1,5 +1,4 @@
 #
-#--
 # GScraper - A web-scraping interface to various Google Services.
 #
 # Copyright (c) 2007-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
@@ -17,7 +16,6 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
-#++
 #
 
 require 'gscraper/search/result'
@@ -87,18 +85,56 @@ module GScraper
       # Filter the search results
       attr_accessor :filtered
 
-      # Search for results similar to the page
-      attr_accessor :similar_to
-
-      # Search for results linking to the page
-      attr_accessor :links_to
-
       #
-      # Creates a new WebQuery object from the given search options. If a
-      # block is given, it will be passed the newly created query object.
+      # Creates a new Web query.
+      #
+      # @param [Hash] options
+      #   Additional options.
+      #
+      # @option options [Integer] :results_per_page
+      #   Specifies the number of results for each page.
+      #
+      # @option options [String] :language
+      #   Search for results in the specified language.
+      #
+      # @option options [String] :region
+      #   Search for results from the specified region.
+      #
+      # @option options [Boolean] :within_past_day
+      #   Search for results that were created within the past day.
+      #
+      # @option options [Boolean] :within_past_week
+      #   Search for results that were created within the past week.
+      #
+      # @option options [Boolean] :within_past_month
+      #   Search for results that were created within the past month.
+      #
+      # @option options [Boolean] :within_past_year
+      #   Search for results that were created within the past year.
       #
+      # @option options [:title, :body, :url] :occurrs_within
+      #   Searches for results where the keywords occurr within a specific
+      #   part of the result page.
+      #
+      # @option options [Symbol] :rights
+      #   Search for results licensed under the specified license.
+      #
+      # @option options [Boolean] :filtered
+      #   Specifies whether or not to use SafeSearch.
+      #
+      # @yield [query]
+      #   If a block is given, it will be passed the new Web query.
+      #
+      # @yieldparam [WebQuery] query
+      #   The new Web query.
+      #
+      # @return [WebQuery]
+      #   The new Web query.
+      #
+      # @example
       #   WebQuery.new(:query => 'ruby', :with_words => 'sow rspec')
       #
+      # @example
       #   WebQuery.new(:exact_phrase => 'fluent interfaces') do |q|
       #     q.within_past_week = true
       #   end
@@ -142,18 +178,37 @@ module GScraper
         @rights = options[:rights]
         @filtered = options[:filtered]
 
-        @similar_to = options[:similar_to]
-        @links_to = options[:links_to]
-
         super(options,&block)
       end
 
+      alias similar_to related
+      alias similar_to= related=
+
+      alias links_to link
+      alias links_to= link=
+
       #
-      # Creates a new WebQuery object from the specified URL. If a block is
-      # given, it will be passed the newly created WebQuery object.
+      # Creates a new Web query from a search URL.
       #
+      # @param [URI::HTTP, String] url
+      #   The search URL.
+      #
+      # @param [Hash] options
+      #   Additional options.
+      #
+      # @yield [query]
+      #   If a block is given, it will be passed the new Web query.
+      #
+      # @yieldparam [WebQuery] query
+      #   The new web query.
+      #
+      # @return [WebQuery]
+      #   The new Web query.
+      #
+      # @example
       #   WebQuery.from_url('http://www.google.com/search?q=ruby+zen')
       #
+      # @example
       #   WebQuery.from_url('http://www.google.com/search?q=ruby') do |q|
       #     q.within_last_month = true
       #     q.occurrs_within = :title
@@ -198,8 +253,10 @@ module GScraper
         end
 
         if (url.query_params['as_nlo'] || url.query_params['as_nhi'])
-          options[:numeric_range] = Range.new(url.query_params['as_nlo'].to_i,
-                                              url.query_params['as_nhi'].to_i)
+          options[:numeric_range] = Range.new(
+            url.query_params['as_nlo'].to_i,
+            url.query_params['as_nhi'].to_i
+          )
         end
 
         case url.query_params['as_occt']
@@ -231,16 +288,19 @@ module GScraper
         end
 
         if url.query_params['as_rq']
-          options[:similar_to] = url.query_params['as_rq']
+          options[:related] = url.query_params['as_rq']
         elsif url.query_params['as_lq']
-          options[:links_to] = url.query_params['as_lq']
+          options[:link] = url.query_params['as_lq']
         end
 
         return self.new(options,&block)
       end
 
       #
-      # Returns the URL that represents the query.
+      # The URL that represents the query.
+      #
+      # @return [URI::HTTP]
+      #   The URL for the query.
       #
       def search_url
         url = URI(SEARCH_URL)
@@ -311,18 +371,17 @@ module GScraper
 
         url.query_params['safe'] = 'active' if @filtered
 
-        if @similar_to
-          url.query_params['as_rq'] = @similar_to
-        elsif @links_to
-          url.query_params['as_lq'] = @links_to
-        end
-
         return url
       end
 
       #
-      # Returns the URL that represents the query at the specific
-      # _page_index_.
+      # Returns the URL that represents the query at a specific page index.
+      #
+      # @param [Integer] page_index
+      #   The page index to create the URL for.
+      #
+      # @return [URI::HTTP]
+      #   The URL for a query at the given page index.
       #
       def page_url(page_index)
         url = search_url
@@ -334,8 +393,13 @@ module GScraper
       end
 
       #
-      # Returns a Page object containing Result objects at the specified
-      # _page_index_.
+      # Returns a page containing results at the specific page index.
+      #
+      # @param [Integer] page_index
+      #   The page index to query.
+      #
+      # @return [Page<Result>]
+      #   The page at the given index for the query.
       #
       def page(page_index)
         Page.new do |new_page|
@@ -379,22 +443,30 @@ module GScraper
       end
 
       #
-      # Returns the first Result on the first_page.
+      # Returns the first result on the first page.
+      #
+      # @return [Result]
+      #   The first result.
       #
       def top_result
         first_page.first
       end
 
       #
-      # Returns the Result at the specified _index_.
+      # Returns the result at the specified index.
+      #
+      # @param [Integer]
+      #   The index of the result.
       #
       def result_at(index)
         page(page_index_of(index))[result_index_of(index)]
       end
 
       #
-      # Returns a SponsoredLinks object containing SponsoredAd objects of
-      # the query.
+      # Returns the sponsored links for the query.
+      #
+      # @return [SponsoredLinks<SponsoredAd>]
+      #   The sponsored links for the query.
       #
       def sponsored_links
         SponsoredLinks.new do |links|
@@ -411,15 +483,26 @@ module GScraper
       end
 
       #
-      # Returns the first sponsored link on the first page of results.
+      # Returns the first sponsored ad on the first page of results.
+      #
+      # @return [SponsoredAd]
+      #   The first sponsored ad.
       #
       def top_sponsored_link
         top_sponsored_links.first
       end
 
       #
-      # Iterates over the sponsored links on the first page of
-      # results passing each to the specified _block_.
+      # Iterates over the sponsored ads on the first page.
+      #
+      # @yield [ad]
+      #   The given block will be passed each sponsored ad.
+      #
+      # @yieldparam [SponsoredAd] ad
+      #   A sponsored ad on the first page.
+      #
+      # @return [Enumerator]
+      #   If no block is given, an Enumerator object will be returned.
       #
       def each_sponsored_link(&block)
         sponsored_links.each(&block)