gscraper 0.2.4 → 0.3.0

data/lib/gscraper.rb

@@ -1,5 +1,4 @@
 #
-#--
 # GScraper - A web-scraping interface to various Google Services.
 #
 # Copyright (c) 2007-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
@@ -17,7 +16,6 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
-#++
 #
 
 require 'gscraper/search'

data/lib/gscraper/extensions.rb

@@ -1,5 +1,4 @@
 #
-#--
 # GScraper - A web-scraping interface to various Google Services.
 #
 # Copyright (c) 2007-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
@@ -17,7 +16,6 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
-#++
 #
 
 require 'gscraper/extensions/uri'

data/lib/gscraper/extensions/uri.rb

@@ -1,5 +1,4 @@
 #
-#--
 # GScraper - A web-scraping interface to various Google Services.
 #
 # Copyright (c) 2007-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
@@ -17,7 +16,6 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
-#++
 #
 
 require 'gscraper/extensions/uri/http'

data/lib/gscraper/extensions/uri/http.rb

@@ -1,5 +1,4 @@
 #
-#--
 # GScraper - A web-scraping interface to various Google Services.
 #
 # Copyright (c) 2007-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
@@ -17,7 +16,6 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
-#++
 #
 
 require 'gscraper/extensions/uri/query_params'

data/lib/gscraper/extensions/uri/query_params.rb

@@ -1,5 +1,4 @@
 #
-#--
 # GScraper - A web-scraping interface to various Google Services.
 #
 # Copyright (c) 2007-2008 Hal Brodigan (postmodern.mod3 at gmail.com)
@@ -17,10 +16,15 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
-#++
 #
 
+require 'cgi'
+
 module URI
+  #
+  # Adds the ability to parse individual parameters from a the query field
+  # of a URI.
+  #
   module QueryParams
     # Query parameters
     attr_reader :query_params
@@ -40,6 +44,16 @@ module URI
     #
     # Sets the query data and updates query_params.
     #
+    # @param [String] query_str
+    #   The new URI query string to use.
+    #
+    # @return [String]
+    #   The new URI query string.
+    #
+    # @example
+    #   url.query = 'a=1&b=2'
+    #   # => "a=1&b=2"
+    #
     def query=(query_str)
       new_query = super(query_str)
       parse_query_params
@@ -70,7 +84,6 @@ module URI
 
     private
 
-    # :nodoc
     def path_query
       str = @path
 
@@ -80,9 +93,9 @@ module URI
             "#{name}=active"
           elsif value
             if value.kind_of?(Array)
-              "#{name}=#{URI.encode(value.join(' '))}"
+              "#{name}=#{CGI.escape(value.join(' '))}"
             else
-              "#{name}=#{URI.encode(value.to_s)}"
+              "#{name}=#{CGI.escape(value.to_s)}"
             end
           else
             "#{name}="

data/lib/gscraper/gscraper.rb

@@ -1,5 +1,4 @@
 #
-#--
 # GScraper - A web-scraping interface to various Google Services.
 #
 # Copyright (c) 2007-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
@@ -17,7 +16,6 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
-#++
 #
 
 require 'uri/http'
@@ -30,22 +28,31 @@ module GScraper
   COMMON_PROXY_PORT = 8080
 
   #
-  # Returns the +Hash+ of proxy information.
+  # The proxy information.
+  #
+  # @return [Hash]
   #
   def GScraper.proxy
     @@gscraper_proxy ||= {:host => nil, :port => COMMON_PROXY_PORT, :user => nil, :password => nil}
   end
 
   #
-  # Creates a HTTP URI based from the given _proxy_info_ hash. The
-  # _proxy_info_ hash defaults to Web.proxy, if not given.
+  # Creates a HTTP URI for the current proxy.
+  #
+  # @param [Hash] proxy_info
+  #   The proxy information.
+  #
+  # @option proxy_info [String] :host
+  #   The proxy host.
+  #
+  # @option proxy_info [Integer] :port (COMMON_PROXY_PORT)
+  #   The proxy port.
   #
-  # _proxy_info_ may contain the following keys:
-  # <tt>:host</tt>:: The proxy host.
-  # <tt>:port</tt>:: The proxy port. Defaults to COMMON_PROXY_PORT,
-  #                  if not specified.
-  # <tt>:user</tt>:: The user-name to login as.
-  # <tt>:password</tt>:: The password to login with.
+  # @option proxy_info [String] :user
+  #   The user-name to login as.
+  #
+  # @option proxy_info [String] :password
+  #   The password to login with.
   #
   def GScraper.proxy_uri(proxy_info=GScraper.proxy)
     if GScraper.proxy[:host]
@@ -59,101 +66,85 @@ module GScraper
   end
   
   #
-  # Returns the supported GScraper User-Agent Aliases.
+  # The supported GScraper User-Agent Aliases.
+  #
+  # @return [Array<String>]
   #
   def GScraper.user_agent_aliases
-    WWW::Mechanize::AGENT_ALIASES
+    Mechanize::AGENT_ALIASES
   end
 
   #
-  # Returns the GScraper User-Agent
+  # The GScraper User-Agent.
+  #
+  # @return [String]
   #
   def GScraper.user_agent
     @@gscraper_user_agent ||= GScraper.user_agent_aliases['Windows IE 6']
   end
 
   #
-  # Sets the GScraper User-Agent to the specified _agent_.
+  # Sets the GScraper User-Agent.
+  #
+  # @param [String] agent
+  #   The new User-Agent string.
+  #
+  # @return [String]
+  #   The new User-Agent string.
   #
   def GScraper.user_agent=(agent)
     @@gscraper_user_agent = agent
   end
 
   #
-  # Sets the GScraper User-Agent using the specified user-agent alias
-  # _name_.
+  # Sets the GScraper User-Agent.
+  #
+  # @param [String] name
+  #   The User-Agent alias.
+  #
+  # @return [String]
+  #   The new User-Agent string.
   # 
   def GScraper.user_agent_alias=(name)
     @@gscraper_user_agent = GScraper.user_agent_aliases[name.to_s]
   end
 
   #
-  # Opens the _uri_ with the given _options_. The contents of the _uri_
-  # will be returned.
+  # Creates a new Mechanize agent.
   #
-  # _options_ may contain the following keys:
-  # <tt>:user_agent_alias</tt>:: The User-Agent Alias to use.
-  # <tt>:user_agent</tt>:: The User-Agent String to use.
-  # <tt>:proxy</tt>:: A +Hash+ of proxy information which may
-  #                   contain the following keys:
-  #                   <tt>:host</tt>:: The proxy host.
-  #                   <tt>:port</tt>:: The proxy port.
-  #                   <tt>:user</tt>:: The user-name to login as.
-  #                   <tt>:password</tt>:: The password to login with.
+  # @param [Hash] options
+  #   Additional options.
   #
-  #   GScraper.open_uri('http://www.hackety.org/')
+  # @option options [String] :user_agent_alias
+  #   The User-Agent Alias to use.
   #
-  #   GScraper.open_uri('http://tenderlovemaking.com/',
-  #     :user_agent_alias => 'Linux Mozilla')
-  #   GScraper.open_uri('http://www.wired.com/',
-  #     :user_agent => 'the future')
+  # @option options [String] :user_agent
+  #   The User-Agent string to use.
   #
-  def GScraper.open_uri(uri,options={})
-    headers = {}
-
-    if options[:user_agent_alias]
-      headers['User-Agent'] = WWW::Mechanize::AGENT_ALIASES[options[:user_agent_alias]]
-    elsif options[:user_agent]
-      headers['User-Agent'] = options[:user_agent]
-    elsif GScraper.user_agent
-      headers['User-Agent'] = GScraper.user_agent
-    end
-
-    proxy = (options[:proxy] || GScraper.proxy)
-    if proxy[:host]
-      headers[:proxy] = GScraper.proxy_uri(proxy)
-    end
-
-    return Kernel.open(uri,headers)
-  end
-
+  # @option options [Hash] :proxy
+  #   The proxy information to use.
   #
-  # Similar to GScraper.open_uri but returns a Nokogiri::HTML document.
+  # @option :proxy [String] :host
+  #   The proxy host.
   #
-  def GScraper.open_page(uri,options={})
-    Nokogiri::HTML(GScraper.open_uri(uri,options))
-  end
-
+  # @option :proxy [Integer] :port
+  #   The proxy port.
   #
-  # Creates a new WWW::Mechanize agent with the given _options_.
+  # @option :proxy [String] :user
+  #   The user-name to login as.
   #
-  # _options_ may contain the following keys:
-  # <tt>:user_agent_alias</tt>:: The User-Agent Alias to use.
-  # <tt>:user_agent</tt>:: The User-Agent string to use.
-  # <tt>:proxy</tt>:: A +Hash+ of proxy information which may
-  #                   contain the following keys:
-  #                   <tt>:host</tt>:: The proxy host.
-  #                   <tt>:port</tt>:: The proxy port.
-  #                   <tt>:user</tt>:: The user-name to login as.
-  #                   <tt>:password</tt>:: The password to login with.
+  # @option :proxy [String] :password
+  #   The password to login with.
   #
+  # @example
   #   GScraper.web_agent
   #
+  # @example
   #   GScraper.web_agent(:user_agent_alias => 'Linux Mozilla')
   #   GScraper.web_agent(:user_agent => 'Google Bot')
   #
-  def GScraper.web_agent(options={},&block)
-    agent = WWW::Mechanize.new
+  def GScraper.web_agent(options={})
+    agent = Mechanize.new
 
     if options[:user_agent_alias]
       agent.user_agent_alias = options[:user_agent_alias]
@@ -168,7 +159,7 @@ module GScraper
       agent.set_proxy(proxy[:host],proxy[:port],proxy[:user],proxy[:password])
     end
 
-    block.call(agent) if block
+    yield agent if block_given?
     return agent
   end
 end

data/lib/gscraper/has_pages.rb

@@ -1,5 +1,4 @@
 #
-#--
 # GScraper - A web-scraping interface to various Google Services.
 #
 # Copyright (c) 2007-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
@@ -17,51 +16,87 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
-#++
 #
 
+require 'enumerator'
+
 module GScraper
   module HasPages
     include Enumerable
 
     #
-    # Returns the first page.
+    # The first page.
+    #
+    # @return [Page]
+    #   The first page.
     #
     def first_page
       page_cache[1]
     end
 
     #
-    # Returns the page at the specified _index_.
+    # The page at the specified index.
+    #
+    # @param [Integer] index
+    #   The index.
+    #
+    # @return [Page]
+    #   The page at the given index.
     #
     def [](index)
       page_cache[index]
     end
 
     #
-    # Returns the pages with the specified _indices_.
+    # The pages with the specified indices.
+    #
+    # @param [Array, Range] indices
+    #   The indices.
+    #
+    # @return [Page]
+    #   The pages at the given indices.
     #
     def pages(indices)
       indices.map { |index| page_cache[index] }
     end
 
     #
-    # Iterates over the pages with the specified _indices_, passing each
-    # to the specified _block_.
+    # Iterates over the pages at the specified indices.
     #
-    def each_page(indices,&block)
-      indices.map { |index| block.call(page_cache[index]) }
+    # @param [Array, Range] indices
+    #   The indices.
+    #
+    # @yield [page]
+    #   The given block will be passed each page.
+    #
+    # @yieldparam [Page] page
+    #   A page at one of the given indices.
+    #
+    def each_page(indices)
+      unless block_given?
+        enum_for(:each_page,indices)
+      else
+        indices.map { |index| yield page_cache[index] }
+      end
     end
 
     #
-    # Iterates over all the pages of the query, passing each to the
-    # specified _block_.
+    # Iterates over all the pages of the query, until an empty page is
+    # encountered.
+    #
+    # @yield [page]
+    #   A page with results from the query.
     #
-    def each(&block)
+    # @yieldparam [Page] page
+    #   A non-empty page from the query.
+    #
+    def each
+      return enum_for(:each) unless block_given?
+
       index = 1
 
       until ((next_page = page_cache[index]).empty?) do
-        block.call(next_page)
+        yield next_page
         index = index + 1
       end
 
@@ -69,16 +104,20 @@ module GScraper
     end
 
     #
-    # Iterates over the elements on the page with the specified _index_,
-    # passing each element to the specified _block_.
+    # Iterates over the elements on the page with the specified index.
+    #
+    # @param [Integer] index
+    #   The index to access.
     #
     def each_on_page(index,&block)
       page_cache[index].each(&block)
     end
 
     #
-    # Iterates over each element on the pages with the specified _indices_,
-    # passing each element to the specified _block_.
+    # Iterates over each element on the pages with the specified indices.
+    #
+    # @param [Array, Range] indices
+    #   The indices to access.
     #
     def each_on_pages(indices,&block)
       each_page(indices) { |page| page.each(&block) }
@@ -87,21 +126,36 @@ module GScraper
     protected
 
     #
-    # Returns the page index for the specified result _rank_.
+    # The page index for the specified result rank.
+    #
+    # @param [Integer] rank
+    #   A result ranking.
+    #
+    # @return [Integer]
+    #   The page index.
     #
     def page_index_of(rank)
       (((rank.to_i - 1) / results_per_page.to_i) + 1)
     end
 
     #
-    # Returns the rank offset for the specified _page_index_.
+    # The rank offset for the specified page-index.
+    #
+    # @param [Integer] page_index
+    #   The result offset within a page.
     #
     def result_offset_of(page_index)
       ((page_index.to_i - 1) * results_per_page.to_i)
     end
 
     #
-    # Returns the in-page index of the specified result _rank_.
+    # The in-page index of the specified result rank.
+    #
+    # @param [Integer] rank
+    #   The result ranking.
+    #
+    # @return [Integer]
+    #   The in-page index.
     #
     def result_index_of(rank)
       ((rank.to_i - 1) % results_per_page.to_i)
@@ -110,6 +164,8 @@ module GScraper
     #
     # The cache of previously requested pages.
     #
+    # @return [Hash]
+    #
     def page_cache
       @page_cache ||= Hash.new { |hash,key| hash[key] = page(key.to_i) }
     end