gscraper 0.1.7 → 0.2.0

data/Rakefile

@@ -2,16 +2,13 @@
 
 require 'rubygems'
 require 'hoe'
+
+require './tasks/spec.rb'
 require './lib/gscraper/version.rb'
 
 Hoe.new('gscraper', GScraper::VERSION) do |p|
   p.rubyforge_name = 'gscraper'
-  p.author = 'Postmodern Modulus III'
-  p.email = 'postmodern.mod3@gmail.com'
-  p.summary = 'A ruby web-scraping interface to various Google Services'
-  p.description = p.paragraphs_of('README.txt', 2..5).join("\n\n")
-  p.url = p.paragraphs_of('README.txt', 0).first.split(/\n/)[1..-1]
-  p.changes = p.paragraphs_of('History.txt', 0..1).join("\n\n")
+  p.developer('Postmodern Modulus III', 'postmodern.mod3@gmail.com')
   p.extra_deps = ['hpricot', 'mechanize']
 end
 

data/lib/gscraper.rb

@@ -1,2 +1,24 @@
+#
+#--
+# GScraper - A web-scraping interface to various Google Services.
+#
+# Copyright (c) 2007-2008 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#++
+#
+
 require 'gscraper/search'
 require 'gscraper/version'

data/lib/gscraper/extensions.rb

@@ -1 +1,23 @@
+#
+#--
+# GScraper - A web-scraping interface to various Google Services.
+#
+# Copyright (c) 2007-2008 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#++
+#
+
 require 'gscraper/extensions/uri'

data/lib/gscraper/extensions/uri.rb

@@ -1 +1,23 @@
+#
+#--
+# GScraper - A web-scraping interface to various Google Services.
+#
+# Copyright (c) 2007-2008 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#++
+#
+
 require 'gscraper/extensions/uri/http'

data/lib/gscraper/extensions/uri/http.rb

@@ -1,79 +1,33 @@
+#
+#--
+# GScraper - A web-scraping interface to various Google Services.
+#
+# Copyright (c) 2007-2008 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#++
+#
+
+require 'gscraper/extensions/uri/query_params'
+
 require 'uri/http'
 
 module URI
   class HTTP < Generic
 
-    # Query parameters
-    attr_reader :query_params
-
-    alias_method :old_initialize, :initialize
-
-    #
-    # Creates a new URI::HTTP object and initializes query_params as a
-    # new Hash.
-    #
-    def initialize(*args)
-      old_initialize(*args)
-
-      @query_params = {}
-      parse_query_params
-    end
-
-    #
-    # Sets the query data and updates query_params.
-    #
-    def query=(query_str)
-      new_query = super(query_str)
-      parse_query_params
-      return new_query
-    end
-
-    protected
-
-    #
-    # Parses the query parameters from the query data, populating
-    # query_params with the parsed parameters.
-    #
-    def parse_query_params
-      @query_params.clear
-
-      if @query
-        @query.split('&').each do |param|
-          name, value = param.split('=')
-
-          if value
-            @query_params[name] = URI.decode(value)
-          else
-            @query_params[name] = nil
-          end
-        end
-      end
-    end
-
-    private
-
-    # :nodoc
-    def path_query
-      str = @path
-
-      unless @query_params.empty?
-        str += '?' + @query_params.to_a.map { |name,value|
-          if value==true
-            "#{name}=active"
-          elsif value
-            if value.kind_of?(Array)
-              "#{name}=#{URI.encode(value.join(' '))}"
-            else
-              "#{name}=#{URI.encode(value.to_s)}"
-            end
-          else
-            "#{name}="
-          end
-        }.join('&')
-      end
-
-      return str
-    end
+    include QueryParams
 
   end
 end

data/lib/gscraper/extensions/uri/query_params.rb

@@ -0,0 +1,96 @@
+#
+#--
+# GScraper - A web-scraping interface to various Google Services.
+#
+# Copyright (c) 2007-2008 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#++
+#
+
+module URI
+  module QueryParams
+    # Query parameters
+    attr_reader :query_params
+
+    #
+    # Creates a new URI::HTTP object and initializes query_params as a
+    # new Hash.
+    #
+    def initialize(*args)
+      @query_params = {}
+
+      super(*args)
+
+      parse_query_params
+    end
+
+    #
+    # Sets the query data and updates query_params.
+    #
+    def query=(query_str)
+      new_query = super(query_str)
+      parse_query_params
+      return new_query
+    end
+
+    protected
+
+    #
+    # Parses the query parameters from the query data, populating
+    # query_params with the parsed parameters.
+    #
+    def parse_query_params
+      @query_params.clear
+
+      if @query
+        @query.split('&').each do |param|
+          name, value = param.split('=')
+
+          if value
+            @query_params[name] = URI.decode(value)
+          else
+            @query_params[name] = nil
+          end
+        end
+      end
+    end
+
+    private
+
+    # :nodoc
+    def path_query
+      str = @path
+
+      unless @query_params.empty?
+        str += '?' + @query_params.to_a.map { |name,value|
+          if value==true
+            "#{name}=active"
+          elsif value
+            if value.kind_of?(Array)
+              "#{name}=#{URI.encode(value.join(' '))}"
+            else
+              "#{name}=#{URI.encode(value.to_s)}"
+            end
+          else
+            "#{name}="
+          end
+        }.join('&')
+      end
+
+      return str
+    end
+  end
+end

data/lib/gscraper/gscraper.rb

@@ -1,3 +1,25 @@
+#
+#--
+# GScraper - A web-scraping interface to various Google Services.
+#
+# Copyright (c) 2007-2008 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#++
+#
+
 require 'uri/http'
 require 'mechanize'
 require 'open-uri'
@@ -54,6 +76,14 @@ module GScraper
     @@gscraper_user_agent = agent
   end
 
+  #
+  # Sets the GScraper User-Agent using the specified user-agent alias
+  # _name_.
+  # 
+  def GScraper.user_agent_alias=(name)
+    @@gscraper_user_agent = GScraper.user_agent_aliases[name.to_s]
+  end
+
   #
   # Opens the _uri_ with the given _options_. The contents of the _uri_
   # will be returned.

data/lib/gscraper/has_pages.rb

@@ -0,0 +1,114 @@
+#
+#--
+# GScraper - A web-scraping interface to various Google Services.
+#
+# Copyright (c) 2007-2008 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#++
+#
+
+module GScraper
+  module HasPages
+    include Enumerable
+
+    #
+    # Returns the first page.
+    #
+    def first_page
+      page_cache[1]
+    end
+
+    #
+    # Returns the page at the specified _index_.
+    #
+    def [](index)
+      page_cache[index]
+    end
+
+    #
+    # Returns the pages with the specified _indices_.
+    #
+    def pages(indices)
+      indices.map { |index| page_cache[index] }
+    end
+
+    #
+    # Iterates over the pages with the specified _indices_, passing each
+    # to the specified _block_.
+    #
+    def each_page(indices,&block)
+      indices.map { |index| block.call(page_cache[index]) }
+    end
+
+    #
+    # Iterates over all the pages of the query, passing each to the
+    # specified _block_.
+    #
+    def each(&block)
+      index = 1
+
+      until ((next_page = page_cache[index]).empty?) do
+        block.call(next_page)
+        index = index + 1
+      end
+
+      return self
+    end
+
+    #
+    # Iterates over the elements on the page with the specified _index_,
+    # passing each element to the specified _block_.
+    #
+    def each_on_page(index,&block)
+      page_cache[index].each(&block)
+    end
+
+    #
+    # Iterates over each element on the pages with the specified _indices_,
+    # passing each element to the specified _block_.
+    #
+    def each_on_pages(indices,&block)
+      each_page(indices) { |page| page.each(&block) }
+    end
+
+    protected
+
+    #
+    # Returns the page index for the specified result _rank_.
+    #
+    def page_index_of(rank)
+      (((rank.to_i - 1) / results_per_page.to_i) + 1)
+    end
+
+    #
+    # Returns the rank offset for the specified _page_index_.
+    #
+    def result_offset_of(page_index)
+      ((page_index.to_i - 1) * results_per_page.to_i)
+    end
+
+    #
+    # Returns the in-page index of the specified result _rank_.
+    #
+    def result_index_of(rank)
+      ((rank.to_i - 1) % results_per_page.to_i)
+    end
+
+    def page_cache
+      @page_cache ||= Hash.new { |hash,key| hash[key] = page(key.to_i) }
+    end
+  end
+end