gscraper 0.1.3 → 0.1.4

data/History.txt

@@ -1,7 +1,22 @@
+== 0.1.4 / 2007-12-23
+
+ * Added Search::Query#result_at for easier access of a single result at
+   a given index.
+ * Adding scraping of the "Cached" and "Similar Pages" URLs of Search
+   Results.
+ * Added methods to Search::Page for accessing cached URLs, cached pages,
+   similar query URLs and similar Queries in mass.
+ * Search::Query#page and Search::Query#first_page now can receive blocks.
+ * Improved the formating of URL query parameters.
+ * Added more unit-tests.
+ * Fixed scraping of Search Result summaries.
+ * Fixed various bugs in Search::Query uncovered during unit-testing.
+ * Fixed typos in Search::Page's documentation.
+
 == 0.1.3 / 2007-12-22
 
- * Added the Page class, which contains many of convenance methods for
-   searching through the results within a Page.
+ * Added the Search::Page class, which contains many of convenance methods
+   for searching through the results within a Page.
 
 == 0.1.2 / 2007-12-22
 

data/Manifest.txt

@@ -17,3 +17,6 @@ lib/gscraper/search/search.rb
 lib/gscraper/search.rb
 test/test_gscraper.rb
 test/search/query_from_url.rb
+test/search/query_result.rb
+test/search/query_pages.rb
+test/search/page_results.rb

data/Rakefile

@@ -12,6 +12,7 @@ Hoe.new('gscraper', GScraper::VERSION) do |p|
   p.description = p.paragraphs_of('README.txt', 2..5).join("\n\n")
   p.url = p.paragraphs_of('README.txt', 0).first.split(/\n/)[1..-1]
   p.changes = p.paragraphs_of('History.txt', 0..1).join("\n\n")
+  p.extra_deps = ['hpricot', 'mechanize']
 end
 
 # vim: syntax=Ruby

data/lib/gscraper/extensions/uri/http.rb

@@ -4,12 +4,14 @@ module URI
     # Query parameters
     attr_reader :query_params
 
+    alias_method :old_initialize, :initialize
+
     #
     # Creates a new URI::HTTP object and initializes query_params as a
     # new Hash.
     #
     def initialize(*args)
-      super(*args)
+      old_initialize(*args)
 
       @query_params = {}
       parse_query_params
@@ -57,7 +59,11 @@ module URI
           if value==true
             "#{name}=active"
           elsif value
-            "#{name}=#{URI.encode(value.to_s)}"
+            if value.kind_of?(Array)
+              "#{name}=#{URI.encode(value.join(' '))}"
+            else
+              "#{name}=#{URI.encode(value.to_s)}"
+            end
           else
             "#{name}="
           end

data/lib/gscraper/gscraper.rb

@@ -13,7 +13,7 @@ module GScraper
   # Returns the GScraper User-Agent
   #
   def GScraper.user_agent
-    @user_agent
+    @user_agent ||= nil
   end
 
   #
@@ -24,10 +24,11 @@ module GScraper
   end
 
   #
-  # Opens the _uri_ with the given _opts_. The contents of the _uri_ will
-  # be returned.
+  # Opens the _uri_ with the given _opts_. The contents of the _uri_ will be
+  # returned.
   #
   #   GScraper.open('http://www.hackety.org/')
+  #
   #   GScraper.open('http://tenderlovemaking.com/',
   #     :user_agent_alias => 'Linux Mozilla')
   #   GScraper.open('http://www.wired.com/', :user_agent => 'the future')

data/lib/gscraper/search/page.rb

@@ -115,77 +115,174 @@ module GScraper
 
       #
       # Returns an Array containing the ranks of the results within the
-      # Page. If _block_ is given, each rank will be passed to the _block_.
+      # Page.
       #
       #   page.ranks # => [...]
       #
-      #   page.ranks do |rank|
-      #     puts ranks
-      #   end
-      #
-      def ranks(&block)
-        mapped = map { |result| result.rank }
-
-        mapped.each(&block) if block
-        return mapped
+      def ranks
+        map { |result| result.rank }
       end
 
       #
       # Returns an Array containing the titles of the results within the
-      # Page. If _block_ is given, each title will be passed to the _block_.
+      # Page.
       #
       #   page.titles # => [...]
       #
-      #   page.titles do |title|
-      #     puts title
-      #   end
-      #
-      def titles(&block)
-        mapped = map { |result| result.title }
-
-        mapped.each(&block) if block
-        return mapped
+      def titles
+        map { |result| result.title }
       end
 
       #
       # Returns an Array containing the URLs of the results within the
-      # Page. If _block_ is given, each URL will be passed to the _block_.
+      # Page.
       #
       #   page.urls # => [...]
       #
-      #   page.urls do |url|
-      #     puts url
-      #   end
-      #
-      def urls(&block)
-        mapped = map { |result| result.url }
-
-        mapped.each(&block) if block
-        return mapped
+      def urls
+        map { |result| result.url }
       end
 
       #
       # Returns an Array containing the summaries of the results within the
-      # Page. If _block_ is given, each summary will be passed to the
-      # _block_.
+      # Page.
       #
       #   page.summaries # => [...]
       #
-      #   page.summaries do |summary|
-      #     puts summary
-      #   end
+      def summaries
+        map { |result| result.summary }
+      end
+
+      #
+      # Returns an Array containing the cached URLs of the results within
+      # the Page.
+      #
+      #   page.cached_urls # => [...]
       #
-      def summaries(&block)
-        mapped = map { |result| result.summaries }
+      def cached_urls
+        map { |result| result.cached_url }
+      end
 
-        mapped.each(&block) if block
-        return mapped
+      #
+      # Returns an Array containing the cached pages of the results within
+      # the Page. If _opts_ are given, they will be used in accessing the
+      # cached page.
+      #
+      #   page.cached_pages # => [...]
+      #
+      def cached_pages(opts={})
+        map { |result| result.cached_page(opts) }
+      end
+
+      #
+      # Returns an Array containing the similar Query URLs of the results
+      # within the Page.
+      #
+      #   page.similar_urls # => [...]
+      #
+      def similar_urls
+        map { |result| result.similar_url }
+      end
+
+      #
+      # Returns an Array containing the similar Queries of the results
+      # within the Page.
+      #
+      #   page.similar_queries # => [...]
+      #
+      def similar_queries
+        map { |result| result.similar_query }
+      end
+
+      #
+      # Iterates over each result's rank within the Page, passing each to
+      # the given _block_.
+      #
+      #   each_rank { |rank| puts rank }
+      #
+      def each_rank(&block)
+        ranks.each(&block)
+      end
+
+      #
+      # Iterates over each result's title within the Page, passing each to
+      # the given _block_.
+      #
+      #   each_title { |title| puts title }
+      #
+      def each_title(&block)
+        titles.each(&block)
+      end
+
+      #
+      # Iterates over each result's url within the Page, passing each to
+      # the given _block_.
+      #
+      #   each_url { |url| puts url }
+      #
+      def each_url(&block)
+        urls.each(&block)
+      end
+
+      #
+      # Iterates over each result's summary within the Page, passing each
+      # to the given _block_.
+      #
+      #   each_summary { |summary| puts summary }
+      #
+      def each_summary(&block)
+        summaries.each(&block)
+      end
+
+      #
+      # Iterates over each result's cached URLs within the Page, passing
+      # each to the given _block_.
+      #
+      #   each_cached_url { |url| puts url }
+      #
+      def each_cached_url(&block)
+        cached_urls.each(&block)
+      end
+
+      #
+      # Iterates over each result's cached pages within the Page, passing
+      # each to the given _block_. If _opts_ are given, they will be used
+      # in accessing the cached pages.
+      #
+      #   each_cached_page { |page| puts page.readlines }
+      #
+      def each_cached_page(opts={},&block)
+        cached_pages(opts).each(&block)
+      end
+
+      #
+      # Iterates over each result's similar Query URLs within the Page,
+      # passing each to the given _block_.
+      #
+      #   each_similar_url { |url| puts url }
+      #
+      def each_similar_url(&block)
+        similar_urls.each(&block)
+      end
+
+      #
+      # Iterates over each result's similar Query within the Page, passing
+      # each to the given _block_.
+      #
+      #   each_similar_query do |q|
+      #     q.first_page do |page|
+      #       puts page.urls.join("\n")
+      #     end
+      #   end
+      #
+      def each_similar_query(&block)
+        similar_queries.each(&block)
       end
 
       #
       # Returns the ranks of the results that match the specified _block_.
       #
-      #   page.ranks_of { |result result.title =~ /awesome/ }
+      #   page.ranks_of { |result| result.title =~ /awesome/ }
       #
       def ranks_of(&block)
         results_with(&block).ranks
@@ -194,7 +291,7 @@ module GScraper
       #
       # Returns the titles of the results that match the specified _block_.
       #
-      #   page.titles_of { |result result.url.include?('www') }
+      #   page.titles_of { |result| result.url.include?('www') }
       #
       def titles_of(&block)
         results_with(&block).titles
@@ -203,7 +300,7 @@ module GScraper
       #
       # Returns the urls of the results that match the specified _block_.
       #
-      #   page.urls_of { |result result.summary =~ /awesome pants/ }
+      #   page.urls_of { |result| result.summary =~ /awesome pants/ }
       #
       def urls_of(&block)
         results_with(&block).urls
@@ -213,12 +310,53 @@ module GScraper
       # Returns the summaries of the results that match the specified
       # _block_.
       #
-      #   page.summaries_of { |result result.title =~ /what if/ }
+      #   page.summaries_of { |result| result.title =~ /what if/ }
       #
       def summaries_of(&block)
         results_with(&block).summaries
       end
 
+      #
+      # Returns the cached URLs of the results that match the specified
+      # _block_.
+      #
+      #   page.cached_urls_of { |result| result.title =~ /howdy/ }
+      #
+      def cached_urls_of(&block)
+        results_with(&block).cached_urls
+      end
+
+      #
+      # Returns the cached pages of the results that match the specified
+      # _block_. If _opts_ are given, they will be used in accessing
+      # the cached pages.
+      #
+      #   page.cached_pages_of { |result| result.title =~ /dude/ }
+      #
+      def cached_pages_of(opts={},&block)
+        results_with(&block).cached_pages(opts)
+      end
+
+      #
+      # Returns the similar query URLs of the results that match the
+      # specified _block_.
+      #
+      #   page.similar_urls_of { |result| result.title =~ /what if/ }
+      #
+      def similar_urls_of(&block)
+        results_with(&block).similar_urls
+      end
+
+      #
+      # Returns the similar Queries of the results that match the
+      # specified _block_.
+      #
+      #   page.similar_queries_of { |result| result.title =~ /hackety/ }
+      #
+      def similar_queries_of(&block)
+        results_with(&block).similar_queries
+      end
+
     end
   end
 end

data/lib/gscraper/search/query.rb

@@ -10,7 +10,8 @@ module GScraper
   module Search
     class Query
 
-      SEARCH_URL = 'http://www.google.com/search'
+      SEARCH_HOST = 'www.google.com'
+      SEARCH_URL = "http://#{SEARCH_HOST}/search"
 
       RESULTS_PER_PAGE = 10
 
@@ -90,7 +91,7 @@ module GScraper
       def initialize(opts={},&block)
         super()
 
-        @results_per_page = opts[:results_per_page] || RESULTS_PER_PAGE
+        @results_per_page = (opts[:results_per_page] || RESULTS_PER_PAGE)
 
         @query = opts[:query]
         @exact_phrase = opts[:exact_phrase]
@@ -104,12 +105,29 @@ module GScraper
 
         if opts[:within_past_day]
           @within_past_day = opts[:within_past_day]
+          @within_past_week = false
+          @within_past_months = false
+          @within_past_year = false
         elsif opts[:within_past_week]
+          @within_past_day = false
           @within_past_week = opts[:within_past_week]
+          @within_past_months = false
+          @within_past_year = false
         elsif opts[:within_past_months]
+          @within_past_day = false
+          @within_past_week = false
           @within_past_months = opts[:within_past_months]
+          @within_past_year = false
         elsif opts[:within_past_year]
+          @within_past_day = false
+          @within_past_week = false
+          @within_past_months = false
           @within_past_year = opts[:within_past_year]
+        else
+          @within_past_day = false
+          @within_past_week = false
+          @within_past_months = false
+          @within_past_year = false
         end
 
         @numeric_range = opts[:numeric_range]
@@ -318,7 +336,7 @@ module GScraper
       def page_url(page_index)
         url = search_url
 
-        url.query_params['start'] = page_index_offset(page_index)
+        url.query_params['start'] = page_result_offset(page_index)
         url.query_params['sa'] = 'N'
 
         return url
@@ -327,33 +345,67 @@ module GScraper
       #
       # Returns a Page object containing Result objects at the specified
       # _page_index_. If _opts_ are given, they will be used in accessing
-      # the SEARCH_URL.
+      # the SEARCH_URL. If a _block_ is given, it will be passed the newly
+      # created Page.
       #
-      def page(page_index,opts={})
+      def page(page_index,opts={},&block)
         doc = Hpricot(GScraper.open(page_url(page_index),opts))
+
         new_page = Page.new
+        results = doc.search('//div.g')[0...@results_per_page.to_i]
+
+        results.each_with_index do |result,index|
+          rank = page_result_offset(page_index) + (index + 1)
+          title = result.at('//h2.r').inner_text
+          url = result.at('//h2.r/a').get_attribute('href')
+
+          summary = result.at('//td.j//font').children[0...-3].inject('') do |accum,elem|
+            accum + elem.inner_text
+          end
+
+          cached_url = nil
+          similar_url = nil
 
-        doc.search('//div.g').each_with_index do |result,index|
-          rank = page_index_offset(page_index) + (index + 1)
-          title = result.search('//h2.r').first.inner_text
-          url = result.search('//h2.r/a').first.get_attribute('href')
-          # TODO: exclude URL and Links from summary text
-          summary = result.search('//td.j').first.inner_text
+          if (cached_link = result.at('//td.j//font/nobr/a:first'))
+            cached_url = cached_link.get_attribute('href')
+          end
 
-          # TODO: scrape Cached and Similar links
+          if (similar_link = result.at('//td.j//font/nobr/a:last'))
+            similar_url = "http://#{SEARCH_HOST}" + similar_link.get_attribute('href')
+          end
 
-          new_page << Result.new(rank,title,url,summary)
+          new_page << Result.new(rank,title,url,summary,cached_url,similar_url)
         end
 
+        block.call(new_page) if block
         return new_page
       end
 
       #
-      # Returns the results on the first page. If _opts_ are given, they
-      # will be used in accessing the SEARCH_URL.
+      # Returns the Results on the first page. If _opts_ are given, they
+      # will be used in accessing the SEARCH_URL. If a _block_ is given
+      # it will be passed the newly created Page.
+      #
+      def first_page(opts={},&block)
+        page(1,opts,&block)
+      end
+
+      #
+      # Returns the Result at the specified _index_. If _opts_ are given,
+      # they will be used in accessing the Page containing the requested
+      # Result.
       #
-      def first_page(opts={})
-        page(1,opts)
+      def result_at(index,opts={})
+        page(result_page_index(index),opts)[page_result_index(index)]
+      end
+
+      #
+      # Returns the first Result at the specified _index_. If _opts_ are
+      # given, they will be used in accessing the Page containing the
+      # requested Result.
+      #
+      def first_result(opts={})
+        result_at(1,opts)
       end
 
       #
@@ -387,8 +439,22 @@ module GScraper
       #
       # Returns the rank offset for the specified _page_index_.
       #
-      def page_index_offset(page_index)
-        (page_index.to_i - 1) * @result_per_page.to_i
+      def page_result_offset(page_index)
+        (page_index.to_i - 1) * @results_per_page.to_i
+      end
+
+      #
+      # Returns the in-Page index of the _result_index_.
+      #
+      def page_result_index(result_index)
+        (result_index.to_i - 1) % @results_per_page.to_i
+      end
+
+      #
+      # Returns the page index for the specified _result_index_
+      #
+      def result_page_index(result_index)
+        ((result_index.to_i - 1) / @results_per_page.to_i) + 1
       end
 
     end

data/lib/gscraper/search/result.rb

@@ -14,15 +14,54 @@ module GScraper
       # Summary from the result page
       attr_reader :summary
 
+      # URL of the cached result page
+      attr_reader :cached_url
+
+      # URL of the similar results Query
+      attr_reader :similar_url
+
       #
       # Creates a new Result object with the given _rank_, _title_
-      # _summary_, _url_ and _size_.
+      # _summary_, _url_, _size_, _cache_url_ and _similar_url_.
       #
-      def initialize(rank,title,url,summary)
+      def initialize(rank,title,url,summary,cached_url=nil,similar_url=nil)
         @rank = rank
         @title = title
         @url = url
         @summary = summary
+        @cached_url = cached_url
+        @similar_url = similar_url
+      end
+
+      #
+      # Opens the URL of the cached page for the Result. If _opts_ are
+      # given, they will be used in accessing the cached page URL.
+      #
+      #   result.cached_page # => File
+      #
+      def cached_page(opts={})
+        if @cached_url
+          return GScraper.open(@cached_url,opts)
+        end
+      end
+
+      #
+      # Create a new Query for results that are similar to the Result. If
+      # a _block_ is given, it will be passed the newly created Query
+      # object.
+      #
+      #   result.similar_query # => Query
+      #
+      #   result.similar_query do |q|
+      #     q.first_page.each_url do |url|
+      #       puts url
+      #     end
+      #   end
+      #
+      def similar_query(&block)
+        if @similar_url
+          return Query.from_url(@similar_url,&block)
+        end
       end
 
       #

data/lib/gscraper/version.rb

@@ -1,3 +1,3 @@
 module GScraper
-  VERSION = '0.1.3'
+  VERSION = '0.1.4'
 end

data/test/search/page_results.rb

@@ -0,0 +1,103 @@
+require 'test/unit'
+require 'gscraper/search/page'
+require 'gscraper/search/query'
+
+class PageResults < Test::Unit::TestCase
+
+  include GScraper
+
+  def setup
+    @query = Search::Query.new(:query => 'ruby')
+    @page = @query.first_page
+  end
+
+  def test_results_per_page
+    assert_equal @page.length, @query.results_per_page
+  end
+
+  def test_first_result
+    assert_not_nil @page[0], "First Page for Query 'ruby' does not have a first Result"
+  end
+
+  def test_last_result
+    assert_not_nil @page[-1], "First Page for Query 'ruby' does not have a last Result"
+  end
+
+  def test_ranks
+    ranks = @page.ranks
+
+    assert_not_nil ranks, "First Page for Query 'ruby' does not have any ranks"
+
+    assert_equal ranks.class, Array, "The ranks of a Page must be an Array"
+
+    assert_equal ranks.empty?, false, "The ranks of the First Page are empty"
+
+    assert_equal ranks.length, @page.length
+  end
+
+  def test_titles
+    titles = @page.titles
+
+    assert_not_nil titles, "First Page for Query 'ruby' does not have any titles"
+
+    assert_equal titles.class, Array, "The titles of a Page must be an Array"
+
+    assert_equal titles.empty?, false, "The titles of the First Page are empty"
+
+    assert_equal titles.length, @page.length
+  end
+
+  def test_urls
+    urls = @page.urls
+
+    assert_not_nil urls, "First Page for Query 'ruby' does not have any urls"
+
+    assert_equal urls.class, Array, "The urls of a Page must be an Array"
+
+    assert_equal urls.empty?, false, "The urls of the First Page are empty"
+
+    assert_equal urls.length, @page.length
+  end
+
+  def test_summaries
+    summaries = @page.summaries
+
+    assert_not_nil summaries, "First Page for Query 'ruby' does not have any summaries"
+
+    assert_equal summaries.class, Array, "The summaries of a Page must be an Array"
+
+    assert_equal summaries.empty?, false, "The summaries of the First Page are empty"
+
+    assert_equal summaries.length, @page.length
+  end
+
+  def test_cached_urls
+    cached_urls = @page.cached_urls
+
+    assert_not_nil cached_urls, "First Page for Query 'ruby' does not have any cached_urls"
+
+    assert_equal cached_urls.class, Array, "The cached_urls of a Page must be an Array"
+
+    assert_equal cached_urls.empty?, false, "The cached_urls of the First Page are empty"
+
+    assert_equal cached_urls.length, @page.length
+  end
+
+  def test_similar_urls
+    similar_urls = @page.similar_urls
+
+    assert_not_nil similar_urls, "First Page for Query 'ruby' does not have any similar URLs"
+
+    assert_equal similar_urls.class, Array, "The similar URLs of a Page must be an Array"
+
+    assert_equal similar_urls.empty?, false, "The similar URLs of the First Page are empty"
+
+    assert_equal similar_urls.length, @page.length
+  end
+
+  def teardown
+    @page = nil
+    @query = nil
+  end
+
+end

data/test/search/query_from_url.rb

@@ -11,10 +11,6 @@ class QueryFromURL < Test::Unit::TestCase
     @query = Search::Query.from_url(QUERY_URL)
   end
 
-  def teardown
-    @query = nil
-  end
-
   def test_query
     assert_equal @query.query, 'test'
   end
@@ -47,4 +43,8 @@ class QueryFromURL < Test::Unit::TestCase
     assert_nil @query.links_to
   end
 
+  def teardown
+    @query = nil
+  end
+
 end

data/test/search/query_pages.rb

@@ -0,0 +1,32 @@
+require 'test/unit'
+require 'gscraper/search/query'
+
+class QueryPages < Test::Unit::TestCase
+
+  include GScraper
+
+  def setup
+    @query = Search::Query.new(:query => 'ruby')
+  end
+
+  def test_first_page
+    page = @query.first_page
+
+    assert_not_nil page
+    assert_equal page.empty?, false, "Query of 'ruby' has zero results"
+    assert_equal page.length, @query.results_per_page
+  end
+
+  def test_second_page
+    page = @query.page(2)
+
+    assert_not_nil page
+    assert_equal page.empty?, false, "Query of 'ruby' has zero results"
+    assert_equal page.length, @query.results_per_page
+  end
+
+  def teardown
+    @query = nil
+  end
+
+end

data/test/search/query_result.rb

@@ -0,0 +1,30 @@
+require 'test/unit'
+require 'gscraper/search/query'
+
+class QueryResult < Test::Unit::TestCase
+
+  include GScraper
+
+  def setup
+    @query = Search::Query.new(:query => 'ruby')
+  end
+
+  def test_first_result
+    result = @query.first_result
+
+    assert_not_nil result, "The Query for 'ruby' has no first-result"
+    assert_equal result.rank, 1, "The first result for the Query 'ruby' does not have the rank of 1"
+  end
+
+  def test_second_result
+    result = @query.result_at(2)
+
+    assert_not_nil result, "The Query for 'ruby' has no second-result"
+    assert_equal result.rank, 2, "The second result for the Query 'ruby' does not have the rank of 2"
+  end
+
+  def teardown
+    @query = nil
+  end
+
+end

data/test/test_gscraper.rb

@@ -2,3 +2,6 @@ $LOAD_PATH.unshift(File.expand_path(File.join(File.dirname(__FILE__),'..','lib')
 
 require 'test/unit'
 require 'search/query_from_url'
+require 'search/query_result'
+require 'search/query_pages'
+require 'search/page_results'

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.9.4
 specification_version: 1
 name: gscraper
 version: !ruby/object:Gem::Version 
-  version: 0.1.3
-date: 2007-12-21 00:00:00 -08:00
+  version: 0.1.4
+date: 2007-12-22 00:00:00 -08:00
 summary: A ruby web-scraping interface to various Google Services
 require_paths: 
 - lib
@@ -48,6 +48,9 @@ files:
 - lib/gscraper/search.rb
 - test/test_gscraper.rb
 - test/search/query_from_url.rb
+- test/search/query_result.rb
+- test/search/query_pages.rb
+- test/search/page_results.rb
 test_files: 
 - test/test_gscraper.rb
 rdoc_options: 
@@ -65,6 +68,24 @@ extensions: []
 requirements: []
 
 dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: hpricot
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Version::Requirement 
+    requirements: 
+    - - ">"
+      - !ruby/object:Gem::Version 
+        version: 0.0.0
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: mechanize
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Version::Requirement 
+    requirements: 
+    - - ">"
+      - !ruby/object:Gem::Version 
+        version: 0.0.0
+    version: 
 - !ruby/object:Gem::Dependency 
   name: hoe
   version_requirement: