gscraper 0.1.5 → 0.1.6

data/lib/gscraper/search/result.rb

@@ -1,7 +1,12 @@
+require 'gscraper/search/query'
+require 'gscraper/web_agent'
+
 module GScraper
   module Search
     class Result
 
+      include WebAgent
+
       # Rank of the result page
       attr_reader :rank
 
@@ -33,18 +38,6 @@ module GScraper
         @similar_url = similar_url
       end
 
-      #
-      # Opens the URL of the cached page for the Result. If _opts_ are
-      # given, they will be used in accessing the cached page URL.
-      #
-      #   result.cached_page # => File
-      #
-      def cached_page(opts={})
-        if @cached_url
-          return GScraper.open(@cached_url,opts)
-        end
-      end
-
       #
       # Create a new Query for results that are similar to the Result. If
       # a _block_ is given, it will be passed the newly created Query
@@ -64,6 +57,14 @@ module GScraper
         end
       end
 
+      #
+      # Fetches the cached page of the result. If a _block_ is given it will
+      # be passed the cached page.
+      #
+      def cached_page(&block)
+        get_page(@cached_url,&block)
+      end
+
       #
       # Returns a string containing the result's title.
       #

data/lib/gscraper/search/search.rb

@@ -3,7 +3,7 @@ require 'gscraper/search/query'
 module GScraper
   module Search
     #
-    # Returns a new Query object with the given _opts_. See Query.new.
+    # Returns a new Query object with the given _options_. See Query.new.
     #
     #   Search.query(:query => 'ruby', :with_words => 'sow rspec')
     #
@@ -11,8 +11,8 @@ module GScraper
     #     q.within_past_week = true
     #   end
     #
-    def Search.query(opts={},&block)
-      Query.new(opts,&block)
+    def Search.query(options={},&block)
+      Query.new(options,&block)
     end
 
     #

data/lib/gscraper/sponsored_ad.rb

@@ -0,0 +1,35 @@
+require 'gscraper/extensions/uri'
+
+module GScraper
+  class SponsoredAd
+
+    # Title of the ad
+    attr_reader :title
+
+    # URL of the ad
+    attr_reader :url
+
+    #
+    # Creates a new SponsoredAd with the specified _title_ and _url_.
+    #
+    def initialize(title,url)
+      @title = title
+      @url = URI.parse(url)
+    end
+
+    #
+    # Returns the direct URL of the ad.
+    #
+    def direct_url
+      @url.query_params['adurl'] || @url.query_params['q']
+    end
+
+    #
+    # Returns the title of the ad.
+    #
+    def to_s
+      @title.to_s
+    end
+
+  end
+end

data/lib/gscraper/sponsored_links.rb

@@ -0,0 +1,151 @@
+require 'gscraper/sponsored_ad'
+
+module GScraper
+  class SponsoredLinks < Array
+    #
+    # Creates a new SponsoredLinks object with the given _ads_.
+    #
+    def initialize(ads=[])
+      super(ads)
+    end
+
+    #
+    # Returns a mapped Array of the ads within the SponsoredLinks
+    # using the given _block_. If the _block_ is not given, the
+    # SponsoredLinks will be returned.
+    #
+    #   sponsored.map # => SponsoredLinks
+    #
+    #   sponsored.map { |ad| ad.url } # => [...]
+    #
+    def map(&block)
+      return self unless block
+
+      mapped = []
+
+      each { |ad| mapped << block.call(ad) }
+      return mapped
+    end
+
+    #
+    # Selects the ads within the SponsoredLinks which match the given _block_.
+    #
+    #   sponsored.select { |ad| ad.title =~ /consume/i }
+    #
+    def select(&block)
+      SponsoredLinks.new(super(&block))
+    end
+
+    #
+    # Selects the ads using the specified _block_.
+    #
+    #   sponsored.ads_with { |ad| ad.title =~ /status symbol/ }
+    #
+    def ads_with(&block)
+      select(&block)
+    end
+
+    #
+    # Selects the ads with the matching _title_. The _title_ may be
+    # either a String or a Regexp. If _block_ is given, each matching
+    # ad will be passed to the _block_.
+    #
+    #   sponsored.ads_with_title('be attractive') #=> SponsoredLinks
+    #
+    #   sponsored.ads_with_title(/buy me/) do |ad|
+    #     puts ad.url
+    #   end
+    #
+    def ads_with_title(title,&block)
+      if title.kind_of?(Regexp)
+        ads = ads_with { |ad| ad.title =~ title }
+      else
+        ads = ads_with { |ad| ad.title == title }
+      end
+
+      ads.each(&block) if block
+      return ads
+    end
+
+    #
+    # Selects the ads with the matching _url_. The _url_ may be
+    # either a String or a Regexp. If _block_ is given, each matching
+    # ad will be passed to the _block_.
+    #
+    #   sponsored.ads_with_url(/\.com/) # => SponsoredLinks
+    #
+    #   sponsored.ads_with_url(/^https:\/\//) do |ad|
+    #     puts ad.title
+    #   end
+    #
+    def ads_with_url(url,&block)
+      if url.kind_of?(Regexp)
+        ads = ads_with { |ad| ad.url =~ url }
+      else
+        ads = ads_with { |ad| ad.url == url }
+      end
+
+      ads.each(&block) if block
+      return ads
+    end
+
+    #
+    # Returns an Array containing the titles of the ads within the
+    # SponsoredLinks.
+    #
+    #   sponsored.titles # => [...]
+    #
+    def titles
+      map { |ad| ad.title }
+    end
+
+    #
+    # Returns an Array containing the URLs of the ads within the
+    # SponsoredLinks.
+    #
+    #   sponsored.urls # => [...]
+    #
+    def urls
+      map { |ad| ad.url }
+    end
+
+    #
+    # Iterates over each ad's title within the SponsoredLinks, passing each to
+    # the given _block_.
+    #
+    #   each_title { |title| puts title }
+    #
+    def each_title(&block)
+      titles.each(&block)
+    end
+
+    #
+    # Iterates over each ad's url within the SponsoredLinks, passing each to
+    # the given _block_.
+    #
+    #   each_url { |url| puts url }
+    #
+    def each_url(&block)
+      urls.each(&block)
+    end
+
+    #
+    # Returns the titles of the ads that match the specified _block_.
+    #
+    #   sponsored.titles_of { |ad| ad.url.include?('www') }
+    #
+    def titles_of(&block)
+      ads_with(&block).titles
+    end
+
+    #
+    # Returns the urls of the ads that match the specified _block_.
+    #
+    #   sponsored.urls_of { |ad| ad.title =~ /buy these pants/ }
+    #
+    def urls_of(&block)
+      ads_with(&block).urls
+    end
+
+  end
+end

data/lib/gscraper/version.rb

@@ -1,3 +1,3 @@
 module GScraper
-  VERSION = '0.1.5'
+  VERSION = '0.1.6'
 end

data/lib/gscraper/web_agent.rb

@@ -0,0 +1,38 @@
+require 'gscraper/gscraper'
+
+module GScraper
+  module WebAgent
+    protected
+
+    #
+    # Returns the WWW::Mechanize agent.
+    #
+    def web_agent(&block)
+      @web_agent ||= GScraper.web_agent
+
+      block.call(@web_agent) if block
+      return @web_agent
+    end
+
+    #
+    # Fetches the specified _url_, with the given _referer_ using the
+    # web_agent.
+    #
+    #   get_page('http://www.hackety.org/')
+    #
+    def get_page(url,referer=nil,&block)
+      web_agent.get(url,referer,&block)
+    end
+
+    #
+    # Posts the specified _url_ and the given _query_ parameters using the
+    # web_agent.
+    #
+    #   post_page('http://www.wired.com/', :q => 'the future')
+    #
+    def post_page(url,query={})
+      web_agent.post(url,query)
+    end
+
+  end
+end

data/test/search/query_result.rb

@@ -10,7 +10,7 @@ class QueryResult < Test::Unit::TestCase
   end
 
   def test_first_result
-    result = @query.first_result
+    result = @query.top_result
 
     assert_not_nil result, "The Query for 'ruby' has no first-result"
     assert_equal result.rank, 1, "The first result for the Query 'ruby' does not have the rank of 1"

data/test/test_gscraper.rb

@@ -1,7 +1,4 @@
 $LOAD_PATH.unshift(File.expand_path(File.join(File.dirname(__FILE__),'..','lib')))
 
 require 'test/unit'
-require 'search/query_from_url'
-require 'search/query_result'
-require 'search/query_pages'
-require 'search/page_results'
+require 'search'

metadata

@@ -1,33 +1,55 @@
 --- !ruby/object:Gem::Specification 
-rubygems_version: 0.9.4
-specification_version: 1
 name: gscraper
 version: !ruby/object:Gem::Version 
-  version: 0.1.5
-date: 2007-12-29 00:00:00 -08:00
-summary: A ruby web-scraping interface to various Google Services
-require_paths: 
-- lib
-email: postmodern.mod3@gmail.com
-homepage: "    by Postmodern Modulus III"
-rubyforge_project: gscraper
-description: "== FEATURES/PROBLEMS:  * Supports the Google Search service. * Provides HTTP access with custom User-Agent strings.  == REQUIREMENTS:  * Hpricot * Mechanize  == INSTALL:"
-autorequire: 
-default_executable: 
-bindir: bin
-has_rdoc: true
-required_ruby_version: !ruby/object:Gem::Version::Requirement 
-  requirements: 
-  - - ">"
-    - !ruby/object:Gem::Version 
-      version: 0.0.0
-  version: 
+  version: 0.1.6
 platform: ruby
-signing_key: 
-cert_chain: 
-post_install_message: 
 authors: 
 - Postmodern Modulus III
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2008-03-15 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: hpricot
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: mechanize
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.5.1
+    version: 
+description: "== FEATURES/PROBLEMS:  * Supports the Google Search service. * Provides access to search results and ranks. * Provides access to the Sponsored Links. * Provides HTTP access with custom User-Agent strings. * Provides proxy settings for HTTP access.  == REQUIREMENTS:  * Hpricot * WWW::Mechanize  == INSTALL:"
+email: postmodern.mod3@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- History.txt
+- LICENSE.txt
+- Manifest.txt
+- README.txt
 files: 
 - History.txt
 - LICENSE.txt
@@ -37,10 +59,13 @@ files:
 - lib/gscraper.rb
 - lib/gscraper/version.rb
 - lib/gscraper/gscraper.rb
+- lib/gscraper/web_agent.rb
 - lib/gscraper/extensions/uri/http.rb
 - lib/gscraper/extensions/uri.rb
 - lib/gscraper/extensions.rb
 - lib/gscraper/licenses.rb
+- lib/gscraper/sponsored_ad.rb
+- lib/gscraper/sponsored_links.rb
 - lib/gscraper/search/result.rb
 - lib/gscraper/search/page.rb
 - lib/gscraper/search/query.rb
@@ -51,47 +76,32 @@ files:
 - test/search/query_result.rb
 - test/search/query_pages.rb
 - test/search/page_results.rb
-test_files: 
-- test/test_gscraper.rb
+has_rdoc: true
+homepage: "    by Postmodern Modulus III"
+post_install_message: 
 rdoc_options: 
 - --main
 - README.txt
-extra_rdoc_files: 
-- History.txt
-- LICENSE.txt
-- Manifest.txt
-- README.txt
-executables: []
-
-extensions: []
-
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
 requirements: []
 
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  name: hpricot
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Version::Requirement 
-    requirements: 
-    - - ">"
-      - !ruby/object:Gem::Version 
-        version: 0.0.0
-    version: 
-- !ruby/object:Gem::Dependency 
-  name: mechanize
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Version::Requirement 
-    requirements: 
-    - - ">"
-      - !ruby/object:Gem::Version 
-        version: 0.0.0
-    version: 
-- !ruby/object:Gem::Dependency 
-  name: hoe
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Version::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: 1.4.0
-    version: 
+rubyforge_project: gscraper
+rubygems_version: 1.0.1
+signing_key: 
+specification_version: 2
+summary: A ruby web-scraping interface to various Google Services
+test_files: 
+- test/test_gscraper.rb