yahoo_site_explorer 0.0.1 → 0.0.2

data/README.rdoc

@@ -2,6 +2,8 @@
 
 The Yahoo! Site Explorer service provides access to Yahoo!'s information about web pages.  The service stores information about links between web pages and and can gauge the popularity of a given page.
 
+The Site Explorer APIs are limited to 5,000 queries per IP address per day and to noncommercial use. See information on {rate limiting}[http://developer.yahoo.com/search/rate.html].
+
 == Installation
 
 To install, simply:
@@ -11,17 +13,43 @@ To install, simply:
 
 == Example
 
-The following example queries Yahoo! Site Explorer for the number of inlinks (backlinks) for www.google.com:
+The following example queries Yahoo! Site Explorer for backlinks (inlink data) information about 'http://www.yahoo.com':
 
     require 'yahoo_site_explorer'
     
-    service = YahooSiteExplorer.new('myapikey')
-    results = service.backlinks('http://www.google.com')
-    puts results.total_results #=> 941822
+    service   = YahooSiteExplorer.new('1234--MyAPIKeyHere=abcd--')
+    backlinks = service.backlinks('http://www.yahoo.com')
+    
+    puts backlinks.total_results_available  #=> 941822
+    puts backlinks.results.first.title      #=> HTML page title
+    puts backlinks.results.first.url        #=> HTML page title
+    puts backlinks.results.first.click_url  #=> HTML page title
+    
+    # Uses a cursor, requeries Yahoo! as necessary, and steps through all
+    # results.
+    backlinks.each do |link|
+      puts link.title
+    end
+
+== Supported Services
+
+This library supports the following endpoints for the Yahoo! Site Explorer web service:
+
+Inlink Data::
+  Shows the pages from other sites linking in to a page.
+
+Page Data::
+  Shows a list of all pages belonging to a domain in the Yahoo! index.
+
+=== Unsupported Services
+
+The following endpoints are not currently supported by this library:
 
-== Caveats
+Ping::
+  Allows you to notify Yahoo! of changes to your site.
 
-Currently, this library only implements the inlinksData, totalResultsAvailable, as that is the only data I need.  I'll extend this to fully support the API, shortly.
+Update Notification::
+  Allows you to notify Yahoo! of changes to your site.
 
 == Copyright
 

data/Rakefile

@@ -58,7 +58,8 @@ Rake::RDocTask.new do |rdoc|
 
   rdoc.rdoc_dir = 'rdoc'
   rdoc.title = "yahoo_site_explorer #{version}"
-  rdoc.rdoc_files.include('README*')
+  rdoc.main = 'README.rdoc'
+  rdoc.rdoc_files.include('README*', 'LICENSE*')
   rdoc.rdoc_files.include('lib/**/*.rb')
 end
 

data/VERSION.yml

@@ -0,0 +1,4 @@
+--- 
+:minor: 0
+:patch: 2
+:major: 0

data/lib/yahoo_site_explorer.rb

@@ -1,9 +1,10 @@
 require 'relax'
 require 'yahoo_site_explorer/api'
 require 'yahoo_site_explorer/backlinks'
+require 'yahoo_site_explorer/page_data'
 
 ##
-# 
+# Provides a Ruby interface ot the Yahoo! Site Explorer web service.
 # 
 class YahooSiteExplorer
   
@@ -13,9 +14,83 @@ class YahooSiteExplorer
   end
   
   
+  ##
+  # Queries Yahoo! Site Explorer for backlinks (inlink data) to the given
+  # URL.
+  # 
+  # === Example
+  # 
+  #     yahoo_site_explorer.backlinks('http://www.yahoo.com', :results => 50)
+  # 
+  # === Options
+  # 
+  # The following options would be passed into the method as a symbolized key
+  # and value pair.
+  # 
+  # results::
+  #   The number of results to return.
+  #   Default: 50.
+  #   Maximum: 100.
+  # 
+  # start::
+  #   The starting result position to return.  The finishing position cannot 
+  #   exceed 1000.
+  #   Default: 1.
+  # 
+  # entire_site::
+  #   Specifies whether to provide results for the entire site, or just the 
+  #   page referenced by the query.  If the query is not a domain URL (i.e. 
+  #   it contains path information, such as 
+  #   http://smallbusiness.yahoo.com/webhosthing/), this parameter has no 
+  #   effect.
+  #   Default: no value.
+  #   Other possible values: '1'
+  # 
+  # omit_inlinks::
+  #   If specified, inlinks will not be returned if they are from pages in the same domain/subdomain as the requested page.
+  #   Default: 'none'
+  #   Other possible values: 'domain', 'subdomain'
+  # 
   def backlinks(url, options = {})
     options[:query] ||= url
-    Backlinks.new(api.inlink_data(options))
+    Backlinks.new(self, options, api.inlink_data(options))
+  end
+  
+  ##
+  # Queries Yahoo! Site Explorer for page data on the given URL.
+  # 
+  # === Example
+  # 
+  #     yahoo_site_explorer.page_data('http://www.yahoo.com', :results => 50)
+  # 
+  # === Options
+  # 
+  # The following options would be passed into the method as a symbolized key
+  # and value pair.
+  # 
+  # results::
+  #   The number of results to return.
+  #   Default: 50.
+  #   Maximum: 100.
+  # 
+  # start::
+  #   The starting result position to return.  The finishing position cannot 
+  #   exceed 1000.
+  #   Default: 1.
+  # 
+  # domain_only::
+  #   Specifies whether to provide results for all subdomains (such as 
+  #   http://search.yahoo.com for http://www.yahoo.com) of the domain query, 
+  #   or just the specifically requested domain.  If the query is not a domain 
+  #   URL (i.e. it contains path information, such as 
+  #   http://smallbusiness.yahoo.com/webhosting/), this parameter has not 
+  #   affect.
+  #   Default: no value
+  #   Other possible values: '1'
+  # 
+  def page_data(url, options = {})
+    options[:query] ||= url
+    PageData.new(self, options, api.page_data(options))
   end
   
   

data/lib/yahoo_site_explorer/api.rb

@@ -5,20 +5,40 @@ class YahooSiteExplorer
     defaults do
       parameter :appid, :required => true
       parameter :query, :required => true
+      parameter :results
+      parameter :start
     end
     
     endpoint 'http://search.yahooapis.com/SiteExplorerService/V1/' do
       
       action :inlink_data, :url => 'inlinkData' do
-        parameter :results
-        parameter :start
         parameter :entire_site
         parameter :omit_inlinks
-        parameter :output
-        parameter :callback
         
         parser 'ResultSet' do
-          element :total_results, :attribute => 'totalResultsAvailable', :xpath => './/@totalResultsAvailable'
+          element :total_results_available, :attribute => 'totalResultsAvailable',  :xpath => './/@totalResultsAvailable'
+          element :first_result_position,   :attribute => 'firstResultPosition',    :xpath => './/@firstResultPosition'
+          element :total_results_returned,  :attribute => 'totalResultsReturned',   :xpath => './/@totalResultsReturned'
+          elements 'Result',                :as => :results do
+            element 'Title',                :as => :title
+            element 'Url',                  :as => :url
+            element 'ClickUrl',             :as => :click_url
+          end
+        end
+      end
+      
+      action :page_data, :url => 'pageData' do
+        parameter :domain_only
+        
+        parser 'ResultSet' do
+          element :total_results_available, :attribute => 'totalResultsAvailable',  :xpath => './/@totalResultsAvailable'
+          element :first_result_position,   :attribute => 'firstResultPosition',    :xpath => './/@firstResultPosition'
+          element :total_results_returned,  :attribute => 'totalResultsReturned',   :xpath => './/@totalResultsReturned'
+          elements 'Result',                :as => :results do
+            element 'Title',                :as => :title
+            element 'Url',                  :as => :url
+            element 'ClickUrl',             :as => :click_url
+          end
         end
       end
       

data/lib/yahoo_site_explorer/backlinks.rb

@@ -1,15 +1,48 @@
+require 'yahoo_site_explorer/results_container'
+require 'yahoo_site_explorer/result'
+
 class YahooSiteExplorer
   
-  class Backlinks #:nodoc:
+  class Backlinks < ResultsContainer
     
-    attr_reader :total_results
-    
-    def initialize(backlinks_hash)
-      self.total_results = backlinks_hash[:total_results]
+    ##
+    # This method will step through all of the results supplied by Yahoo! for 
+    # your given query.  This method acts like a cursor and will 
+    # automatically re-query Yahoo! after for any subsequent set of +results+ 
+    # results (i.e. if your original query asked for 50 results at a time,
+    # this method will act as a cursor, pulling 50 results at a time over the 
+    # entire resulting collection).
+    # 
+    def each
+      backlinks = self
+      records   = self.results
+      while !records.empty?
+        records.each { |record| yield record }
+        backlinks = backlinks.next_set
+        records   = backlinks.results
+      end
+      self
     end
     
-    def total_results=(count)
-      @total_results = count ? count.to_i : nil
+    ##
+    # Returns the next Backlinks set based on available results from Yahoo!
+    # specific to the current query and request options.
+    # 
+    def next_set #:nodoc:
+      if next_starting_position <= @total_results_available
+        @service.backlinks( @request_options.delete(:url), 
+                            @request_options.merge({
+                              :start => next_starting_position
+                            })
+                          )
+      else
+        Backlinks.new(@service, @request_options, {
+          :total_results_available  => @total_results_available,
+          :total_results_returned   => 0,
+          :first_result_position    => next_starting_position,
+          :results                  => []
+        })
+      end
     end
     
   end

data/lib/yahoo_site_explorer/page_data.rb

@@ -0,0 +1,50 @@
+require 'yahoo_site_explorer/results_container'
+require 'yahoo_site_explorer/result'
+
+class YahooSiteExplorer
+  
+  class PageData < ResultsContainer
+    
+    ##
+    # This method will step through all of the results supplied by Yahoo! for 
+    # your given query.  This method acts like a cursor and will 
+    # automatically re-query Yahoo! after for any subsequent set of +results+ 
+    # results (i.e. if your original query asked for 50 results at a time,
+    # this method will act as a cursor, pulling 50 results at a time over the 
+    # entire resulting collection).
+    #
+    def each
+      page_data = self
+      records   = self.results
+      while !records.empty?
+        records.each { |record| yield record }
+        page_data = page_data.next_set
+        records   = page_data.results
+      end
+      self
+    end
+    
+    ##
+    # Returns the next PageData set based on available results from Yahoo!
+    # specific to the current query and request options.
+    #
+    def next_set #:nodoc:
+      if next_starting_position <= @total_results_available
+        @service.page_data( @request_options.delete(:url), 
+                            @request_options.merge({
+                              :start => next_starting_position
+                            })
+                          )
+      else
+        PageData.new(@service, @request_options, {
+          :total_results_available  => @total_results_available,
+          :total_results_returned   => 0,
+          :first_result_position    => next_starting_position,
+          :results                  => []
+        })
+      end
+    end
+    
+  end
+  
+end

data/lib/yahoo_site_explorer/result.rb

@@ -0,0 +1,16 @@
+class YahooSiteExplorer
+  
+  ##
+  # Wraps each result returned by Yahoo! backlinks (inlink data) and page data
+  # requests.
+  # 
+  class Result
+    attr_accessor :title, :url, :click_url
+    
+    def initialize(title, url, click_url) #:nodoc:
+      @title, @url, @click_url = title, url, click_url
+    end
+    
+  end
+  
+end

data/lib/yahoo_site_explorer/results_container.rb

@@ -0,0 +1,53 @@
+class YahooSiteExplorer
+  
+  class ResultsContainer #:nodoc:
+    include Enumerable
+    
+    attr_reader :total_results_available,
+                :total_results_returned,
+                :first_result_position,
+                :results
+    
+    
+    def initialize(service, request_options, results_hash = {}) #:nodoc:
+      @service          = service
+      @request_options  = request_options
+      parse_hash(results_hash)
+    end
+    
+    
+    
+    protected
+    
+    
+    ##
+    # Returns the starting position for a query that would subsequently 
+    # follow the current result set.
+    # 
+    def next_starting_position
+      @first_result_position + @total_results_returned
+    end
+    
+    def parse_hash(backlinks_hash) #:nodoc:
+      @total_results_available  = numeric(backlinks_hash[:total_results_available])
+      @total_results_returned   = numeric(backlinks_hash[:total_results_returned])
+      @first_result_position    = numeric(backlinks_hash[:first_result_position])
+      @results                  = collect_results(backlinks_hash[:results])
+    end
+    
+    def collect_results(results) #:nodoc:
+      collection = []
+      return unless results.respond_to?(:each)
+      results.each do |result|
+        collection << Result.new(result[:title], result[:url], result[:click_url])
+      end
+      collection
+    end
+    
+    def numeric(value, nil_value = nil) #:nodoc:
+      value ? value.to_i : nil_value
+    end
+    
+  end
+  
+end

data/test/integrations/yahoo_site_explorer_test.rb

@@ -7,7 +7,7 @@ class YahooSiteExplorerTest < Test::Unit::TestCase
     context 'backlinks' do
       
       setup do
-        mock_response_for('http://search.yahooapis.com:80/SiteExplorerService/V1/inlinkData?appid=testid&query=http://www.google.com',
+        mock_response_for('http://search.yahooapis.com:80/SiteExplorerService/V1/inlinkData?appid=testid&query=http://www.yahoo.com',
                           :response => mock_inlink_data_successful_response)
       end
       
@@ -16,11 +16,132 @@ class YahooSiteExplorerTest < Test::Unit::TestCase
       end
       
       should 'return Backlinks' do
-        assert_kind_of YahooSiteExplorer::Backlinks, site_explorer.backlinks('http://www.google.com')
+        assert_kind_of YahooSiteExplorer::Backlinks, site_explorer.backlinks('http://www.yahoo.com')
       end
       
-      should 'return total_results as an integer' do
-        assert_equal 941822, site_explorer.backlinks('http://www.google.com').total_results
+      should 'be Enumerable' do
+        assert_kind_of Enumerable, site_explorer.backlinks('http://www.yahoo.com')
+      end
+      
+      should 'return total_results_available as a numeric' do
+        assert_equal 941822, site_explorer.backlinks('http://www.yahoo.com').total_results_available
+      end
+      
+      should 'return first_result_position as a numeric' do
+        assert_equal 1, site_explorer.backlinks('http://www.yahoo.com').first_result_position
+      end
+      
+      should 'return total_results_returned as a numeric' do
+        assert_equal 2, site_explorer.backlinks('http://www.yahoo.com').total_results_returned
+      end
+      
+      should 'return a collection of results' do
+        assert_equal 2, site_explorer.backlinks('http://www.yahoo.com').results.size
+      end
+      
+      should 'contain the result title' do
+        assert_equal 'Common Dreams News Center', site_explorer.backlinks('http://www.yahoo.com').results.first.title
+      end
+      
+      should 'contain the result url' do
+        assert_equal 'http://www.commondreams.org/', site_explorer.backlinks('http://www.yahoo.com').results.first.url
+      end
+      
+      should 'contain the result click_url' do
+        assert_equal 'http://www.commondreams.org/', site_explorer.backlinks('http://www.yahoo.com').results.first.click_url
+      end
+      
+      context 'each' do
+        
+        setup do
+          mock_response_for('http://search.yahooapis.com:80/SiteExplorerService/V1/inlinkData?query=http://www.yahoo.com&appid=testid',
+                            :response => mock_inlink_data_successful_response_set_1)
+          mock_response_for('http://search.yahooapis.com:80/SiteExplorerService/V1/inlinkData?start=51&query=http://www.yahoo.com&appid=testid',
+                            :response => mock_inlink_data_successful_response_set_2)
+        end
+        
+        should 'traverse multiple, contiguous query sets' do
+          results = 0
+          backlinks = site_explorer.backlinks('http://www.yahoo.com')
+          backlinks.each { |b| results += 1 }
+          assert_equal 99, results
+        end
+        
+        should 'yield a Result' do
+          assert site_explorer.backlinks('http://www.yahoo.com').all? { |result| result.kind_of?(YahooSiteExplorer::Result) }
+        end
+        
+      end
+      
+    end
+    
+    context 'page_data' do
+      
+      setup do
+        mock_response_for('http://search.yahooapis.com:80/SiteExplorerService/V1/pageData?appid=testid&query=http://www.yahoo.com',
+                          :response => mock_page_data_successful_response_set_1)
+      end
+      
+      teardown do
+        FakeWeb.clean_registry
+      end
+      
+      should 'return PageData' do
+        assert_kind_of YahooSiteExplorer::PageData, site_explorer.page_data('http://www.yahoo.com')
+      end
+      
+      should 'be Enumerable' do
+        assert_kind_of Enumerable, site_explorer.page_data('http://www.yahoo.com')
+      end
+      
+      should 'return total_results_available as a numeric' do
+        assert_equal 99, site_explorer.page_data('http://www.yahoo.com').total_results_available
+      end
+      
+      should 'return first_result_position as a numeric' do
+        assert_equal 1, site_explorer.page_data('http://www.yahoo.com').first_result_position
+      end
+      
+      should 'return total_results_returned as a numeric' do
+        assert_equal 50, site_explorer.page_data('http://www.yahoo.com').total_results_returned
+      end
+      
+      should 'return a collection of results' do
+        assert_equal 50, site_explorer.page_data('http://www.yahoo.com').results.size
+      end
+      
+      should 'contain the result title' do
+        assert_equal 'Site Explorer - Yahoo! Site Explorer', site_explorer.page_data('http://www.yahoo.com').results.first.title
+      end
+      
+      should 'contain the result url' do
+        assert_equal 'http://siteexplorer.search.yahoo.com/', site_explorer.page_data('http://www.yahoo.com').results.first.url
+      end
+      
+      should 'contain the result click_url' do
+        assert_equal 'http://siteexplorer.search.yahoo.com/', site_explorer.page_data('http://www.yahoo.com').results.first.click_url
+      end
+      
+      context 'each' do
+        
+        setup do
+          mock_response_for('http://search.yahooapis.com:80/SiteExplorerService/V1/pageData?query=http://www.yahoo.com&appid=testid',
+                            :response => mock_page_data_successful_response_set_1)
+          mock_response_for('http://search.yahooapis.com:80/SiteExplorerService/V1/pageData?start=51&query=http://www.yahoo.com&appid=testid',
+                            :response => mock_page_data_successful_response_set_2)
+        end
+        
+        should 'traverse multiple, contiguous query sets' do
+          results = 0
+          pages = site_explorer.page_data('http://www.yahoo.com')
+          pages.each { |b| results += 1 }
+          assert_equal 99, results
+        end
+        
+        should 'yield a Result' do
+          assert site_explorer.page_data('http://www.yahoo.com').all? { |result| result.kind_of?(YahooSiteExplorer::Result) }
+        end
+        
       end
       
     end