resourceful 0.3.1 → 0.5.0

data/lib/resourceful/net_http_adapter.rb

@@ -3,7 +3,7 @@ require 'net/https'
 require 'addressable/uri'
 
 require 'pathname'
-require Pathname(__FILE__).dirname + 'header'
+require 'resourceful/header'
 
 module Addressable
   class URI
@@ -20,18 +20,22 @@ end
 module Resourceful
 
   class NetHttpAdapter
+    # Make an HTTP request using the standard library net/http.
+    #
+    # Will use a proxy defined in the http_proxy environment variable, if set.
     def self.make_request(method, uri, body = nil, header = nil)
       uri = uri.is_a?(Addressable::URI) ? uri : Addressable::URI.parse(uri)
 
       req = net_http_request_class(method).new(uri.absolute_path)
-      header.each { |k,v| req[k] = v } if header
-      conn = Net::HTTP.new(uri.host, uri.port)
-      conn.use_ssl = (/https/i === uri.scheme)
+      header.each_field { |k,v| req[k] = v } if header
+      https = ("https" == uri.scheme)
+      conn = Net::HTTP.Proxy(*proxy_details).new(uri.host, uri.port || (https ? 443 : 80))
+      conn.use_ssl = https
       begin 
         conn.start
         res = conn.request(req, body)
       ensure
-        conn.finish
+        conn.finish if conn.started?
       end
 
       [ Integer(res.code),
@@ -44,6 +48,12 @@ module Resourceful
 
     private
 
+    # Parse proxy details from http_proxy environment variable
+    def self.proxy_details
+      proxy = Addressable::URI.parse(ENV["http_proxy"])
+      [proxy.host, proxy.port, proxy.user, proxy.password] if proxy
+    end
+
     def self.net_http_request_class(method)
       case method
       when :get     then Net::HTTP::Get

data/lib/resourceful/request.rb

@@ -1,16 +1,19 @@
 require 'pathname'
 require 'benchmark'
-require Pathname(__FILE__).dirname + 'response'
-require Pathname(__FILE__).dirname + 'net_http_adapter'
+require 'resourceful/response'
+require 'resourceful/net_http_adapter'
+require 'resourceful/exceptions'
 
 module Resourceful
 
   class Request
 
     REDIRECTABLE_METHODS = [:get, :head]
+    CACHEABLE_METHODS = [:get, :head]
+    INVALIDATING_METHODS = [:post, :put, :delete]
 
     attr_accessor :method, :resource, :body, :header
-    attr_reader   :request_time
+    attr_reader   :request_time, :accessor
 
     # @param [Symbol] http_method
     #   :get, :put, :post, :delete or :head
@@ -19,28 +22,159 @@ module Resourceful
     # @param [Resourceful::Header, Hash] header
     def initialize(http_method, resource, body = nil, header = nil)
       @method, @resource, @body = http_method, resource, body
-      @header = header.is_a?(Resourceful::Header) ? header : Resourceful::Header.new(header || {})
+      @accessor = @resource.accessor
+      @header = header.is_a?(Resourceful::Header) ? header : Resourceful::Header.new(header)
 
-      @header['Accept-Encoding'] = 'gzip, identity'
-      # 'Host' is a required HTTP/1.1 header, so set it if it isn't already
-      @header['Host'] ||= Addressable::URI.parse(resource.uri).host
+      # Resourceful handled gzip encoding transparently, so set that up
+      @header.accept_encoding ||= 'gzip, identity'
+
+      # 'Host' is a required HTTP/1.1 header, overrides Host in user-provided headers
+      @header.host = @resource.host
 
       # Setting the date isn't a bad idea, either
-      @header['Date'] ||= Time.now.httpdate
+      @header.date ||= Time.now.httpdate
+
+      # Add any auth credentials we might want
+      add_credentials!
+
+    end
+
+    # Uses the auth manager to add any valid credentials to this request
+    def add_credentials!
+      @accessor.auth_manager.add_credentials(self)
+    end
+
+    # Performs all the work. Handles caching, redirects, auth retries, etc
+    def fetch_response
+      if cached_response
+        if needs_revalidation?(cached_response)
+          logger.info("    Cache needs revalidation")
+          set_validation_headers!(cached_response)
+        else
+          # We're done!
+          return cached_response
+        end
+      end
+
+      response = perform!
+
+      response = revalidate_cached_response(response) if cached_response && response.not_modified?
+      response = follow_redirect(response)            if should_be_redirected?(response)
+      response = retry_with_auth(response)            if needs_authorization?(response)
+
+      raise UnsuccessfulHttpRequestError.new(self, response) if response.error?
+
+      if cacheable?(response)
+        store_in_cache(response)
+      elsif invalidates_cache?
+        invalidate_cache
+      end
+
+      return response
+    end
+
+    # Should we look for a response to this request in the cache?
+    def skip_cache?
+      return true unless method.in? CACHEABLE_METHODS
+      header.cache_control && header.cache_control.include?('no-cache')
+    end
+
+    # The cached response
+    def cached_response
+      return if skip_cache?
+      return if @cached_response.nil? && @already_checked_cache
+      @cached_response ||= begin
+        @already_checked_cache = true
+        resp = accessor.cache_manager.lookup(self)
+        logger.info("    Retrieved from cache")
+        resp
+      end
+    end
+
+    # Revalidate the cached response with what we got from a 304 response
+    def revalidate_cached_response(not_modified_response)
+      logger.info("    Resource not modified")
+      cached_response.revalidate!(not_modified_response)
+      cached_response
+    end
+
+    # Follow a redirect response
+    def follow_redirect(response)
+      raise MalformedServerResponse.new(self, response) unless response.header.location
+      if response.moved_permanently?
+        new_uri = response.header.location.first
+        logger.info("    Permanently redirected to #{new_uri} - Storing new location.")
+        resource.update_uri new_uri
+        @header.host = resource.host
+        response = fetch_response
+      elsif response.see_other? # Always use GET for this redirect, regardless of initial method
+        redirected_resource = Resourceful::Resource.new(self.accessor, response.header['Location'].first)
+        response = Request.new(:get, redirected_resource, body, header).fetch_response
+      else
+        redirected_resource = Resourceful::Resource.new(self.accessor, response.header['Location'].first)
+        logger.info("    Redirected to #{redirected_resource.uri} - Caching new location.")
+        response = Request.new(method, redirected_resource, body, header).fetch_response
+      end
     end
 
-    def response
+    # Add any auth headers from the response to the auth manager, and try the request again
+    def retry_with_auth(response)
+      @already_tried_with_auth = true
+      logger.info("Authentication Required. Retrying with auth info")
+      accessor.auth_manager.associate_auth_info(response)
+      add_credentials!
+      response = fetch_response
+    end
+
+    # Does this request need to be authorized? Will only be true if we haven't already tried with auth
+    def needs_authorization?(response)
+      !@already_tried_with_auth && response.unauthorized?
+    end
+
+    # Store the response to this request in the cache
+    def store_in_cache(response)
+      # RFC2618 - 14.18 : A received message that does not have a Date header 
+      # field MUST be assigned one by the recipient if the message will be cached 
+      # by that recipient.
+      response.header.date ||= response.response_time.httpdate
+
+      accessor.cache_manager.store(self, response) 
+    end
+
+    # Invalidated the cache for this uri (eg, after a POST)
+    def invalidate_cache
+      accessor.cache_manager.invalidate(resource.uri)
+    end
+
+    # Is this request & response permitted to be stored in this (private) cache?
+    def cacheable?(response)
+      return false unless response.success?
+      return false unless method.in? CACHEABLE_METHODS
+      return false if header.cache_control && header.cache_control.include?('no-store')
+      true
+    end
+
+    # Does this request invalidate the cache?
+    def invalidates_cache?
+      return true if method.in? INVALIDATING_METHODS
+    end
+
+    # Perform the request, with no magic handling of anything.
+    def perform!
       @request_time = Time.now
+      logger.debug("DEBUG: Request Header: #{@header.inspect}")
 
       http_resp = NetHttpAdapter.make_request(@method, @resource.uri, @body, @header)
-      response = Resourceful::Response.new(uri, *http_resp)
-      response.request_time = @request_time
+      @response = Resourceful::Response.new(uri, *http_resp)
+      @response.request_time = @request_time
+      @response.authoritative = true
 
-      response.authoritative = true
-      response
+      @response
     end
 
-    def should_be_redirected?
+    # Is this a response a redirect, and are we permitted to follow it?
+    def should_be_redirected?(response)
+      return false unless response.redirect?
       if resource.on_redirect.nil?
         return true if method.in? REDIRECTABLE_METHODS
         false
@@ -49,21 +183,49 @@ module Resourceful
       end
     end
 
-    def set_validation_headers(response)
+    # Do we need to revalidate our cache?
+    def needs_revalidation?(response)
+      return true if forces_revalidation?
+      return true if response.stale?
+      return true if max_age && response.current_age > max_age
+      return true if response.must_be_revalidated?
+      false
+    end
+
+    # Set the validation headers of a request based on the response in the cache
+    def set_validation_headers!(response)
       @header['If-None-Match'] = response.header['ETag'] if response.header.has_key?('ETag')
       @header['If-Modified-Since'] = response.header['Last-Modified'] if response.header.has_key?('Last-Modified')
-      @header['Cache-Control'] = 'max-age=0' if response.header.has_key?('Cache-Control') and response.header['Cache-Control'].include?('must-revalidate')
+      @header['Cache-Control'] = 'max-age=0' if response.must_be_revalidated?    
     end
 
     # @return [String]   The URI against which this request will be, or was, made.
     def uri
       resource.uri
     end
-    
+
+    # Does this request force us to revalidate the cache?
+    def forces_revalidation?
+      if max_age == 0 || header.cache_control && cc.include?('no-cache')
+        logger.info("    Client forced revalidation")
+        true
+      else 
+        false
+      end
+    end
+
+    # Indicates the maxmimum response age in seconds we are willing to accept
+    #
+    # Returns nil if we don't care how old the response is
+    def max_age
+      if header['Cache-Control'] and header['Cache-Control'].include?('max-age')
+        header['Cache-Control'].split(',').grep(/max-age/).first.split('=').last.to_i
+      end
+    end
+
     def logger
       resource.logger
     end
-
   end
 
 end

data/lib/resourceful/resource.rb

@@ -1,25 +1,11 @@
 require 'pathname'
-require Pathname(__FILE__).dirname + 'request'
+require 'resourceful/request'
 
 module Resourceful
 
-  # This exception used to indicate that the request did not succeed.
-  # The HTTP response is included so that the appropriate actions can
-  # be taken based on the details of that response
-  class UnsuccessfulHttpRequestError < Exception
-    attr_reader :http_response, :http_request
- 
-    # Initialize new error from the HTTP request and response attributes.
-    def initialize(http_request, http_response)
-      super("#{http_request.method} request to <#{http_request.uri}> failed with code #{http_response.code}")
-      @http_request = http_request
-      @http_response = http_response
-    end
-  end
-
   class Resource
     attr_reader :accessor
-    attr_accessor :default_options
+    attr_accessor :default_header
 
     # Build a new resource for a uri
     #
@@ -27,9 +13,9 @@ module Resourceful
     #   The parent http accessor
     # @param uri<String, Addressable::URI> 
     #   The uri for the location of the resource
-    def initialize(accessor, uri, options = {})
+    def initialize(accessor, uri, default_header = {})
       @accessor, @uris = accessor, [uri]
-      @default_options = options
+      @default_header = Resourceful::Header.new({'User-Agent' => Resourceful::RESOURCEFUL_USER_AGENT_TOKEN}.merge(default_header))
       @on_redirect = nil
     end
 
@@ -44,6 +30,16 @@ module Resourceful
     end
     alias uri effective_uri
 
+    # Returns the host for this Resource's current uri
+    def host
+      Addressable::URI.parse(uri).host
+    end
+
+    # Updates the effective uri after following a permanent redirect
+    def update_uri(uri)
+      @uris.unshift(uri)
+    end
+
     # When performing a redirect, this callback will be executed first. If the callback
     # returns true, then the redirect is followed, otherwise it is not. The request that
     # triggered the redirect and the response will be passed into the block. This can be
@@ -76,9 +72,7 @@ module Resourceful
     # @raise [UnsuccessfulHttpRequestError] unless the request is a
     #   success, ie the final request returned a 2xx response code
     def get(header = {})
-      log_request_with_time "GET [#{uri}]" do
-        do_read_request(:get, header)
-      end
+      request(:get, nil, header)
     end
 
     # :call-seq:
@@ -97,12 +91,9 @@ module Resourceful
     # @raise [ArgumentError] unless :content-type is specified in options
     # @raise [UnsuccessfulHttpRequestError] unless the request is a
     #   success, ie the final request returned a 2xx response code
-    def post(data = "", options = {})
-      raise ArgumentError, ":content_type must be specified" unless options.has_key?(:content_type)
-
-      log_request_with_time "POST [#{uri}]" do
-        do_write_request(:post, data, options)
-      end
+    def post(data = nil, header = {})
+      check_content_type_exists(data, header)
+      request(:post, data, header)
     end
 
     # :call-seq:
@@ -121,12 +112,9 @@ module Resourceful
     # @raise [ArgumentError] unless :content-type is specified in options
     # @raise [UnsuccessfulHttpRequestError] unless the request is a
     #   success, ie the final request returned a 2xx response code
-    def put(data = "", options = {})
-      raise ArgumentError, ":content_type must be specified" unless options.has_key?(:content_type)
-
-      log_request_with_time "PUT [#{uri}]" do
-        do_write_request(:put, data, options)
-      end
+    def put(data, header = {})
+      check_content_type_exists(data, header)
+      request(:put, data, header)
     end
 
     # Performs a DELETE on the resource, following redirects as neccessary.
@@ -135,120 +123,33 @@ module Resourceful
     #
     # @raise [UnsuccessfulHttpRequestError] unless the request is a
     #   success, ie the final request returned a 2xx response code
-    def delete(options = {})
-      log_request_with_time "DELETE [#{uri}]" do
-        do_write_request(:delete, {}, options)
-      end
+    def delete(header = {})
+      request(:delete, nil, header)
     end
 
-    # Performs a read request (HEAD, GET). Users should use the #get, etc methods instead.
-    #
-    # This method handles all the work of following redirects.
-    #
-    # @param method<Symbol> The method to perform
-    #
-    # @return <Response>
-    #
-    # @raise [UnsuccessfulHttpRequestError] unless the request is a
-    #   success, ie the final request returned a 2xx response code
-    #
-    def do_read_request(method, header = {})
-      request = Resourceful::Request.new(method, self, nil, default_options.merge(header))
-      accessor.auth_manager.add_credentials(request)
-
-      cached_response = accessor.cache_manager.lookup(request)
-      if cached_response
-        logger.info("    Retrieved from cache")
-        if not cached_response.stale?
-          # We're done!
-          return cached_response
-        else
-          logger.info("    Cache entry is stale")
-          request.set_validation_headers(cached_response)
-        end
-      end
-
-      response = request.response
-
-      if response.is_not_modified?
-        logger.info("    Resource not modified")
-        cached_response.header.merge!(response.header)
-        cached_response.request_time = response.request_time
-        response = cached_response
-        response.authoritative = true
-      end
-
-      if response.is_redirect? and request.should_be_redirected?
-        if response.is_permanent_redirect?
-          @uris.unshift response.header['Location'].first
-          logger.info("    Permanently redirected to #{uri} - Storing new location.")
-          response = do_read_request(method, header)
-        else
-          redirected_resource = Resourceful::Resource.new(self.accessor, response.header['Location'].first)
-          logger.info("    Redirected to #{redirected_resource.uri} - Storing new location.")
-          response = redirected_resource.do_read_request(method, header)
-        end
-      end
-
-      if response.is_not_authorized? && !@already_tried_with_auth
-        @already_tried_with_auth = true
-        accessor.auth_manager.associate_auth_info(response)
-        logger.info("Authentication Required. Retrying with auth info")
-        response = do_read_request(method, header)
-      end
-
-      raise UnsuccessfulHttpRequestError.new(request,response) unless response.is_success?
-
-      accessor.cache_manager.store(request, response) if response.is_success?
-
-      return response
+    def logger
+      accessor.logger
     end
 
-    # Performs a write request (POST, PUT, DELETE). Users should use the #post, etc 
-    # methods instead.
-    #
-    # This method handles all the work of following redirects.
-    #
-    # @param [Symbol] method  The method to perform
-    # @param [String] data    Body of the http request.
-    # @param [Hash]   header  Header for the HTTP resquest.
-    #
-    # @return [Response]
-    #
-    # @raise [UnsuccessfulHttpRequestError] unless the request is a
-    #   success, ie the final request returned a 2xx response code
-    def do_write_request(method, data = nil, header = {})
-      request = Resourceful::Request.new(method, self, data, default_options.merge(header))
-      accessor.auth_manager.add_credentials(request)
-
-      response = request.response
+    private
 
-      if response.is_redirect? and request.should_be_redirected?
-        if response.is_permanent_redirect?
-          @uris.unshift response.header['Location'].first
-          response = do_write_request(method, data, header)
-        elsif response.code == 303 # see other, must use GET for new location
-          redirected_resource = Resourceful::Resource.new(self.accessor, response.header['Location'].first)
-          response = redirected_resource.do_read_request(:get, header)
-        else
-          redirected_resource = Resourceful::Resource.new(self.accessor, response.header['Location'].first)
-          response = redirected_resource.do_write_request(method, data, header)
-        end
+    # Ensures that the request has a content type header
+    # TODO Move this to request
+    def check_content_type_exists(body, header)
+      if body
+        raise MissingContentType unless header.has_key?(:content_type) or default_header.has_key?(:content_type)
       end
+    end
 
-      if response.is_not_authorized? && !@already_tried_with_auth
-        @already_tried_with_auth = true
-        accessor.auth_manager.associate_auth_info(response)
-        logger.debug("Authentication Required. Retrying with auth info")
-        response = do_write_request(method, data, header)
+    # Actually make the request
+    def request(method, data, header)
+      log_request_with_time "#{method.to_s.upcase} [#{uri}]" do
+        request = Request.new(method, self, data, default_header.merge(header))
+        request.fetch_response
       end
-
-      raise UnsuccessfulHttpRequestError.new(request,response) unless response.is_success?
-
-      accessor.cache_manager.invalidate(uri)
-      return response
     end
 
+    # Log it took the time to make the request
     def log_request_with_time(msg, indent = 2)
       logger.info(" " * indent + msg)
       result = nil
@@ -257,10 +158,6 @@ module Resourceful
       result
     end
 
-    def logger
-      accessor.logger
-    end
-
   end
 
 end