openlogic-resourceful 1.2.0

data/lib/resourceful/http_accessor.rb

@@ -0,0 +1,103 @@
+require 'net/http'
+
+require 'resourceful/authentication_manager'
+require 'resourceful/cache_manager'
+require 'resourceful/resource'
+
+require 'options'
+
+module Resourceful
+  # This is an imitation Logger used when no real logger is
+  # registered.  This allows most of the code to assume that there
+  # is always a logger available, which significantly improved the
+  # readability of the logging related code.
+  class BitBucketLogger
+    def warn(*args); end
+    def info(*args); end
+    def debug(*args); end
+  end
+
+  # This is the simplest logger. It just writes everything to STDOUT.
+  class StdOutLogger
+    def warn(*args); puts args; end
+    def info(*args); puts args; end
+    def debug(*args); puts args; end
+  end
+  
+  # This class provides a simple interface to the functionality
+  # provided by the Resourceful library.  Conceptually this object
+  # acts a collection of all the resources available via HTTP.
+  class HttpAccessor
+    # A logger object to which messages about the activities of this
+    # object will be written.  This should be an object that responds
+    # to +#info(message)+ and +#debug(message)+.  
+    #
+    # Errors will not be logged.  Instead an exception will be raised
+    # and the application code should log it if appropriate.
+    attr_accessor :logger, :cache_manager
+    
+    attr_reader :auth_manager
+    attr_reader :user_agent_tokens  
+    
+    ##
+    # The adapter this accessor will use to make the actual HTTP requests.
+    attr_reader :http_adapter
+    
+    # Initializes a new HttpAccessor.  Valid options:
+    #
+    #  `:logger`
+    #  :    A Logger object that the new HTTP accessor should send log messages
+    #
+    #  `:user_agent`
+    #  :    One or more additional user agent tokens to added to the user agent string.
+    #
+    #  `:cache_manager`
+    #  :    The cache manager this accessor should use.
+    #
+    #  `:authenticator`
+    #  :    Add a single authenticator for this accessor.
+    #
+    #  `:authenticators`
+    #  :    Enumerable of the authenticators for this accessor.
+    #
+    #  `http_adapter`
+    #  :    The HttpAdapter to be used by this accessor
+    #
+    # 
+    def initialize(options = {})
+      options = Options.for(options).validate(:logger, :user_agent, :cache_manager, :authenticator, :authenticators, :http_adapter)
+
+      @user_agent_tokens = [RESOURCEFUL_USER_AGENT_TOKEN]
+      @auth_manager = AuthenticationManager.new()
+
+      
+      @user_agent_tokens.push(*Array(options.getopt(:user_agent)).flatten.reverse)
+      self.logger    = options.getopt(:logger) || BitBucketLogger.new
+      @cache_manager = options.getopt(:cache_manager) || NullCacheManager.new
+      @http_adapter  = options.getopt(:http_adapter) || NetHttpAdapter.new
+      
+      Array(options.getopt([:authenticator, :authenticators])).flatten.each do |an_authenticator|
+        add_authenticator(an_authenticator)
+      end
+    end
+    
+    # Returns the string that identifies this HTTP accessor.  If you
+    # want to add a token to the user agent string simply add the new
+    # token to the end of +#user_agent_tokens+.
+    def user_agent_string
+      user_agent_tokens.reverse.join(' ')
+    end
+    
+    # Returns a resource object representing the resource indicated
+    # by the specified URI.  A resource object will be created if necessary.
+    def resource(uri, opts = {})
+      resource = Resource.new(self, uri, opts)
+    end
+    alias [] resource
+
+    # Adds an Authenticator to the set used by the accessor.
+    def add_authenticator(an_authenticator)
+      auth_manager.add_auth_handler(an_authenticator)
+    end
+  end
+end

data/lib/resourceful/memcache_cache_manager.rb

@@ -0,0 +1,75 @@
+require "resourceful/cache_manager"
+
+require 'memcache'
+
+module Resourceful
+  class MemcacheCacheManager < AbstractCacheManager
+  
+    # Create a new Memcached backed cache manager
+    #
+    # @param [*String] memcache_servers  
+    #   list of all Memcached servers this cache manager should use.
+    def initialize(*memcache_servers)
+      @memcache = MemCache.new(memcache_servers, :multithread => true)
+    end
+
+    # Finds a previously cached response to the provided request.  The
+    # response returned may be stale.
+    #
+    # @param [Resourceful::Request] request
+    #   The request for which we are looking for a response.
+    #
+    # @return [Resourceful::Response, nil] 
+    #   A (possibly stale) response for the request provided or nil if 
+    #   no matching response is found.
+    def lookup(request)
+      resp = cache_entries_for(request)[request]
+      return if resp.nil?
+
+      resp.authoritative = false
+
+      resp
+    end
+
+    # Store a response in the cache. 
+    #
+    # This method is smart enough to not store responses that cannot be 
+    # cached (Vary: * or Cache-Control: no-cache, private, ...)
+    #
+    # @param [Resourceful::Request] request
+    #   The request used to obtain the response. This is needed so the 
+    #   values from the response's Vary header can be stored.
+    # @param [Resourceful::Response] response
+    #   The response to be stored.
+    def store(request, response)
+      return unless response.cachable?
+
+      entries = cache_entries_for(request)
+      entries[request] = response
+
+      @memcache[request.to_mc_key] = entries
+    end
+
+    # Invalidates a all cached entries for a uri. 
+    #
+    # This is used, for example, to invalidate the cache for a resource 
+    # that gets POSTed to.
+    #
+    # @param [String] uri
+    #   The uri of the resource to be invalidated
+    def invalidate(uri)
+      @memcache.delete(uri_hash(uri))
+    end
+
+
+    private 
+
+    ##
+    # The memcache proxy.
+    attr_reader :memcache
+
+    def cache_entries_for(a_request)
+      @memcache.get(uri_hash(a_request.uri)) || Resourceful::CacheEntryCollection.new
+    end
+  end
+end

data/lib/resourceful/multipart_form_data.rb

@@ -0,0 +1,46 @@
+require 'resourceful/abstract_form_data'
+
+module Resourceful
+  class MultipartFormData < AbstractFormData
+    FileParamValue = Struct.new(:content, :file_name, :content_type)
+
+    def add_file(name, file_name, content_type="application/octet-stream")
+      add(name, FileParamValue.new(File.new(file_name, 'r'), File.basename(file_name), content_type))
+    end
+
+    def content_type
+      "multipart/form-data; boundary=#{boundary}"
+    end
+
+    def read
+      StringIO.new.tap do |out|
+        first = true
+        form_data.each do |key, val|
+          out << "\r\n" unless first
+          out << "--" << boundary
+          out << "\r\nContent-Disposition: form-data; name=\"#{key}\""
+          if val.kind_of?(FileParamValue)
+            out << "; filename=\"#{val.file_name}\""
+            out << "\r\nContent-Type: #{val.content_type}"
+          end
+          out << "\r\n\r\n"
+          if val.kind_of?(FileParamValue)
+            out << val.content.read
+          else
+            out << val.to_s
+          end
+          first = false
+        end
+        out << "\r\n--#{boundary}--"
+      end.string
+    end
+
+    protected
+
+    def boundary
+      @boundary ||= (0..30).map{BOUNDARY_CHARS[rand(BOUNDARY_CHARS.length)]}.join
+    end
+
+    BOUNDARY_CHARS = [('a'..'z').to_a,('A'..'Z').to_a,(0..9).to_a].flatten
+  end
+end

data/lib/resourceful/net_http_adapter.rb

@@ -0,0 +1,84 @@
+require 'net/http'
+require 'net/https'
+require 'addressable/uri'
+
+require 'pathname'
+require 'resourceful/header'
+
+module Addressable
+  class URI
+    def absolute_path
+      absolute_path = ""
+      absolute_path << self.path.to_s
+      absolute_path << "?#{self.query}" if self.query != nil
+      absolute_path << "##{self.fragment}" if self.fragment != nil
+      return absolute_path
+    end
+  end
+end
+
+module Resourceful
+
+  class NetHttpAdapter
+    # Make an HTTP request using the standard library net/http.
+    #
+    # Will use a proxy defined in the http_proxy environment variable, if set.
+    #
+    # @param [#read] body
+    #   An IO-ish thing containing the body of the request
+    #
+    def make_request(method, uri, body = nil, header = nil)
+      uri = uri.is_a?(Addressable::URI) ? uri : Addressable::URI.parse(uri)
+
+      if [:put, :post].include? method
+        body = body ? body.read : ""
+        header[:content_length] = body.size 
+      end
+
+      req = net_http_request_class(method).new(uri.absolute_path)
+      header.each_field { |k,v| req[k] = v } if header
+      https = ("https" == uri.scheme)
+      conn_class = proxy_details ? Net::HTTP.Proxy(*proxy_details) : Net::HTTP
+      conn = conn_class.new(uri.host, uri.port || (https ? 443 : 80))
+      conn.use_ssl = https
+      begin 
+        conn.start
+        res = if body
+                conn.request(req, body)
+              else
+                conn.request(req)
+              end
+      ensure
+        conn.finish if conn.started?
+      end
+
+      [ Integer(res.code),
+        Resourceful::Header.new(res.header.to_hash),
+        res.body
+      ]
+    ensure
+      
+    end
+
+    private
+
+    # Parse proxy details from http_proxy environment variable
+    def proxy_details
+      proxy = Addressable::URI.parse(ENV["http_proxy"])
+      [proxy.host, proxy.port, proxy.user, proxy.password] if proxy
+    end
+
+    def net_http_request_class(method)
+      case method
+      when :get     then Net::HTTP::Get
+      when :head    then Net::HTTP::Head
+      when :post    then Net::HTTP::Post
+      when :put     then Net::HTTP::Put
+      when :delete  then Net::HTTP::Delete
+      end
+
+    end
+
+  end
+
+end

data/lib/resourceful/promiscuous_basic_authenticator.rb

@@ -0,0 +1,18 @@
+module Resourceful  
+  # This class provides HTTP basic authentication without regard to
+  # the realm of receiving resource.  This will send your username and
+  # password with any request made while it is in play.
+  class PromiscuousBasicAuthenticator < BasicAuthenticator
+    def initialize(username, password)
+      super(nil, username, password)
+    end
+    
+    def valid_for?(challenge_response)
+      true
+    end
+
+    def can_handle?(request)
+      true
+    end
+  end
+end

data/lib/resourceful/request.rb

@@ -0,0 +1,235 @@
+require 'pathname'
+require 'benchmark'
+require 'resourceful/response'
+require 'resourceful/net_http_adapter'
+require 'resourceful/exceptions'
+
+module Resourceful
+
+  class Request
+
+    REDIRECTABLE_METHODS = [:get, :head]
+    CACHEABLE_METHODS = [:get, :head]
+    INVALIDATING_METHODS = [:post, :put, :delete]
+
+    attr_accessor :method, :resource, :body, :header
+    attr_reader   :request_time, :accessor
+
+    # @param [Symbol] http_method
+    #   :get, :put, :post, :delete or :head
+    # @param [Resourceful::Resource] resource
+    # @param [#read, #rewind] body
+    # @param [Resourceful::Header, Hash] header
+    def initialize(http_method, resource, body = nil, header = nil)
+      @method, @resource, @body = http_method, resource, body
+      @accessor = @resource.accessor
+      @header = header.is_a?(Resourceful::Header) ? header : Resourceful::Header.new(header)
+
+      # Resourceful handled gzip encoding transparently, so set that up
+      @header.accept_encoding ||= 'gzip, identity'
+
+      # 'Host' is a required HTTP/1.1 header, overrides Host in user-provided headers
+      @header.host = @resource.host
+
+      # Setting the date isn't a bad idea, either
+      @header.date ||= Time.now.httpdate
+
+      # Add any auth credentials we might want
+      add_credentials!
+
+    end
+
+    # Uses the auth manager to add any valid credentials to this request
+    def add_credentials!
+      @accessor.auth_manager.add_credentials(self)
+    end
+
+    # Performs all the work. Handles caching, redirects, auth retries, etc
+    def fetch_response
+      if cached_response
+        if needs_revalidation?(cached_response)
+          logger.info("    Cache needs revalidation")
+          set_validation_headers!(cached_response)
+        else
+          # We're done!
+          return cached_response
+        end
+      end
+
+      response = perform!
+
+      response = revalidate_cached_response(response) if cached_response && response.not_modified?
+      response = follow_redirect(response)            if should_be_redirected?(response)
+      response = retry_with_auth(response)            if needs_authorization?(response)
+
+      raise UnsuccessfulHttpRequestError.new(self, response) if response.error?
+
+      if cacheable?(response)
+        store_in_cache(response)
+      elsif invalidates_cache?
+        invalidate_cache
+      end
+
+      return response
+    end
+
+    # Should we look for a response to this request in the cache?
+    def skip_cache?
+      return true unless method.in? CACHEABLE_METHODS
+      header.cache_control && header.cache_control.include?('no-cache')
+    end
+
+    # The cached response
+    def cached_response
+      return if skip_cache?
+      return if @cached_response.nil? && @already_checked_cache
+      @cached_response ||= begin
+        @already_checked_cache = true
+        resp = accessor.cache_manager.lookup(self)
+        logger.info("    Retrieved from cache") if resp
+        resp
+      end
+    end
+
+    # Revalidate the cached response with what we got from a 304 response
+    def revalidate_cached_response(not_modified_response)
+      logger.info("    Resource not modified")
+      cached_response.revalidate!(not_modified_response)
+      cached_response
+    end
+
+    # Follow a redirect response
+    def follow_redirect(response)
+      raise MalformedServerResponse.new(self, response) unless response.header.location
+      if response.moved_permanently?
+        new_uri = response.header.location.first
+        logger.info("    Permanently redirected to #{new_uri} - Storing new location.")
+        resource.update_uri new_uri
+        @header.host = resource.host
+        response = fetch_response
+      elsif response.see_other? # Always use GET for this redirect, regardless of initial method
+        redirected_resource = Resourceful::Resource.new(self.accessor, response.header['Location'].first)
+        response = Request.new(:get, redirected_resource, body, header).fetch_response
+      else
+        redirected_resource = Resourceful::Resource.new(self.accessor, response.header['Location'].first)
+        logger.info("    Redirected to #{redirected_resource.uri} - Caching new location.")
+        response = Request.new(method, redirected_resource, body, header).fetch_response
+      end
+    end
+
+    # Add any auth headers from the response to the auth manager, and try the request again
+    def retry_with_auth(response)
+      @already_tried_with_auth = true
+      logger.info("Authentication Required. Retrying with auth info")
+      accessor.auth_manager.associate_auth_info(response)
+      add_credentials!
+      @body.rewind if @body # Its a stringIO, and we already fed it to the adapter once, so rewind it when we try again
+      response = fetch_response
+    end
+
+    # Does this request need to be authorized? Will only be true if we haven't already tried with auth
+    def needs_authorization?(response)
+      !@already_tried_with_auth && response.unauthorized?
+    end
+
+    # Store the response to this request in the cache
+    def store_in_cache(response)
+      # RFC2618 - 14.18 : A received message that does not have a Date header 
+      # field MUST be assigned one by the recipient if the message will be cached 
+      # by that recipient.
+      response.header.date ||= response.response_time.httpdate
+
+      accessor.cache_manager.store(self, response) 
+    end
+
+    # Invalidated the cache for this uri (eg, after a POST)
+    def invalidate_cache
+      accessor.cache_manager.invalidate(resource.uri)
+    end
+
+    # Is this request & response permitted to be stored in this (private) cache?
+    def cacheable?(response)
+      return false unless response.success?
+      return false unless method.in? CACHEABLE_METHODS
+      return false if header.cache_control && header.cache_control.include?('no-store')
+      true
+    end
+
+    # Does this request invalidate the cache?
+    def invalidates_cache?
+      return true if method.in? INVALIDATING_METHODS
+    end
+
+    # Perform the request, with no magic handling of anything.
+    def perform!
+      @request_time = Time.now
+
+      http_resp = adapter.make_request(@method, @resource.uri, @body, @header)
+      @response = Resourceful::Response.new(uri, *http_resp)
+      @response.request_time = @request_time
+      @response.authoritative = true
+
+      @response
+    end
+
+    # Is this a response a redirect, and are we permitted to follow it?
+    def should_be_redirected?(response)
+      return false unless response.redirect?
+      if resource.on_redirect.nil?
+        return true if method.in? REDIRECTABLE_METHODS
+        false
+      else
+        resource.on_redirect.call(self, response)
+      end
+    end
+
+    # Do we need to revalidate our cache?
+    def needs_revalidation?(response)
+      return true if forces_revalidation?
+      return true if response.stale?
+      return true if max_age && response.current_age > max_age
+      return true if response.must_be_revalidated?
+      false
+    end
+
+    # Set the validation headers of a request based on the response in the cache
+    def set_validation_headers!(response)
+      @header['If-None-Match'] = response.header['ETag'] if response.header.has_key?('ETag')
+      @header['If-Modified-Since'] = response.header['Last-Modified'] if response.header.has_key?('Last-Modified')
+      @header['Cache-Control'] = 'max-age=0' if response.must_be_revalidated?    
+    end
+
+    # @return [String]   The URI against which this request will be, or was, made.
+    def uri
+      resource.uri
+    end
+
+    # Does this request force us to revalidate the cache?
+    def forces_revalidation?
+      if max_age == 0 || skip_cache?
+        logger.info("    Client forced revalidation")
+        true
+      else 
+        false
+      end
+    end
+
+    # Indicates the maxmimum response age in seconds we are willing to accept
+    #
+    # Returns nil if we don't care how old the response is
+    def max_age
+      if header['Cache-Control'] and header['Cache-Control'].include?('max-age')
+        header['Cache-Control'].split(',').grep(/max-age/).first.split('=').last.to_i
+      end
+    end
+
+    def logger
+      resource.logger
+    end
+
+    def adapter
+      accessor.http_adapter
+    end
+  end
+
+end