wrest 1.0.2 → 1.1.0

data/lib/wrest.rb

@@ -38,15 +38,46 @@ module Wrest
   end
 
   # Switch Wrest to using Net::HTTP.
-  def self.use_native
+  def self.use_native!
     silence_warnings{ Wrest.const_set('Http', Wrest::Native) }
   end
 
   # Switch Wrest to using libcurl.
-  def self.use_curl
+  def self.use_curl!
     require "#{Wrest::Root}/wrest/curl"
     silence_warnings{ Wrest.const_set('Http', Wrest::Curl) }
   end
+
+  # Loads the Memcached caching back-end and the Dalli gem 
+  def self.enable_memcached_caching!
+    require "#{Wrest::Root}/wrest/components/cache_store/memcached"
+  end
+
+  # Assign the default cache store to be used. Default is none.
+  def self.default_cachestore=(cachestore)
+    @default_cachestore = cachestore
+  end
+
+  # Returns the default cache store, if any is set.
+  def self.default_cachestore
+    @default_cachestore
+  end
+
+  # Configures Wrest to cache all requests. This will use the Memcached backend.
+  def self.always_cache_using_memcached!
+    self.enable_memcached_caching!
+    self.default_cachestore=Wrest::Components::CacheStore::Memcached.new
+  end
+
+  # Configures Wrest to cache all requests. This will use a Ruby Hash.
+  # WARNING: This should NEVER be used in a real environment. The Hash will keep on growing since Wrest does not limit the size of a cache store.
+  #
+  # Use the Memcached caching back-end for production since the Memcached process uses an LRU based cache removal policy
+  # that keeps the number of entries stored within bounds.
+  def self.always_cache_using_hash!
+    Wrest.logger.warn "Using an in-memory Hash as a cache store. This is dangerous if used in a production environment."
+    self.default_cachestore=Hash.new
+  end
 end
 
 Wrest.logger = ActiveSupport::BufferedLogger.new(STDOUT)
@@ -56,16 +87,19 @@ RUBY_PLATFORM =~ /java/ ? gem('json-jruby', '>= 1.4.2') : gem('json', '>= 1.4.2'
 ActiveSupport::JSON.backend = "JSONGem"
 
 require "#{Wrest::Root}/wrest/core_ext/string"
+require "#{Wrest::Root}/wrest/hash_with_case_insensitive_access"
 
 # Load XmlMini Extensions
 require "#{Wrest::Root}/wrest/xml_mini"
 
 # Load Wrest Core
 require "#{Wrest::Root}/wrest/version"
+require "#{Wrest::Root}/wrest/cache_proxy"
 require "#{Wrest::Root}/wrest/http_shared"
 require "#{Wrest::Root}/wrest/http_codes"
 require "#{Wrest::Root}/wrest/native"
 
+
 # Load Wrest Wrappers
 require "#{Wrest::Root}/wrest/uri"
 require "#{Wrest::Root}/wrest/uri_template"

data/lib/wrest/cache_proxy.rb

@@ -0,0 +1,111 @@
+module Wrest
+
+  class CacheProxy
+    class << self
+      def new(get, cache_store)
+        if cache_store
+          DefaultCacheProxy.new(get, cache_store)
+        else
+          NullCacheProxy.new(get)
+        end
+      end
+    end
+
+    class NullCacheProxy
+      def initialize(get)
+        @get = get
+      end
+      def get
+        @get.invoke_without_cache_check
+      end
+    end
+
+    class DefaultCacheProxy
+      HOP_BY_HOP_HEADERS = ["connection",
+                           "keep-alive",
+                           "proxy-authenticate",
+                           "proxy-authorization",
+                           "te",
+                           "trailers",
+                           "transfer-encoding",
+                           "upgrade"]
+
+      def initialize(get, cache_store)
+        @get         = get
+        @cache_store = cache_store
+      end
+
+      def log_cached_response
+          Wrest.logger.debug "<*> (GET #{@get.hash}) #{@get.uri.protocol}://#{@get.uri.host}:#{@get.uri.port}#{@get.http_request.path}"
+      end
+      
+      def get
+        cached_response = @cache_store[@get.hash]
+        return get_fresh_response if cached_response.nil?
+
+        if cached_response.expired?
+          if cached_response.can_be_validated?
+            get_validated_response_for(cached_response)
+          else
+            get_fresh_response
+          end
+        else
+          log_cached_response
+          cached_response
+        end
+      end
+
+      def update_cache_headers_for(cached_response, new_response)
+        # RFC 2616 13.5.3 (Combining Headers)
+        cached_response.headers.merge!(new_response.headers.select {|key, value| not (HOP_BY_HOP_HEADERS.include? key.downcase)})
+      end
+
+      def cache(response)
+        @cache_store[@get.hash] = response.clone if response && response.cacheable?
+      end
+
+      #:nodoc:
+      def get_fresh_response
+        @cache_store.delete @get.hash
+
+        response = @get.invoke_without_cache_check
+
+        cache(response)
+
+        response
+      end
+
+      #:nodoc:
+      def get_validated_response_for(cached_response)
+        new_response = send_validation_request_for(cached_response)
+        if new_response.code == "304"
+          update_cache_headers_for(cached_response, new_response)
+          log_cached_response
+          cached_response
+        else
+          cache(new_response)
+          new_response
+        end
+      end
+
+      #:nodoc:
+      # Send a cache-validation request to the server. This would be the actual Get request with extra cache-validation headers.
+      # If a 304 (Not Modified) is received, Wrest would use the cached_response itself. Otherwise the new response is cached and used.
+      def send_validation_request_for(cached_response)
+        last_modified            = cached_response.last_modified
+        etag                     = cached_response.headers["etag"]
+
+        cache_validation_headers = {}
+        cache_validation_headers["if-modified-since"] = last_modified unless last_modified.nil?
+        cache_validation_headers["if-none-match"] = etag unless etag.nil?
+
+        new_headers =@get.headers.clone.merge cache_validation_headers
+        new_options =@get.options.clone.tap { |opts| opts.delete :cache_store } # do not run this through the caching mechanism.
+
+        new_request = Wrest::Native::Get.new(@get.uri, @get.parameters, new_headers, new_options)
+
+        new_request.invoke
+      end
+    end
+  end
+end

data/lib/wrest/components/cache_store/memcached.rb

@@ -0,0 +1,34 @@
+begin
+  gem 'dalli', '~> 1.0.1'
+rescue Gem::LoadError => e
+  Wrest.logger.debug "Dalli ~> 1.0.1 not found. Dalli is necessary to use the memcached caching back-end. To install dalli run `(sudo) gem install dalli`."
+  raise e
+end
+
+require 'dalli'
+
+module Wrest::Components::CacheStore
+  class Memcached
+
+    def initialize(server_urls=nil, options={})
+      @memcached = Dalli::Client.new(server_urls, options)
+    end
+
+    def [](key)
+      @memcached.get(key)
+    end
+
+    def []=(key, value)
+      @memcached.set(key, value)
+    end
+
+    # should be compatible with Hash - return value of the deleted element.
+    def delete(key)
+      value = self[key]
+      
+      @memcached.delete key
+
+      return value
+    end
+  end
+end

data/lib/wrest/curl.rb

@@ -8,9 +8,9 @@
 # See the License for the specific language governing permissions and limitations under the License.
 
 begin
-  gem 'patron', '~> 0.4.9'
+  gem 'patron', '~> 0.4.11'
 rescue Gem::LoadError => e
-  Wrest.logger.debug "Patron ~> 0.4.9 not found. Patron is necessary to use libcurl. To install Patron run `sudo gem install patron` (patron is not available on JRuby, but you shouldn't need it anyway)."
+  Wrest.logger.debug "Patron ~> 0.4.11 not found. Patron is necessary to use libcurl. To install Patron run `sudo gem install patron` (patron is not available on JRuby, but you shouldn't need it anyway)."
   raise e
 end
 require 'patron'

data/lib/wrest/curl/request.rb

@@ -68,8 +68,8 @@ module Wrest
       # Data about the request is and logged to Wrest.logger
       # The log entry contains the following information:
       #
-      #   --> indicates a request
-      #   <-- indicates a response
+      #   <- indicates a request
+      #   -> indicates a response
       #
       # The type of request is mentioned in caps, followed by a hash
       # uniquely uniquely identifying a particular request/response pair.

data/lib/wrest/hash_with_case_insensitive_access.rb

@@ -0,0 +1,52 @@
+module Wrest
+
+  # A hash with case-insensitive key access.
+  #
+  #   hash = Wrest::HashWithCaseInsensitiveAccess.new 'Abcd' => 1, 'xyz' => 2
+  #
+  #   hash['abcd']  #=> 1
+  #   hash['aBCd'] #=> 1
+  #
+  class HashWithCaseInsensitiveAccess < ::Hash #:nodoc:
+
+    def initialize(hash={})
+      super()
+      hash.each do |key, value|
+        self[convert_key(key)] = value
+      end
+    end
+    def [](key)
+      super(convert_key(key))
+    end
+
+    def []=(key, value)
+      super(convert_key(key), value)
+    end
+
+    def delete(key)
+      super(convert_key(key))
+    end
+
+    def values_at(*indices)
+      indices.collect { |key| self[convert_key(key)] }
+    end
+
+    def merge(other)
+      dup.merge!(other)
+    end
+
+    def merge!(other)
+      other.each do |key, value|
+        self[convert_key(key)] = value
+      end
+      self
+    end
+
+    protected
+
+      def convert_key(key)
+        key.is_a?(String) ? key.downcase : key
+      end
+
+  end
+end

data/lib/wrest/native/get.rb

@@ -8,18 +8,48 @@
 
 module Wrest::Native
   class Get < Request
+
+    attr_reader :cache_proxy
+
     def initialize(wrest_uri, parameters = {}, headers = {}, options = {})
       follow_redirects = options[:follow_redirects]
       options[:follow_redirects] = (follow_redirects == nil ? true : follow_redirects)
-      options[:cache_store] ||= {}
+      @cache_proxy = Wrest::CacheProxy::new(self, options[:cache_store] || Wrest.default_cachestore)
       super(
-            wrest_uri, 
-            Net::HTTP::Get, 
+            wrest_uri,
+            Net::HTTP::Get,
             parameters,
             nil,
             headers,
             options
           )
     end
+
+    # Checks equality between two Wrest::Native::Get objects.
+    # Comparing two Wrest::Native::Get objects with identical values for the following properties would return True.
+    #   uri, parameters, username, password and ssh verify_mode.
+    def ==(other)
+      return true if self.equal?(other)
+      return false unless other.class == self.class
+      return true if self.uri == other.uri and
+        self.parameters == other.parameters and
+        self.username == other.username and
+        self.password == other.password and
+        self.verify_mode == other.verify_mode
+      false
+    end
+
+    # Returns a hash value for this Wrest::Native::Get object.
+    # Objects that returns true when compared using the == operator would return the same hash value also.
+    def hash
+      self.uri.hash + self.parameters.hash + self.username.hash + self.password.hash + self.verify_mode.hash + 20110106
+    end
+
+    #:nodoc:
+    def invoke_with_cache_check
+      cache_proxy.get
+    end
+
+    alias_method_chain :invoke, :cache_check
   end
 end

data/lib/wrest/native/redirection.rb

@@ -30,7 +30,7 @@ module Wrest #:nodoc:
 
         raise Wrest::Exceptions::AutoRedirectLimitExceeded if (redirect_request_options[:follow_redirects_count] += 1) > redirect_request_options[:follow_redirects_limit]
 
-        Wrest.logger.debug "--| Redirecting to #{target}"
+        Wrest.logger.debug "-| Redirecting to #{target}"
         Wrest::Uri.new(target, redirect_request_options).get
       end
     end

data/lib/wrest/native/request.rb

@@ -13,7 +13,7 @@ module Wrest::Native
   # or Wrest::Native::Get etc. instead.
   class Request
     attr_reader :http_request, :uri, :body, :headers, :username, :password, :follow_redirects,
-                :follow_redirects_limit, :follow_redirects_count, :timeout, :connection, :parameters, :cache_store
+                :follow_redirects_limit, :follow_redirects_count, :timeout, :connection, :parameters, :cache_store, :verify_mode, :options
     # Valid tuples for the options are:
     #   :username => String, defaults to nil
     #   :password => String, defaults to nil
@@ -32,12 +32,12 @@ module Wrest::Native
     #   :connection => The HTTP Connection object to use. This is how a keep-alive connection can be
     #                  used for multiple requests.
     #   :verify_mode => The  verification mode to be used for Net::HTTP https connections. Defaults to OpenSSL::SSL::VERIFY_PEER
-    #   :cache_store => The object which should be used as cache store for cacheable responses (caching is not supported in this version)
+    #   :cache_store => The object which should be used as cache store for cacheable responses. If not supplied, caching will be disabled.
     #   :detailed_http_logging => nil/$stdout/$stderr or File/Logger/IO object. Defaults to nil (recommended).
     #   :callback => A Hash whose keys are the response codes (or Range of response codes),
     #                        and the values are the callback functions to be executed.
     #                        eg: { <response code> => lambda { |response| some_operation } }
-    #   
+    #
     # *WARNING* : detailed_http_logging causes a serious security hole. Never use it in production code.
     #
     def initialize(wrest_uri, http_request_klass, parameters = {}, body = nil, headers = {}, options = {})
@@ -66,8 +66,8 @@ module Wrest::Native
     # Data about the request is and logged to Wrest.logger
     # The log entry contains the following information:
     #
-    #   --> indicates a request
-    #   <-- indicates a response
+    #   <- indicates a request
+    #   -> indicates a response
     #
     # The type of request is mentioned in caps, followed by a hash 
     # uniquely identifying a particular request/response pair.
@@ -85,7 +85,7 @@ module Wrest::Native
       @connection.set_debug_output @detailed_http_logging
       http_request.basic_auth username, password unless username.nil? || password.nil?
 
-      prefix = "#{http_request.method} #{http_request.hash} #{@connection.hash}"
+      prefix = "#{http_request.method} #{self.hash} #{@connection.hash}"
       
       Wrest.logger.debug "<- (#{prefix}) #{@uri.protocol}://#{@uri.host}:#{@uri.port}#{@http_request.path}"
       time = Benchmark.realtime { response = Wrest::Native::Response.new( do_request ) }

data/lib/wrest/native/response.rb

@@ -27,24 +27,50 @@ module Wrest #:nodoc:
       include HttpCodes
       
       extend Forwardable
-      def_delegators  :@http_response,  :code, :message, :body, :Http_version,
-              :[], :content_length, :content_type, :each_header, :each_name, :each_value, :fetch,
-              :get_fields, :key?, :type_params
+      def_delegators  :@http_response,  :code, :message, :body, :http_version,
+              :content_length, :content_type
 
-      # We're overriding :new to act as a factory so 
+      def_delegators :headers, :[]
+
+      # TODO : Are these needed in the Response namespace itself? Can be accessed from the headers method.
+      def_delegators :@http_response, :each_header, :each_name, :each_value, :fetch,
+                     :get_fields, :key?, :type_params
+
+      # We're overriding :new to act as a factory so
       # we can build the appropriate Response instance based
-      # on th response code.
+      # on the response code.
       def self.new(http_response)
         code = http_response.code.to_i
         instance = ((300..303).include?(code) || (305..399).include?(code)  ? Wrest::Native::Redirection : self).allocate
         instance.send :initialize, http_response
         instance
       end
-              
+
       def initialize(http_response)
         @http_response = http_response
       end
 
+      def initialize_copy(source)
+        @headers = source.headers.clone
+      end
+
+      # Checks equality between two Wrest::Native::Response objects.
+      def ==(other)
+        return true if self.equal?(other)
+        return false unless other.class == self.class
+        return true if self.code == other.code and
+            self.headers == other.headers and
+            self.http_version == other.http_version and
+            self.message == other.message and
+            self.body == other.body
+        false
+      end
+
+      # Return the hash of a Wrest::Native::Response object.
+      def hash
+        self.code.hash + self.message.hash + self.headers.hash + self.http_version.hash + self.body.hash
+      end
+
 
       def deserialise(options = {})
         deserialise_using(Wrest::Components::Translators.lookup(@http_response.content_type),options)
@@ -54,13 +80,22 @@ module Wrest #:nodoc:
         translator.deserialise(@http_response,options)
       end
 
+      # Gives a hash of the response headers. The keys of the hash are case-insensitive.
       def headers
-        @http_response.to_hash
+        return @headers if @headers
+
+        nethttp_headers_with_string_values=@http_response.to_hash.inject({}) {|new_headers, (old_key, old_value)|
+          new_headers[old_key] = old_value.is_a?(Array) ? old_value.join(",") : old_value
+          new_headers
+          }
+        
+        @headers=Wrest::HashWithCaseInsensitiveAccess.new(nethttp_headers_with_string_values)
+
       end
-      
+
       # A null object implementation - invoking this method on
       # a response simply returns the same response unless
-      # the response is Redirection (code 3xx), in which case a 
+      # the response is Redirection (code 3xx), in which case a
       # get is invoked on the url stored in the response headers
       # under the key 'location' and the new Response is returned.
       def follow(redirect_request_options = {})
@@ -71,12 +106,29 @@ module Wrest #:nodoc:
         self[Native::StandardHeaders::Connection].downcase == Native::StandardTokens::Close.downcase
       end
 
+      # Returns whether this response is cacheable. 
       def cacheable?
-        code_cacheable? && no_cache_flag_not_set? && no_store_flag_not_set? && expires_header_not_in_past?
+        code_cacheable? && no_cache_flag_not_set? && no_store_flag_not_set? &&
+            (not max_age.nil? or (expires_not_in_our_past? && expires_not_in_its_past?)) && pragma_nocache_not_set? &&
+            vary_tag_not_set?
       end
 
+      #:nodoc:
       def code_cacheable?
-        !code.nil? && !/2\d{2}/.match(code).nil?
+        !code.nil? && ([200, 203, 300, 301, 302, 304, 307].include?(code.to_i))
+      end
+
+      #:nodoc:
+      def max_age
+        return @max_age if @max_age
+
+        max_age  =cache_control_headers.grep(/max-age/)
+
+        @max_age = unless max_age.empty?
+                     max_age.first.split('=').last.to_i
+                   else
+                     nil
+                   end
       end
 
       def no_cache_flag_not_set?
@@ -87,40 +139,122 @@ module Wrest #:nodoc:
         not cache_control_headers.include?('no-store')
       end
 
-      def expires_header_not_in_past?
-        expires_header = cache_control_headers.find{ |h| h.include? 'Expires' }
-        if expires_header.nil?
-          true
+      def pragma_nocache_not_set?
+        headers['pragma'].nil? || (not headers['pragma'].include? 'no-cache')
+      end
+
+      #:nodoc:
+      def vary_tag_not_set?
+        headers['vary'].nil?
+      end
+
+      # Returns the Date from the response headers.
+      def response_date
+        return @response_date if @response_date
+        @response_date = parse_datefield(headers, "date")
+      end
+
+
+      # Returns the Expires date from the response headers.
+      def expires
+        return @expires if @expires
+        @expires = parse_datefield(headers, "expires")
+      end
+
+      # Returns whether the Expires header of this response is earlier than current time.    
+      def expires_not_in_our_past?
+        if expires.nil?
+          false
         else
-          expires_on = DateTime.parse(expires_header.split("=")[1])
-          expires_on > DateTime.now
+          expires.to_i > Time.now.to_i
         end
       end
 
+      # Is the Expires of this response earlier than its Date header.
+      def expires_not_in_its_past?
+        # Invalid header value for Date or Expires means the response is not cacheable
+        if  expires.nil? || response_date.nil?
+          false
+        else
+           expires > response_date
+        end
+      end
+
+      # Age of the response calculated according to RFC 2616 13.2.3
+      def current_age
+        current_time = Time.now.to_i
+
+        # RFC 2616 13.2.3 Age Calculations. TODO: include response_delay in the calculation as defined in RFC. For this, include original Request with Response.
+        date_value             = DateTime.parse(headers['date']).to_i rescue current_time
+        age_value              = headers['age'].to_i || 0
+
+        apparent_age           = current_time - date_value
+
+        [apparent_age, age_value].max
+      end
+
+      # The values in Cache-Control header as an array.
       def cache_control_headers
-        @cache_control_headers unless @cache_control_headers.nil?
-        if headers['Cache-Control'].nil? then
-          @cache_control_headers = []
-        else 
-          cache_headers = headers['Cache-Control'].split(",")
-          @cache_control_headers = correct_expires_headers(cache_headers)
-          @cache_control_headers.collect
+        @cache_control_headers ||= recalculate_cache_control_headers
+      end
+
+      #:nodoc:
+      def recalculate_cache_control_headers
+        headers['cache-control'].split(",").collect {|cc| cc.strip } rescue []
+      end
+
+      # How long (in seconds) is this response expected to be fresh
+      def freshness_lifetime
+        @freshness_lifetime ||= recalculate_freshness_lifetime
+      end
+
+      #:nodoc:
+      def recalculate_freshness_lifetime
+        return max_age if max_age
+
+        response_date = DateTime.parse(headers['date']).to_i
+        expires_date  = DateTime.parse(headers['expires']).to_i
+
+        return (expires_date - response_date)
+      end
+
+      # Has this response expired? The expiry is calculated from the Max-Age/Expires header.
+      def expired?
+        freshness=freshness_lifetime
+        if freshness <= 0
+          return true
         end
+
+        freshness <= current_age
       end
 
-      :private
+      def last_modified
+        headers['last-modified']
+      end
+
+      # Can this response be validated by sending a validation request to the server. The response need to have either
+      # Last-Modified or ETag header (or both) for it to be validatable.
+      def can_be_validated?
+        not (last_modified.nil? and headers['etag'].nil?)
+      end
 
-      def correct_expires_headers(cache_headers)
-        # The expires header "Expires = Sun, 06 Nov 1994 08:49:37 GMT" would have split into two ['Expires = Sun',' 06 Nov 1994 08:49:37 GMT']
-        expires_index = cache_headers.find_index(){ |a| a.include? 'Expires' }
-        if expires_index
-          expires_part_1 = cache_headers.delete(cache_headers[expires_index])
-          # earlier delete shifted the second part on same index
-          expires_part_2 = cache_headers.delete(cache_headers[expires_index])
-          cache_headers.push(expires_part_1+','+expires_part_2)
+
+      #:nodoc:
+      # helper function. Used to parse date fields.
+      # this function is used and tested by the expires and response_date methods
+      def parse_datefield(hash, key)
+        if hash[key]
+          # Can't trust external input. Do not crash even if invalid dates are passed.
+          begin
+            DateTime.parse(hash[key].to_s)
+          rescue ArgumentError
+            nil
+          end
+        else
+          nil
         end
-        cache_headers
       end
+
     end
   end
 end