activesupport 3.0.pre → 3.0.0.rc

data/lib/active_support/cache/compressed_mem_cache_store.rb

@@ -1,19 +1,12 @@
 module ActiveSupport
   module Cache
     class CompressedMemCacheStore < MemCacheStore
-      def read(name, options = nil)
-        if value = super(name, (options || {}).merge(:raw => true))
-          if raw?(options)
-            value
-          else
-            Marshal.load(ActiveSupport::Gzip.decompress(value))
-          end
-        end
-      end
-
-      def write(name, value, options = nil)
-        value = ActiveSupport::Gzip.compress(Marshal.dump(value)) unless raw?(options)
-        super(name, value, (options || {}).merge(:raw => true))
+      def initialize(*args)
+        ActiveSupport::Deprecation.warn('ActiveSupport::Cache::CompressedMemCacheStore has been deprecated in favor of ActiveSupport::Cache::MemCacheStore(:compress => true).', caller)
+        addresses = args.dup
+        options = addresses.extract_options!
+        args = addresses + [options.merge(:compress => true)]
+        super(*args)
       end
     end
   end

data/lib/active_support/cache/file_store.rb

@@ -1,75 +1,174 @@
 require 'active_support/core_ext/file/atomic'
+require 'active_support/core_ext/string/conversions'
 
 module ActiveSupport
   module Cache
     # A cache store implementation which stores everything on the filesystem.
+    #
+    # FileStore implements the Strategy::LocalCache strategy which implements
+    # an in memory cache inside of a block.
     class FileStore < Store
       attr_reader :cache_path
 
-      def initialize(cache_path)
+      DIR_FORMATTER = "%03X"
+      ESCAPE_FILENAME_CHARS = /[^a-z0-9_.-]/i
+      UNESCAPE_FILENAME_CHARS = /%[0-9A-F]{2}/
+
+      def initialize(cache_path, options = nil)
+        super(options)
         @cache_path = cache_path
+        extend Strategy::LocalCache
       end
 
-      # Reads a value from the cache.
-      #
-      # Possible options:
-      # - +:expires_in+ - the number of seconds that this value may stay in
-      #   the cache.
-      def read(name, options = nil)
-        super do
-          file_name = real_file_path(name)
-          expires = expires_in(options)
-
-          if File.exist?(file_name) && (expires <= 0 || Time.now - File.mtime(file_name) < expires)
-            File.open(file_name, 'rb') { |f| Marshal.load(f) }
-          end
+      def clear(options = nil)
+        root_dirs = Dir.entries(cache_path).reject{|f| ['.', '..'].include?(f)}
+        FileUtils.rm_r(root_dirs.collect{|f| File.join(cache_path, f)})
+      end
+
+      def cleanup(options = nil)
+        options = merged_options(options)
+        each_key(options) do |key|
+          entry = read_entry(key, options)
+          delete_entry(key, options) if entry && entry.expired?
         end
       end
 
-      # Writes a value to the cache.
-      def write(name, value, options = nil)
-        super do
-          ensure_cache_path(File.dirname(real_file_path(name)))
-          File.atomic_write(real_file_path(name), cache_path) { |f| Marshal.dump(value, f) }
-          value
+      def increment(name, amount = 1, options = nil)
+        file_name = key_file_path(namespaced_key(name, options))
+        lock_file(file_name) do
+          options = merged_options(options)
+          if num = read(name, options)
+            num = num.to_i + amount
+            write(name, num, options)
+            num
+          else
+            nil
+          end
         end
-      rescue => e
-        logger.error "Couldn't create cache directory: #{name} (#{e.message})" if logger
       end
 
-      def delete(name, options = nil)
-        super do
-          File.delete(real_file_path(name))
+      def decrement(name, amount = 1, options = nil)
+        file_name = key_file_path(namespaced_key(name, options))
+        lock_file(file_name) do
+          options = merged_options(options)
+          if num = read(name, options)
+            num = num.to_i - amount
+            write(name, num, options)
+            num
+          else
+            nil
+          end
         end
-      rescue SystemCallError => e
-        # If there's no cache, then there's nothing to complain about
       end
 
       def delete_matched(matcher, options = nil)
-        super do
-          search_dir(@cache_path) do |f|
-            if f =~ matcher
-              begin
-                File.delete(f)
-              rescue SystemCallError => e
-                # If there's no cache, then there's nothing to complain about
+        options = merged_options(options)
+        instrument(:delete_matched, matcher.inspect) do
+          matcher = key_matcher(matcher, options)
+          search_dir(cache_path) do |path|
+            key = file_path_key(path)
+            delete_entry(key, options) if key.match(matcher)
+          end
+        end
+      end
+
+      protected
+
+        def read_entry(key, options)
+          file_name = key_file_path(key)
+          if File.exist?(file_name)
+            entry = File.open(file_name) { |f| Marshal.load(f) }
+            if entry && !entry.expired? && !entry.expires_in && !self.options[:expires_in]
+              # Check for deprecated use of +:expires_in+ option from versions < 3.0
+              deprecated_expires_in = options[:expires_in]
+              if deprecated_expires_in
+                ActiveSupport::Deprecation.warn('Setting :expires_in on read has been deprecated in favor of setting it on write.', caller)
+                if entry.created_at + deprecated_expires_in.to_f <= Time.now.to_f
+                  delete_entry(key, options)
+                  entry = nil
+                end
               end
             end
+            entry
           end
+        rescue
+          nil
         end
-      end
 
-      def exist?(name, options = nil)
-        super do
-          File.exist?(real_file_path(name))
+        def write_entry(key, entry, options)
+          file_name = key_file_path(key)
+          ensure_cache_path(File.dirname(file_name))
+          File.atomic_write(file_name, cache_path) {|f| Marshal.dump(entry, f)}
+          true
+        end
+
+        def delete_entry(key, options)
+          file_name = key_file_path(key)
+          if File.exist?(file_name)
+            begin
+              File.delete(file_name)
+              delete_empty_directories(File.dirname(file_name))
+              true
+            rescue => e
+              # Just in case the error was caused by another process deleting the file first.
+              raise e if File.exist?(file_name)
+              false
+            end
+          end
         end
-      end
 
       private
-        def real_file_path(name)
-          '%s/%s.cache' % [@cache_path, name.gsub('?', '.').gsub(':', '.')]
+        # Lock a file for a block so only one process can modify it at a time.
+        def lock_file(file_name, &block) # :nodoc:
+          if File.exist?(file_name)
+            File.open(file_name, 'r') do |f|
+              begin
+                f.flock File::LOCK_EX
+                yield
+              ensure
+                f.flock File::LOCK_UN
+              end
+            end
+          else
+            yield
+          end
+        end
+
+        # Translate a key into a file path.
+        def key_file_path(key)
+          fname = key.to_s.gsub(ESCAPE_FILENAME_CHARS){|match| "%#{match.ord.to_s(16).upcase}"}
+          hash = Zlib.adler32(fname)
+          hash, dir_1 = hash.divmod(0x1000)
+          dir_2 = hash.modulo(0x1000)
+          fname_paths = []
+          # Make sure file name is < 255 characters so it doesn't exceed file system limits.
+          if fname.size <= 255
+            fname_paths << fname
+          else
+            while fname.size <= 255
+              fname_path << fname[0, 255]
+              fname = fname[255, -1]
+            end
+          end
+          File.join(cache_path, DIR_FORMATTER % dir_1, DIR_FORMATTER % dir_2, *fname_paths)
+        end
+
+        # Translate a file path into a key.
+        def file_path_key(path)
+          fname = path[cache_path.size, path.size].split(File::SEPARATOR, 4).last
+          fname.gsub(UNESCAPE_FILENAME_CHARS){|match| $1.ord.to_s(16)}
+        end
+
+        # Delete empty directories in the cache.
+        def delete_empty_directories(dir)
+          return if dir == cache_path
+          if Dir.entries(dir).reject{|f| ['.', '..'].include?(f)}.empty?
+            File.delete(dir) rescue nil
+            delete_empty_directories(File.dirname(dir))
+          end
         end
 
+        # Make sure a file path's directories exist.
         def ensure_cache_path(path)
           FileUtils.makedirs(path) unless File.exist?(path)
         end

data/lib/active_support/cache/mem_cache_store.rb

@@ -1,4 +1,10 @@
-require 'memcache'
+begin
+  require 'memcache'
+rescue LoadError => e
+  $stderr.puts "You don't have memcache installed in your application. Please add it to your Gemfile and run bundle install"
+  raise e
+end
+require 'digest/md5'
 
 module ActiveSupport
   module Cache
@@ -12,8 +18,9 @@ module ActiveSupport
     #   and MemCacheStore will load balance between all available servers. If a
     #   server goes down, then MemCacheStore will ignore it until it goes back
     #   online.
-    # - Time-based expiry support. See #write and the <tt>:expires_in</tt> option.
-    # - Per-request in memory cache for all communication with the MemCache server(s).
+    #
+    # MemCacheStore implements the Strategy::LocalCache strategy which implements
+    # an in memory cache inside of a block.
     class MemCacheStore < Store
       module Response # :nodoc:
         STORED      = "STORED\r\n"
@@ -23,10 +30,12 @@ module ActiveSupport
         DELETED     = "DELETED\r\n"
       end
 
+      ESCAPE_KEY_CHARS = /[\x00-\x20%\x7F-\xFF]/
+
       def self.build_mem_cache(*addresses)
         addresses = addresses.flatten
         options = addresses.extract_options!
-        addresses = ["localhost"] if addresses.empty?
+        addresses = ["localhost:11211"] if addresses.empty?
         MemCache.new(addresses, options)
       end
 
@@ -38,109 +47,145 @@ module ActiveSupport
       #
       # If no addresses are specified, then MemCacheStore will connect to
       # localhost port 11211 (the default memcached port).
+      #
+      # Instead of addresses one can pass in a MemCache-like object. For example:
+      #
+      #   require 'memcached' # gem install memcached; uses C bindings to libmemcached
+      #   ActiveSupport::Cache::MemCacheStore.new(Memcached::Rails.new("localhost:11211"))
       def initialize(*addresses)
+        addresses = addresses.flatten
+        options = addresses.extract_options!
+        super(options)
+
         if addresses.first.respond_to?(:get)
           @data = addresses.first
         else
-          @data = self.class.build_mem_cache(*addresses)
+          mem_cache_options = options.dup
+          UNIVERSAL_OPTIONS.each{|name| mem_cache_options.delete(name)}
+          @data = self.class.build_mem_cache(*(addresses + [mem_cache_options]))
         end
 
         extend Strategy::LocalCache
+        extend LocalCacheWithRaw
       end
 
-      # Reads multiple keys from the cache.
-      def read_multi(*keys)
-        @data.get_multi keys
-      end
-
-      def read(key, options = nil) # :nodoc:
-        super do
-          @data.get(key, raw?(options))
-        end
-      rescue MemCache::MemCacheError => e
-        logger.error("MemCacheError (#{e}): #{e.message}")
-        nil
-      end
-
-      # Writes a value to the cache.
-      #
-      # Possible options:
-      # - <tt>:unless_exist</tt> - set to true if you don't want to update the cache
-      #   if the key is already set.
-      # - <tt>:expires_in</tt> - the number of seconds that this value may stay in
-      #   the cache. See ActiveSupport::Cache::Store#write for an example.
-      def write(key, value, options = nil)
-        super do
-          method = options && options[:unless_exist] ? :add : :set
-          # memcache-client will break the connection if you send it an integer
-          # in raw mode, so we convert it to a string to be sure it continues working.
-          value = value.to_s if raw?(options)
-          response = @data.send(method, key, value, expires_in(options), raw?(options))
-          response == Response::STORED
-        end
-      rescue MemCache::MemCacheError => e
-        logger.error("MemCacheError (#{e}): #{e.message}")
-        false
-      end
-
-      def delete(key, options = nil) # :nodoc:
-        super do
-          response = @data.delete(key, expires_in(options))
-          response == Response::DELETED
+      # Reads multiple keys from the cache using a single call to the
+      # servers for all keys. Options can be passed in the last argument.
+      def read_multi(*names)
+        options = names.extract_options!
+        options = merged_options(options)
+        keys_to_names = names.inject({}){|map, name| map[escape_key(namespaced_key(name, options))] = name; map}
+        raw_values = @data.get_multi(keys_to_names.keys, :raw => true)
+        values = {}
+        raw_values.each do |key, value|
+          entry = deserialize_entry(value)
+          values[keys_to_names[key]] = entry.value unless entry.expired?
         end
-      rescue MemCache::MemCacheError => e
-        logger.error("MemCacheError (#{e}): #{e.message}")
-        false
+        values
       end
 
-      def exist?(key, options = nil) # :nodoc:
-        # Doesn't call super, cause exist? in memcache is in fact a read
-        # But who cares? Reading is very fast anyway
-        # Local cache is checked first, if it doesn't know then memcache itself is read from
-        super do
-          !read(key, options).nil?
+      # Increment a cached value. This method uses the memcached incr atomic
+      # operator and can only be used on values written with the :raw option.
+      # Calling it on a value not stored with :raw will initialize that value
+      # to zero.
+      def increment(name, amount = 1, options = nil) # :nodoc:
+        options = merged_options(options)
+        response = instrument(:increment, name, :amount => amount) do
+          @data.incr(escape_key(namespaced_key(name, options)), amount)
         end
-      end
-
-      def increment(key, amount = 1) # :nodoc:
-        response = instrument(:increment, key, :amount => amount) do
-          @data.incr(key, amount)
-        end
-
-        response == Response::NOT_FOUND ? nil : response
+        response == Response::NOT_FOUND ? nil : response.to_i
       rescue MemCache::MemCacheError
         nil
       end
 
-      def decrement(key, amount = 1) # :nodoc:
-        response = instrument(:decrement, key, :amount => amount) do
-          @data.decr(key, amount)
+      # Decrement a cached value. This method uses the memcached decr atomic
+      # operator and can only be used on values written with the :raw option.
+      # Calling it on a value not stored with :raw will initialize that value
+      # to zero.
+      def decrement(name, amount = 1, options = nil) # :nodoc:
+        options = merged_options(options)
+        response = instrument(:decrement, name, :amount => amount) do
+          @data.decr(escape_key(namespaced_key(name, options)), amount)
         end
-
-        response == Response::NOT_FOUND ? nil : response
+        response == Response::NOT_FOUND ? nil : response.to_i
       rescue MemCache::MemCacheError
         nil
       end
 
-      def delete_matched(matcher, options = nil) # :nodoc:
-        # don't do any local caching at present, just pass
-        # through and let the error happen
-        super
-        raise "Not supported by Memcache"
-      end
-
-      def clear
+      # Clear the entire cache on all memcached servers. This method should
+      # be used with care when using a shared cache.
+      def clear(options = nil)
         @data.flush_all
       end
 
+      # Get the statistics from the memcached servers.
       def stats
         @data.stats
       end
 
+      protected
+        # Read an entry from the cache.
+        def read_entry(key, options) # :nodoc:
+          deserialize_entry(@data.get(escape_key(key), true))
+        rescue MemCache::MemCacheError => e
+          logger.error("MemCacheError (#{e}): #{e.message}") if logger
+          nil
+        end
+
+        # Write an entry to the cache.
+        def write_entry(key, entry, options) # :nodoc:
+          method = options && options[:unless_exist] ? :add : :set
+          value = options[:raw] ? entry.value.to_s : entry
+          expires_in = options[:expires_in].to_i
+          if expires_in > 0 && !options[:raw]
+            # Set the memcache expire a few minutes in the future to support race condition ttls on read
+            expires_in += 5.minutes
+          end
+          response = @data.send(method, escape_key(key), value, expires_in, options[:raw])
+          response == Response::STORED
+        rescue MemCache::MemCacheError => e
+          logger.error("MemCacheError (#{e}): #{e.message}") if logger
+          false
+        end
+
+        # Delete an entry from the cache.
+        def delete_entry(key, options) # :nodoc:
+          response = @data.delete(escape_key(key))
+          response == Response::DELETED
+        rescue MemCache::MemCacheError => e
+          logger.error("MemCacheError (#{e}): #{e.message}") if logger
+          false
+        end
+
       private
-        def raw?(options)
-          options && options[:raw]
+        def escape_key(key)
+          key = key.to_s.gsub(ESCAPE_KEY_CHARS){|match| "%#{match.getbyte(0).to_s(16).upcase}"}
+          key = "#{key[0, 213]}:md5:#{Digest::MD5.hexdigest(key)}" if key.size > 250
+          key
+        end
+
+        def deserialize_entry(raw_value)
+          if raw_value
+            entry = Marshal.load(raw_value) rescue raw_value
+            entry.is_a?(Entry) ? entry : Entry.new(entry)
+          else
+            nil
+          end
         end
+
+      # Provide support for raw values in the local cache strategy.
+      module LocalCacheWithRaw # :nodoc:
+        protected
+          def write_entry(key, entry, options) # :nodoc:
+            retval = super
+            if options[:raw] && local_cache && retval
+              raw_entry = Entry.new(entry.value.to_s)
+              raw_entry.expires_at = entry.expires_at
+              local_cache.write_entry(key, raw_entry, options)
+            end
+            retval
+          end
+      end
     end
   end
 end