sprockets 2.2.3 → 4.0.0

data/lib/sprockets/cache/file_store.rb

@@ -1,32 +1,208 @@
-require 'digest/md5'
+# frozen_string_literal: true
 require 'fileutils'
-require 'pathname'
+require 'logger'
+require 'sprockets/encoding_utils'
+require 'sprockets/path_utils'
+require 'zlib'
 
 module Sprockets
-  module Cache
-    # A simple file system cache store.
+  class Cache
+    # Public: A file system cache store that automatically cleans up old keys.
+    #
+    # Assign the instance to the Environment#cache.
     #
     #     environment.cache = Sprockets::Cache::FileStore.new("/tmp")
     #
+    # See Also
+    #
+    #   ActiveSupport::Cache::FileStore
+    #
     class FileStore
-      def initialize(root)
-        @root = Pathname.new(root)
+      # Internal: Default key limit for store.
+      DEFAULT_MAX_SIZE = 25 * 1024 * 1024
+      EXCLUDED_DIRS = ['.', '..'].freeze
+      GITKEEP_FILES = ['.gitkeep', '.keep'].freeze
+      
+      # Internal: Default standard error fatal logger.
+      #
+      # Returns a Logger.
+      def self.default_logger
+        logger = Logger.new($stderr)
+        logger.level = Logger::FATAL
+        logger
       end
 
-      # Lookup value in cache
-      def [](key)
-        pathname = @root.join(key)
-        pathname.exist? ? pathname.open('rb') { |f| Marshal.load(f) } : nil
+      # Public: Initialize the cache store.
+      #
+      # root     - A String path to a directory to persist cached values to.
+      # max_size - A Integer of the maximum size the store will hold (in bytes).
+      #            (default: 25MB).
+      # logger   - The logger to which some info will be printed.
+      #            (default logger level is FATAL and won't output anything).
+      def initialize(root, max_size = DEFAULT_MAX_SIZE, logger = self.class.default_logger)
+        @root     = root
+        @max_size = max_size
+        @gc_size  = max_size * 0.75
+        @logger   = logger
+      end
+
+      # Public: Retrieve value from cache.
+      #
+      # This API should not be used directly, but via the Cache wrapper API.
+      #
+      # key - String cache key.
+      #
+      # Returns Object or nil or the value is not set.
+      def get(key)
+        path = File.join(@root, "#{key}.cache")
+
+        value = safe_open(path) do |f|
+          begin
+            EncodingUtils.unmarshaled_deflated(f.read, Zlib::MAX_WBITS)
+          rescue Exception => e
+            @logger.error do
+              "#{self.class}[#{path}] could not be unmarshaled: " +
+                "#{e.class}: #{e.message}"
+            end
+            nil
+          end
+        end
+
+        if value
+          FileUtils.touch(path)
+          value
+        end
       end
 
-      # Save value to cache
-      def []=(key, value)
+      # Public: Set a key and value in the cache.
+      #
+      # This API should not be used directly, but via the Cache wrapper API.
+      #
+      # key   - String cache key.
+      # value - Object value.
+      #
+      # Returns Object value.
+      def set(key, value)
+        path = File.join(@root, "#{key}.cache")
+
         # Ensure directory exists
-        FileUtils.mkdir_p @root.join(key).dirname
+        FileUtils.mkdir_p File.dirname(path)
+
+        # Check if cache exists before writing
+        exists = File.exist?(path)
+
+        # Serialize value
+        marshaled = Marshal.dump(value)
+
+        # Compress if larger than 4KB
+        if marshaled.bytesize > 4 * 1024
+          deflater = Zlib::Deflate.new(
+            Zlib::BEST_COMPRESSION,
+            Zlib::MAX_WBITS,
+            Zlib::MAX_MEM_LEVEL,
+            Zlib::DEFAULT_STRATEGY
+          )
+          deflater << marshaled
+          raw = deflater.finish
+        else
+          raw = marshaled
+        end
+
+        # Write data
+        PathUtils.atomic_write(path) do |f|
+          f.write(raw)
+          @size = size + f.size unless exists
+        end
+
+        # GC if necessary
+        gc! if size > @max_size
 
-        @root.join(key).open('w') { |f| Marshal.dump(value, f)}
         value
       end
+
+      # Public: Pretty inspect
+      #
+      # Returns String.
+      def inspect
+        "#<#{self.class} size=#{size}/#{@max_size}>"
+      end
+
+      # Public: Clear the cache
+      #
+      # adapted from ActiveSupport::Cache::FileStore#clear
+      #
+      # Deletes all items from the cache. In this case it deletes all the entries in the specified
+      # file store directory except for .keep or .gitkeep. Be careful which directory is specified
+      # as @root because everything in that directory will be deleted.
+      #
+      # Returns true
+      def clear(options=nil)
+        if File.exist?(@root)
+          root_dirs = Dir.entries(@root).reject { |f| (EXCLUDED_DIRS + GITKEEP_FILES).include?(f) }
+          FileUtils.rm_r(root_dirs.collect{ |f| File.join(@root, f) })
+        end
+        true
+      end
+
+      private
+        # Internal: Get all cache files along with stats.
+        #
+        # Returns an Array of [String filename, File::Stat] pairs sorted by
+        # mtime.
+        def find_caches
+          Dir.glob(File.join(@root, '**/*.cache')).reduce([]) { |stats, filename|
+            stat = safe_stat(filename)
+            # stat maybe nil if file was removed between the time we called
+            # dir.glob and the next stat
+            stats << [filename, stat] if stat
+            stats
+          }.sort_by { |_, stat| stat.mtime.to_i }
+        end
+
+        def size
+          @size ||= compute_size(find_caches)
+        end
+
+        def compute_size(caches)
+          caches.inject(0) { |sum, (_, stat)| sum + stat.size }
+        end
+
+        def safe_stat(fn)
+          File.stat(fn)
+        rescue Errno::ENOENT
+          nil
+        end
+
+        def safe_open(path, &block)
+          if File.exist?(path)
+            File.open(path, 'rb', &block)
+          end
+        rescue Errno::ENOENT
+        end
+
+        def gc!
+          start_time = Time.now
+
+          caches = find_caches
+          size = compute_size(caches)
+
+          delete_caches, keep_caches = caches.partition { |filename, stat|
+            deleted = size > @gc_size
+            size -= stat.size
+            deleted
+          }
+
+          return if delete_caches.empty?
+
+          FileUtils.remove(delete_caches.map(&:first), force: true)
+          @size = compute_size(keep_caches)
+
+          @logger.warn do
+            secs = Time.now.to_f - start_time.to_f
+            "#{self.class}[#{@root}] garbage collected " +
+              "#{delete_caches.size} files (#{(secs * 1000).to_i}ms)"
+          end
+        end
     end
   end
 end

data/lib/sprockets/cache/memory_store.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+module Sprockets
+  class Cache
+    # Public: Basic in memory LRU cache.
+    #
+    # Assign the instance to the Environment#cache.
+    #
+    #     environment.cache = Sprockets::Cache::MemoryStore.new(1000)
+    #
+    # See Also
+    #
+    #   ActiveSupport::Cache::MemoryStore
+    #
+    class MemoryStore
+      # Internal: Default key limit for store.
+      DEFAULT_MAX_SIZE = 1000
+
+      # Public: Initialize the cache store.
+      #
+      # max_size - A Integer of the maximum number of keys the store will hold.
+      #            (default: 1000).
+      def initialize(max_size = DEFAULT_MAX_SIZE)
+        @max_size = max_size
+        @cache = {}
+      end
+
+      # Public: Retrieve value from cache.
+      #
+      # This API should not be used directly, but via the Cache wrapper API.
+      #
+      # key - String cache key.
+      #
+      # Returns Object or nil or the value is not set.
+      def get(key)
+        exists = true
+        value = @cache.delete(key) { exists = false }
+        if exists
+          @cache[key] = value
+        else
+          nil
+        end
+      end
+
+      # Public: Set a key and value in the cache.
+      #
+      # This API should not be used directly, but via the Cache wrapper API.
+      #
+      # key   - String cache key.
+      # value - Object value.
+      #
+      # Returns Object value.
+      def set(key, value)
+        @cache.delete(key)
+        @cache[key] = value
+        @cache.shift if @cache.size > @max_size
+        value
+      end
+
+      # Public: Pretty inspect
+      #
+      # Returns String.
+      def inspect
+        "#<#{self.class} size=#{@cache.size}/#{@max_size}>"
+      end
+
+      # Public: Clear the cache
+      #
+      # Returns true
+      def clear(options=nil)
+        @cache.clear
+        true
+      end
+    end
+  end
+end

data/lib/sprockets/cache/null_store.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+module Sprockets
+  class Cache
+    # Public: A compatible cache store that doesn't store anything. Used by
+    # default when no Environment#cache is configured.
+    #
+    # Assign the instance to the Environment#cache.
+    #
+    #     environment.cache = Sprockets::Cache::NullStore.new
+    #
+    # See Also
+    #
+    #   ActiveSupport::Cache::NullStore
+    #
+    class NullStore
+      # Public: Simulate a cache miss.
+      #
+      # This API should not be used directly, but via the Cache wrapper API.
+      #
+      # key - String cache key.
+      #
+      # Returns nil.
+      def get(key)
+        nil
+      end
+
+      # Public: Simulate setting a value in the cache.
+      #
+      # This API should not be used directly, but via the Cache wrapper API.
+      #
+      # key   - String cache key.
+      # value - Object value.
+      #
+      # Returns Object value.
+      def set(key, value)
+        value
+      end
+
+      # Public: Pretty inspect
+      #
+      # Returns String.
+      def inspect
+        "#<#{self.class}>"
+      end
+
+      # Public: Simulate clearing the cache
+      #
+      # Returns true
+      def clear(options=nil)
+        true
+      end
+    end
+  end
+end

data/lib/sprockets/cache.rb

@@ -0,0 +1,271 @@
+# frozen_string_literal: true
+require 'logger'
+require 'sprockets/digest_utils'
+
+module Sprockets
+  # Public: Wrapper interface to backend cache stores. Ensures a consistent API
+  # even when the backend uses get/set or read/write.
+  #
+  # Public cache interface
+  #
+  # Always assign the backend store instance to Environment#cache=.
+  #
+  #     environment.cache = Sprockets::Cache::MemoryStore.new(1000)
+  #
+  # Environment#cache will always return a wrapped Cache interface. See the
+  # methods marked public on this class.
+  #
+  #
+  # Backend cache interface
+  #
+  # The Backend cache store must implement two methods.
+  #
+  # get(key)
+  #
+  #   key - An opaque String with a length less than 250 characters.
+  #
+  #   Returns an JSON serializable object.
+  #
+  # set(key, value)
+  #
+  #   Will only be called once per key. Setting a key "foo" with value "bar",
+  #   then later key "foo" with value "baz" is an undefined behavior.
+  #
+  #   key   - An opaque String with a length less than 250 characters.
+  #   value - A JSON serializable object.
+  #
+  #   Returns argument value.
+  #
+  # clear(options)
+  #
+  #   Clear the entire cache. Be careful with this method since it could
+  #   affect other processes if shared cache is being used.
+  #
+  #   The options hash is passed to the underlying cache implementation.
+  class Cache
+    # Builtin cache stores.
+    autoload :FileStore,   'sprockets/cache/file_store'
+    autoload :MemoryStore, 'sprockets/cache/memory_store'
+    autoload :NullStore,   'sprockets/cache/null_store'
+
+    # Internal: Cache key version for this class. Rarely should have to change
+    # unless the cache format radically changes. Will be bump on major version
+    # releases though.
+    VERSION = '4.0.0'
+
+    def self.default_logger
+      logger = Logger.new($stderr)
+      logger.level = Logger::FATAL
+      logger
+    end
+
+    # Internal: Wrap a backend cache store.
+    #
+    # Always assign a backend cache store instance to Environment#cache= and
+    # use Environment#cache to retreive a wrapped interface.
+    #
+    # cache - A compatible backend cache store instance.
+    def initialize(cache = nil, logger = self.class.default_logger)
+      @cache_wrapper = get_cache_wrapper(cache)
+      @fetch_cache   = Cache::MemoryStore.new(1024)
+      @logger        = logger
+    end
+
+    # Public: Prefer API to retrieve and set values in the cache store.
+    #
+    # key   - JSON serializable key
+    # block -
+    #   Must return a consistent JSON serializable object for the given key.
+    #
+    # Examples
+    #
+    #   cache.fetch("foo") { "bar" }
+    #
+    # Returns a JSON serializable object.
+    def fetch(key)
+      start = Time.now.to_f
+      expanded_key = expand_key(key)
+      value = @fetch_cache.get(expanded_key)
+      if value.nil?
+        value = @cache_wrapper.get(expanded_key)
+        if value.nil?
+          value = yield
+          @cache_wrapper.set(expanded_key, value)
+          @logger.debug do
+            ms = "(#{((Time.now.to_f - start) * 1000).to_i}ms)"
+            "Sprockets Cache miss #{peek_key(key)}  #{ms}"
+          end
+        end
+        @fetch_cache.set(expanded_key, value)
+      end
+      value
+    end
+
+    # Public: Low level API to retrieve item directly from the backend cache
+    # store.
+    #
+    # This API may be used publicly, but may have undefined behavior
+    # depending on the backend store being used. Prefer the
+    # Cache#fetch API over using this.
+    #
+    # key   - JSON serializable key
+    # local - Check local cache first (default: false)
+    #
+    # Returns a JSON serializable object or nil if there was a cache miss.
+    def get(key, local = false)
+      expanded_key = expand_key(key)
+
+      if local && value = @fetch_cache.get(expanded_key)
+        return value
+      end
+
+      value = @cache_wrapper.get(expanded_key)
+      @fetch_cache.set(expanded_key, value) if local
+
+      value
+    end
+
+    # Public: Low level API to set item directly to the backend cache store.
+    #
+    # This API may be used publicly, but may have undefined behavior
+    # depending on the backend store being used. Prefer the
+    # Cache#fetch API over using this.
+    #
+    # key   - JSON serializable key
+    # value - A consistent JSON serializable object for the given key. Setting
+    #         a different value for the given key has undefined behavior.
+    # local - Set on local cache (default: false)
+    #
+    # Returns the value argument.
+    def set(key, value, local = false)
+      expanded_key = expand_key(key)
+      @fetch_cache.set(expanded_key, value) if local
+      @cache_wrapper.set(expanded_key, value)
+    end
+
+    # Public: Pretty inspect
+    #
+    # Returns String.
+    def inspect
+      "#<#{self.class} local=#{@fetch_cache.inspect} store=#{@cache_wrapper.cache.inspect}>"
+    end
+
+    # Public: Clear cache
+    #
+    # Returns truthy on success, potentially raises exception on failure
+    def clear(options=nil)
+      @cache_wrapper.clear
+      @fetch_cache.clear
+    end
+
+    private
+      # Internal: Expand object cache key into a short String key.
+      #
+      # The String should be under 250 characters so its compatible with
+      # Memcache.
+      #
+      # key - JSON serializable key
+      #
+      # Returns a String with a length less than 250 characters.
+      def expand_key(key)
+        digest_key = DigestUtils.pack_urlsafe_base64digest(DigestUtils.digest(key))
+        namespace = digest_key[0, 2]
+        "sprockets/v#{VERSION}/#{namespace}/#{digest_key}"
+      end
+
+      PEEK_SIZE = 100
+
+      # Internal: Show first 100 characters of cache key for logging purposes.
+      #
+      # Returns a String with a length less than 100 characters.
+      def peek_key(key)
+        case key
+        when Integer
+          key.to_s
+        when String
+          key[0, PEEK_SIZE].inspect
+        when Array
+          str = []
+          key.each { |k| str << peek_key(k) }
+          str.join(':')[0, PEEK_SIZE]
+        else
+          peek_key(DigestUtils.pack_urlsafe_base64digest(DigestUtils.digest(key)))
+        end
+      end
+
+      def get_cache_wrapper(cache)
+        if cache.is_a?(Cache)
+          cache
+
+        # `Cache#get(key)` for Memcache
+        elsif cache.respond_to?(:get)
+          GetWrapper.new(cache)
+
+        # `Cache#[key]` so `Hash` can be used
+        elsif cache.respond_to?(:[])
+          HashWrapper.new(cache)
+
+        # `Cache#read(key)` for `ActiveSupport::Cache` support
+        elsif cache.respond_to?(:read)
+          ReadWriteWrapper.new(cache)
+
+        else
+          cache = Sprockets::Cache::NullStore.new
+          GetWrapper.new(cache)
+        end
+      end
+
+      class Wrapper < Struct.new(:cache)
+      end
+
+      class GetWrapper < Wrapper
+        def get(key)
+          cache.get(key)
+        end
+
+        def set(key, value)
+          cache.set(key, value)
+        end
+
+        def clear(options=nil)
+          # dalli has a #flush method so try it
+          if cache.respond_to?(:flush)
+            cache.flush(options)
+          else
+            cache.clear(options)
+          end
+          true
+        end
+      end
+
+      class HashWrapper < Wrapper
+        def get(key)
+          cache[key]
+        end
+
+        def set(key, value)
+          cache[key] = value
+        end
+
+        def clear(options=nil)
+          cache.clear
+          true
+        end
+      end
+
+      class ReadWriteWrapper < Wrapper
+        def get(key)
+          cache.read(key)
+        end
+
+        def set(key, value)
+          cache.write(key, value)
+        end
+
+        def clear(options=nil)
+          cache.clear(options)
+          true
+        end
+      end
+  end
+end

data/lib/sprockets/cached_environment.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+require 'sprockets/base'
+
+module Sprockets
+  # `CachedEnvironment` is a special cached version of `Environment`.
+  #
+  # The expection is that all of its file system methods are cached
+  # for the instances lifetime. This makes `CachedEnvironment` much faster. This
+  # behavior is ideal in production environments where the file system
+  # is immutable.
+  #
+  # `CachedEnvironment` should not be initialized directly. Instead use
+  # `Environment#cached`.
+  class CachedEnvironment < Base
+    def initialize(environment)
+      initialize_configuration(environment)
+
+      @cache   = environment.cache
+      @stats   = {}
+      @entries = {}
+      @uris    = {}
+      @processor_cache_keys = {}
+      @resolved_dependencies = {}
+    end
+
+    # No-op return self as cached environment.
+    def cached
+      self
+    end
+    alias_method :index, :cached
+
+    # Internal: Cache Environment#entries
+    def entries(path)
+      @entries[path] ||= super(path)
+    end
+
+    # Internal: Cache Environment#stat
+    def stat(path)
+      @stats[path] ||= super(path)
+    end
+
+    # Internal: Cache Environment#load
+    def load(uri)
+      @uris[uri] ||= super(uri)
+    end
+
+    # Internal: Cache Environment#processor_cache_key
+    def processor_cache_key(str)
+      @processor_cache_keys[str] ||= super(str)
+    end
+
+    # Internal: Cache Environment#resolve_dependency
+    def resolve_dependency(str)
+      @resolved_dependencies[str] ||= super(str)
+    end
+
+    private
+      # Cache is immutable, any methods that try to change the runtime config
+      # should bomb.
+      def config=(config)
+        raise RuntimeError, "can't modify immutable cached environment"
+      end
+  end
+end