sprockets 2.12.5 → 3.0.0.beta.1

data/lib/sprockets/bower.rb

@@ -0,0 +1,56 @@
+require 'json'
+
+module Sprockets
+  module Bower
+    # Internal: All supported bower.json files.
+    #
+    # https://github.com/bower/json/blob/0.4.0/lib/json.js#L7
+    POSSIBLE_BOWER_JSONS = ['bower.json', 'component.json', '.bower.json']
+
+    # Internal: Override resolve_alternates to install bower.json behavior.
+    #
+    # load_path    - String environment path
+    # logical_path - String path relative to base
+    #
+    # Returns nothing.
+    def resolve_alternates(load_path, logical_path, &block)
+      super
+
+      # bower.json can only be nested one level deep
+      if !logical_path.index('/')
+        dirname = File.join(load_path, logical_path)
+        stat    = self.stat(dirname)
+
+        if stat && stat.directory?
+          filenames = POSSIBLE_BOWER_JSONS.map { |basename| File.join(dirname, basename) }
+          filename  = filenames.detect { |fn| self.file?(fn) }
+
+          if filename
+            read_bower_main(dirname, filename, &block)
+          end
+        end
+      end
+
+      nil
+    end
+
+    # Internal: Read bower.json's main directive.
+    #
+    # dirname  - String path to component directory.
+    # filename - String path to bower.json.
+    #
+    # Returns nothing.
+    def read_bower_main(dirname, filename)
+      bower = JSON.parse(File.read(filename), create_additions: false)
+
+      case bower['main']
+      when String
+        yield File.expand_path(bower['main'], dirname)
+      when Array
+        bower['main'].each do |name|
+          yield File.expand_path(name, dirname)
+        end
+      end
+    end
+  end
+end

data/lib/sprockets/bundle.rb

@@ -0,0 +1,32 @@
+module Sprockets
+  # Internal: Bundle processor takes a single file asset and prepends all the
+  # `:required` URIs to the contents.
+  #
+  # Uses pipeline metadata:
+  #
+  #   :required - Ordered Set of asset URIs to prepend
+  #   :stubbed  - Set of asset URIs to substract from the required set.
+  #
+  # Also see DirectiveProcessor.
+  class Bundle
+    def self.call(input)
+      env  = input[:environment]
+      type = input[:content_type]
+
+      # TODO: Rebuilding this URI is a bit of a smell
+      processed_uri = AssetURI.build(input[:filename], type: type, skip_bundle: true)
+
+      cache = Hash.new do |h, uri|
+        h[uri] = env.find_asset_by_uri(uri)
+      end
+
+      find_required = proc { |uri| cache[uri].metadata[:required] }
+      required = Utils.dfs(processed_uri, &find_required)
+      stubbed  = Utils.dfs(cache[processed_uri].metadata[:stubbed], &find_required)
+      required.subtract(stubbed)
+      assets = required.map { |uri| cache[uri] }
+
+      env.process_bundle_reducers(assets, env.unwrap_bundle_reducers(type)).merge(included: assets.map(&:uri))
+    end
+  end
+end

data/lib/sprockets/cache.rb

@@ -0,0 +1,220 @@
+require 'sprockets/utils'
+require 'logger'
+
+module Sprockets
+  # Public: Wrapper interface to backend cache stores. Ensures a consistent API
+  # even when the backend uses get/set or read/write.
+  #
+  # Public cache interface
+  #
+  # Always assign the backend store instance to Environment#cache=.
+  #
+  #     environment.cache = Sprockets::Cache::MemoryStore.new(1000)
+  #
+  # Environment#cache will always return a wrapped Cache interface. See the
+  # methods marked public on this class.
+  #
+  #
+  # Backend cache interface
+  #
+  # The Backend cache store must implement two methods.
+  #
+  # get(key)
+  #
+  #   key - An opaque String with a length less than 250 characters.
+  #
+  #   Returns an JSON serializable object.
+  #
+  # set(key, value)
+  #
+  #   Will only be called once per key. Setting a key "foo" with value "bar",
+  #   then later key "foo" with value "baz" is an undefined behavior.
+  #
+  #   key   - An opaque String with a length less than 250 characters.
+  #   value - A JSON serializable object.
+  #
+  #   Returns argument value.
+  #
+  class Cache
+    # Builtin cache stores.
+    autoload :FileStore,   'sprockets/cache/file_store'
+    autoload :MemoryStore, 'sprockets/cache/memory_store'
+    autoload :NullStore,   'sprockets/cache/null_store'
+
+    # Internal: Cache key version for this class. Rarely should have to change
+    # unless the cache format radically changes. Will be bump on major version
+    # releases though.
+    VERSION = '3.0'
+
+    def self.default_logger
+      logger = Logger.new($stderr)
+      logger.level = Logger::FATAL
+      logger
+    end
+
+    # Internal: Wrap a backend cache store.
+    #
+    # Always assign a backend cache store instance to Environment#cache= and
+    # use Environment#cache to retreive a wrapped interface.
+    #
+    # cache - A compatible backend cache store instance.
+    def initialize(cache = nil, logger = self.class.default_logger)
+      @cache_wrapper = get_cache_wrapper(cache)
+      @fetch_cache   = Cache::MemoryStore.new(1024)
+      @logger        = logger
+    end
+
+    # Public: Prefer API to retrieve and set values in the cache store.
+    #
+    # key   - JSON serializable key
+    # block -
+    #   Must return a consistent JSON serializable object for the given key.
+    #
+    # Examples
+    #
+    #   cache.fetch("foo") { "bar" }
+    #
+    # Returns a JSON serializable object.
+    def fetch(key)
+      start = Utils.benchmark_start
+      expanded_key = expand_key(key)
+      value = @fetch_cache.get(expanded_key)
+      if value.nil?
+        value = @cache_wrapper.get(expanded_key)
+        if value.nil?
+          value = yield
+          @cache_wrapper.set(expanded_key, value)
+          @logger.debug do
+            ms = "(#{Utils.benchmark_end(start)}ms)"
+            "Sprockets Cache miss #{peek_key(key)}  #{ms}"
+          end
+        end
+        @fetch_cache.set(expanded_key, value)
+      end
+      value
+    end
+
+    # Public: Low level API to retrieve item directly from the backend cache
+    # store.
+    #
+    # This API may be used publicaly, but may have undefined behavior
+    # depending on the backend store being used. Therefore it must be used
+    # with caution, which is why its prefixed with an underscore. Prefer the
+    # Cache#fetch API over using this.
+    #
+    # key   - JSON serializable key
+    # value - A consistent JSON serializable object for the given key. Setting
+    #         a different value for the given key has undefined behavior.
+    #
+    # Returns a JSON serializable object or nil if there was a cache miss.
+    def _get(key)
+      @cache_wrapper.get(expand_key(key))
+    end
+
+    # Public: Low level API to set item directly to the backend cache store.
+    #
+    # This API may be used publicaly, but may have undefined behavior
+    # depending on the backend store being used. Therefore it must be used
+    # with caution, which is why its prefixed with an underscore. Prefer the
+    # Cache#fetch API over using this.
+    #
+    # key - JSON serializable key
+    #
+    # Returns the value argument.
+    def _set(key, value)
+      @cache_wrapper.set(expand_key(key), value)
+    end
+
+    # Public: Pretty inspect
+    #
+    # Returns String.
+    def inspect
+      "#<#{self.class} local=#{@fetch_cache.inspect} store=#{@cache_wrapper.cache.inspect}>"
+    end
+
+    private
+      # Internal: Expand object cache key into a short String key.
+      #
+      # The String should be under 250 characters so its compatible with
+      # Memcache.
+      #
+      # key - JSON serializable key
+      #
+      # Returns a String with a length less than 250 characters.
+      def expand_key(key)
+        "sprockets/v#{VERSION}/#{Utils.hexdigest(key)}"
+      end
+
+      PEEK_SIZE = 100
+
+      # Internal: Show first 100 characters of cache key for logging purposes.
+      #
+      # Returns a String with a length less than 100 characters.
+      def peek_key(key)
+        if key.is_a?(String)
+          key[0, PEEK_SIZE].inspect
+        elsif key.is_a?(Array)
+          str = []
+          key.each { |k| str << peek_key(k) }
+          str.join(':')[0, PEEK_SIZE]
+        else
+          peek_key(Utils.hexdigest(key))
+        end
+      end
+
+      def get_cache_wrapper(cache)
+        if cache.is_a?(Cache)
+          cache
+
+        # `Cache#get(key)` for Memcache
+        elsif cache.respond_to?(:get)
+          GetWrapper.new(cache)
+
+        # `Cache#[key]` so `Hash` can be used
+        elsif cache.respond_to?(:[])
+          HashWrapper.new(cache)
+
+        # `Cache#read(key)` for `ActiveSupport::Cache` support
+        elsif cache.respond_to?(:read)
+          ReadWriteWrapper.new(cache)
+
+        else
+          cache = Sprockets::Cache::NullStore.new
+          GetWrapper.new(cache)
+        end
+      end
+
+      class Wrapper < Struct.new(:cache)
+      end
+
+      class GetWrapper < Wrapper
+        def get(key)
+          cache.get(key)
+        end
+
+        def set(key, value)
+          cache.set(key, value)
+        end
+      end
+
+      class HashWrapper < Wrapper
+        def get(key)
+          cache[key]
+        end
+
+        def set(key, value)
+          cache[key] = value
+        end
+      end
+
+      class ReadWriteWrapper < Wrapper
+        def get(key)
+          cache.read(key)
+        end
+
+        def set(key, value)
+          cache.write(key, value)
+        end
+      end
+  end
+end

data/lib/sprockets/cache/file_store.rb

@@ -1,32 +1,164 @@
 require 'digest/md5'
 require 'fileutils'
-require 'pathname'
+require 'logger'
 
 module Sprockets
-  module Cache
-    # A simple file system cache store.
+  class Cache
+    # Public: A file system cache store that automatically cleans up old keys.
+    #
+    # Assign the instance to the Environment#cache.
     #
     #     environment.cache = Sprockets::Cache::FileStore.new("/tmp")
     #
+    # See Also
+    #
+    #   ActiveSupport::Cache::FileStore
+    #
     class FileStore
-      def initialize(root)
-        @root = Pathname.new(root)
+      # Internal: Default key limit for store.
+      DEFAULT_MAX_SIZE = 25 * 1024 * 1024
+
+      # Internal: Default standard error fatal logger.
+      #
+      # Returns a Logger.
+      def self.default_logger
+        logger = Logger.new($stderr)
+        logger.level = Logger::FATAL
+        logger
+      end
+
+      # Public: Initialize the cache store.
+      #
+      # root     - A String path to a directory to persist cached values to.
+      # max_size - A Integer of the maximum number of keys the store will hold.
+      #            (default: 1000).
+      def initialize(root, max_size = DEFAULT_MAX_SIZE, logger = self.class.default_logger)
+        @root     = root
+        @size     = find_caches.inject(0) { |n, (_, stat)| n + stat.size }
+        @max_size = max_size
+        @gc_size  = max_size * 0.75
+        @logger   = logger
       end
 
-      # Lookup value in cache
-      def [](key)
-        pathname = @root.join(key)
-        pathname.exist? ? pathname.open('rb') { |f| Marshal.load(f) } : nil
+      # Public: Retrieve value from cache.
+      #
+      # This API should not be used directly, but via the Cache wrapper API.
+      #
+      # key - String cache key.
+      #
+      # Returns Object or nil or the value is not set.
+      def get(key)
+        path = File.join(@root, "#{key}.cache")
+
+        value = safe_open(path) do |f|
+          begin
+            Marshal.load(f)
+          rescue Exception => e
+            @logger.error do
+              "#{self.class}[#{path}] could not be unmarshaled: " +
+                "#{e.class}: #{e.message}"
+            end
+            nil
+          end
+        end
+
+        if value
+          FileUtils.touch(path)
+          value
+        end
       end
 
-      # Save value to cache
-      def []=(key, value)
+      # Public: Set a key and value in the cache.
+      #
+      # This API should not be used directly, but via the Cache wrapper API.
+      #
+      # key   - String cache key.
+      # value - Object value.
+      #
+      # Returns Object value.
+      def set(key, value)
+        path = File.join(@root, "#{key}.cache")
+
         # Ensure directory exists
-        FileUtils.mkdir_p @root.join(key).dirname
+        FileUtils.mkdir_p File.dirname(path)
+
+        # Check if cache exists before writing
+        exists = File.exist?(path)
+
+        # Write data
+        PathUtils.atomic_write(path) do |f|
+          Marshal.dump(value, f)
+          @size += f.size unless exists
+        end
+
+        # GC if necessary
+        gc! if @size > @max_size
 
-        @root.join(key).open('w') { |f| Marshal.dump(value, f)}
         value
       end
+
+      # Public: Pretty inspect
+      #
+      # Returns String.
+      def inspect
+        "#<#{self.class} size=#{@size}/#{@max_size}>"
+      end
+
+      private
+        # Internal: Get all cache files along with stats.
+        #
+        # Returns an Array of [String filename, File::Stat] pairs sorted by
+        # mtime.
+        def find_caches
+          Dir.glob(File.join(@root, '**/*.cache')).reduce([]) { |stats, filename|
+            stat = safe_stat(filename)
+            # stat maybe nil if file was removed between the time we called
+            # dir.glob and the next stat
+            stats << [filename, stat] if stat
+            stats
+          }.sort_by { |_, stat| stat.mtime.to_i }
+        end
+
+        def compute_size(caches)
+          caches.inject(0) { |sum, (_, stat)| sum + stat.size }
+        end
+
+        def safe_stat(fn)
+          File.stat(fn)
+        rescue Errno::ENOENT
+          nil
+        end
+
+        def safe_open(path, &block)
+          if File.exist?(path)
+            File.open(path, 'rb', &block)
+          end
+        rescue Errno::ENOENT
+        end
+
+        def gc!
+          start_time = Time.now
+
+          caches = find_caches
+          size = compute_size(caches)
+
+          delete_caches, keep_caches = caches.partition { |filename, stat|
+            deleted = size > @gc_size
+            size -= stat.size
+            deleted
+          }
+
+          return if delete_caches.empty?
+
+          FileUtils.remove(delete_caches.map(&:first), force: true)
+          @size = compute_size(keep_caches)
+
+          @logger.warn do
+            secs = Time.now.to_f - start_time.to_f
+            "#{self.class}[#{@root}] garbage collected " +
+              "#{delete_caches.size} files (#{(secs * 1000).to_i}ms)"
+          end
+        end
     end
   end
 end