sprockets 3.0.1 → 3.7.5

data/lib/sprockets/legacy_tilt_processor.rb

@@ -22,8 +22,8 @@ module Sprockets
       data     = input[:data]
       context  = input[:environment].context_class.new(input)
 
-      data = @klass.new(filename) { data }.render(context)
-      context.metadata.merge(data: data)
+      data = @klass.new(filename) { data }.render(context, {})
+      context.metadata.merge(data: data.to_str)
     end
   end
 end

data/lib/sprockets/loader.rb

@@ -10,34 +10,54 @@ require 'sprockets/processor_utils'
 require 'sprockets/resolve'
 require 'sprockets/transformers'
 require 'sprockets/uri_utils'
+require 'sprockets/unloaded_asset'
 
 module Sprockets
+
   # The loader phase takes a asset URI location and returns a constructed Asset
   # object.
   module Loader
     include DigestUtils, PathUtils, ProcessorUtils, URIUtils
     include Engines, Mime, Processing, Resolve, Transformers
 
-    # Public: Load Asset by AssetURI.
+
+    # Public: Load Asset by Asset URI.
+    #
+    # uri - A String containing complete URI to a file including schema
+    #       and full path such as:
+    #       "file:///Path/app/assets/js/app.js?type=application/javascript"
     #
-    # uri - AssetURI
     #
     # Returns Asset.
     def load(uri)
-      filename, params = parse_asset_uri(uri)
-      if params.key?(:id)
-        asset = cache.fetch("asset-uri:#{VERSION}#{uri}") do
-          load_asset_by_id_uri(uri, filename, params)
+      unloaded = UnloadedAsset.new(uri, self)
+      if unloaded.params.key?(:id)
+        unless asset = asset_from_cache(unloaded.asset_key)
+          id = unloaded.params.delete(:id)
+          uri_without_id = build_asset_uri(unloaded.filename, unloaded.params)
+          asset = load_from_unloaded(UnloadedAsset.new(uri_without_id, self))
+          if asset[:id] != id
+            @logger.warn "Sprockets load error: Tried to find #{uri}, but latest was id #{asset[:id]}"
+          end
         end
       else
-        asset = fetch_asset_from_dependency_cache(uri, filename) do |paths|
+        asset = fetch_asset_from_dependency_cache(unloaded) do |paths|
+          # When asset is previously generated, its "dependencies" are stored in the cache.
+          # The presence of `paths` indicates dependencies were stored.
+          # We can check to see if the dependencies have not changed by "resolving" them and
+          # generating a digest key from the resolved entries. If this digest key has not
+          # changed the asset will be pulled from cache.
+          #
+          # If this `paths` is present but the cache returns nothing then `fetch_asset_from_dependency_cache`
+          # will confusingly be called again with `paths` set to nil where the asset will be
+          # loaded from disk.
           if paths
-            digest = digest(resolve_dependencies(paths))
-            if id_uri = cache.get("asset-uri-digest:#{VERSION}:#{uri}:#{digest}", true)
-              cache.get("asset-uri:#{VERSION}:#{id_uri}", true)
+            digest = DigestUtils.digest(resolve_dependencies(paths))
+            if uri_from_cache = cache.get(unloaded.digest_key(digest), true)
+              asset_from_cache(UnloadedAsset.new(uri_from_cache, self).asset_key)
             end
           else
-            load_asset_by_uri(uri, filename, params)
+            load_from_unloaded(unloaded)
           end
         end
       end
@@ -45,47 +65,58 @@ module Sprockets
     end
 
     private
-      def load_asset_by_id_uri(uri, filename, params)
-        # Internal assertion, should be routed through load_asset_by_uri
-        unless id = params.delete(:id)
-          raise ArgumentError, "expected uri to have an id: #{uri}"
-        end
 
-        uri = build_asset_uri(filename, params)
-        asset = load_asset_by_uri(uri, filename, params)
+      # Internal: Load asset hash from cache
+      #
+      # key - A String containing lookup information for an asset
+      #
+      # This method converts all "compressed" paths to absolute paths.
+      # Returns a hash of values representing an asset
+      def asset_from_cache(key)
+        asset = cache.get(key, true)
+        if asset
+          asset[:uri]       = expand_from_root(asset[:uri])
+          asset[:load_path] = expand_from_root(asset[:load_path])
+          asset[:filename]  = expand_from_root(asset[:filename])
+          asset[:metadata][:included].map!          { |uri| expand_from_root(uri) } if asset[:metadata][:included]
+          asset[:metadata][:links].map!             { |uri| expand_from_root(uri) } if asset[:metadata][:links]
+          asset[:metadata][:stubbed].map!           { |uri| expand_from_root(uri) } if asset[:metadata][:stubbed]
+          asset[:metadata][:required].map!          { |uri| expand_from_root(uri) } if asset[:metadata][:required]
+          asset[:metadata][:dependencies].map!      { |uri| uri.start_with?("file-digest://") ? expand_from_root(uri) : uri } if asset[:metadata][:dependencies]
 
-        if id && asset[:id] != id
-          raise VersionNotFound, "could not find specified id: #{uri}##{id}"
+          asset[:metadata].each_key do |k|
+            next unless k =~ /_dependencies\z/
+            asset[:metadata][k].map! { |uri| expand_from_root(uri) }
+          end
         end
-
         asset
       end
 
-      def load_asset_by_uri(uri, filename, params)
-        # Internal assertion, should be routed through load_asset_by_id_uri
-        if params.key?(:id)
-          raise ArgumentError, "expected uri to have no id: #{uri}"
-        end
-
-        unless file?(filename)
-          raise FileNotFound, "could not find file: #{filename}"
+      # Internal: Loads an asset and saves it to cache
+      #
+      # unloaded - An UnloadedAsset
+      #
+      # This method is only called when the given unloaded asset could not be
+      # successfully pulled from cache.
+      def load_from_unloaded(unloaded)
+        unless file?(unloaded.filename)
+          raise FileNotFound, "could not find file: #{unloaded.filename}"
         end
 
-        load_path, logical_path = paths_split(config[:paths], filename)
+        load_path, logical_path = paths_split(config[:paths], unloaded.filename)
 
         unless load_path
-          raise FileOutsidePaths, "#{filename} is no longer under a load path: #{self.paths.join(', ')}"
+          raise FileOutsidePaths, "#{unloaded.filename} is no longer under a load path: #{self.paths.join(', ')}"
         end
 
         logical_path, file_type, engine_extnames, _ = parse_path_extnames(logical_path)
-        logical_path = normalize_logical_path(logical_path)
         name = logical_path
 
-        if pipeline = params[:pipeline]
+        if pipeline = unloaded.params[:pipeline]
           logical_path += ".#{pipeline}"
         end
 
-        if type = params[:type]
+        if type = unloaded.params[:type]
           logical_path += config[:mime_types][type][:extensions].first
         end
 
@@ -103,72 +134,190 @@ module Sprockets
           result = call_processors(processors, {
             environment: self,
             cache: self.cache,
-            uri: uri,
-            filename: filename,
+            uri: unloaded.uri,
+            filename: unloaded.filename,
             load_path: load_path,
             name: name,
             content_type: type,
             metadata: { dependencies: dependencies }
           })
+          validate_processor_result!(result)
           source = result.delete(:data)
-          metadata = result.merge!(
-            charset: source.encoding.name.downcase,
-            digest: digest(source),
-            length: source.bytesize
-          )
+          metadata = result
+          metadata[:charset] = source.encoding.name.downcase unless metadata.key?(:charset)
+          metadata[:digest]  = digest(source)
+          metadata[:length]  = source.bytesize
         else
+          dependencies << build_file_digest_uri(unloaded.filename)
           metadata = {
-            digest: file_digest(filename),
-            length: self.stat(filename).size,
+            digest: file_digest(unloaded.filename),
+            length: self.stat(unloaded.filename).size,
             dependencies: dependencies
           }
         end
 
         asset = {
-          uri: uri,
+          uri: unloaded.uri,
           load_path: load_path,
-          filename: filename,
+          filename: unloaded.filename,
           name: name,
           logical_path: logical_path,
           content_type: type,
           source: source,
           metadata: metadata,
-          integrity: integrity_uri(metadata[:digest], type),
-          dependencies_digest: digest(resolve_dependencies(metadata[:dependencies]))
+          dependencies_digest: DigestUtils.digest(resolve_dependencies(metadata[:dependencies]))
         }
 
         asset[:id]  = pack_hexdigest(digest(asset))
-        asset[:uri] = build_asset_uri(filename, params.merge(id: asset[:id]))
+        asset[:uri] = build_asset_uri(unloaded.filename, unloaded.params.merge(id: asset[:id]))
 
         # Deprecated: Avoid tracking Asset mtime
         asset[:mtime] = metadata[:dependencies].map { |u|
           if u.start_with?("file-digest:")
             s = self.stat(parse_file_digest_uri(u))
-            s ? s.mtime.to_i : 0
+            s ? s.mtime.to_i : nil
           else
-            0
+            nil
           end
-        }.max
-
-        cache.set("asset-uri:#{VERSION}:#{asset[:uri]}", asset, true)
-        cache.set("asset-uri-digest:#{VERSION}:#{uri}:#{asset[:dependencies_digest]}", asset[:uri], true)
+        }.compact.max
+        asset[:mtime] ||= self.stat(unloaded.filename).mtime.to_i
 
+        store_asset(asset, unloaded)
         asset
       end
 
-      def fetch_asset_from_dependency_cache(uri, filename, limit = 3)
-        key = "asset-uri-cache-dependencies:#{VERSION}:#{uri}:#{file_digest(filename)}"
-        history = cache.get(key) || []
+      # Internal: Save a given asset to the cache
+      #
+      # asset - A hash containing values of loaded asset
+      # unloaded - The UnloadedAsset used to lookup the `asset`
+      #
+      # This method converts all absolute paths to "compressed" paths
+      # which are relative if they're in the root.
+      def store_asset(asset, unloaded)
+        # Save the asset in the cache under the new URI
+        cached_asset             = asset.dup
+        cached_asset[:uri]       = compress_from_root(asset[:uri])
+        cached_asset[:filename]  = compress_from_root(asset[:filename])
+        cached_asset[:load_path] = compress_from_root(asset[:load_path])
 
+        if cached_asset[:metadata]
+          # Deep dup to avoid modifying `asset`
+          cached_asset[:metadata] = cached_asset[:metadata].dup
+          if cached_asset[:metadata][:included] && !cached_asset[:metadata][:included].empty?
+            cached_asset[:metadata][:included] = cached_asset[:metadata][:included].dup
+            cached_asset[:metadata][:included].map! { |uri| compress_from_root(uri) }
+          end
+
+          if cached_asset[:metadata][:links] && !cached_asset[:metadata][:links].empty?
+            cached_asset[:metadata][:links] = cached_asset[:metadata][:links].dup
+            cached_asset[:metadata][:links].map! { |uri| compress_from_root(uri) }
+          end
+
+          if cached_asset[:metadata][:stubbed] && !cached_asset[:metadata][:stubbed].empty?
+            cached_asset[:metadata][:stubbed] = cached_asset[:metadata][:stubbed].dup
+            cached_asset[:metadata][:stubbed].map! { |uri| compress_from_root(uri) }
+          end
+
+          if cached_asset[:metadata][:required] && !cached_asset[:metadata][:required].empty?
+            cached_asset[:metadata][:required] = cached_asset[:metadata][:required].dup
+            cached_asset[:metadata][:required].map! { |uri| compress_from_root(uri) }
+          end
+
+          if cached_asset[:metadata][:dependencies] && !cached_asset[:metadata][:dependencies].empty?
+            cached_asset[:metadata][:dependencies] = cached_asset[:metadata][:dependencies].dup
+            cached_asset[:metadata][:dependencies].map! do |uri|
+              uri.start_with?("file-digest://".freeze) ? compress_from_root(uri) : uri
+            end
+          end
+
+          # compress all _dependencies in metadata like `sass_dependencies`
+          cached_asset[:metadata].each do |key, value|
+            next unless key =~ /_dependencies\z/
+            cached_asset[:metadata][key] = value.dup
+            cached_asset[:metadata][key].map! {|uri| compress_from_root(uri) }
+          end
+        end
+
+        # Unloaded asset and stored_asset now have a different URI
+        stored_asset = UnloadedAsset.new(asset[:uri], self)
+        cache.set(stored_asset.asset_key, cached_asset, true)
+
+        # Save the new relative path for the digest key of the unloaded asset
+        cache.set(unloaded.digest_key(asset[:dependencies_digest]), stored_asset.compressed_path, true)
+      end
+
+
+      # Internal: Resolve set of dependency URIs.
+      #
+      # uris - An Array of "dependencies" for example:
+      #        ["environment-version", "environment-paths", "processors:type=text/css&file_type=text/css",
+      #           "file-digest:///Full/path/app/assets/stylesheets/application.css",
+      #           "processors:type=text/css&file_type=text/css&pipeline=self",
+      #           "file-digest:///Full/path/app/assets/stylesheets"]
+      #
+      # Returns back array of things that the given uri dpends on
+      # For example the environment version, if you're using a different version of sprockets
+      # then the dependencies should be different, this is used only for generating cache key
+      # for example the "environment-version" may be resolved to "environment-1.0-3.2.0" for
+      #  version "3.2.0" of sprockets.
+      #
+      # Any paths that are returned are converted to relative paths
+      #
+      # Returns array of resolved dependencies
+      def resolve_dependencies(uris)
+        uris.map { |uri| resolve_dependency(uri) }
+      end
+
+      # Internal: Retrieves an asset based on its digest
+      #
+      # unloaded - An UnloadedAsset
+      # limit    - A Fixnum which sets the maximum number of versions of "histories"
+      #            stored in the cache
+      #
+      # This method attempts to retrieve the last `limit` number of histories of an asset
+      # from the cache a "history" which is an array of unresolved "dependencies" that the asset needs
+      # to compile. In this case A dependency can refer to either an asset i.e. index.js
+      # may rely on jquery.js (so jquery.js is a depndency), or other factors that may affect
+      # compilation, such as the VERSION of sprockets (i.e. the environment) and what "processors"
+      # are used.
+      #
+      # For example a history array may look something like this
+      #
+      #   [["environment-version", "environment-paths", "processors:type=text/css&file_type=text/css",
+      #     "file-digest:///Full/path/app/assets/stylesheets/application.css",
+      #     "processors:type=text/css&file_digesttype=text/css&pipeline=self",
+      #     "file-digest:///Full/path/app/assets/stylesheets"]]
+      #
+      # Where the first entry is a Set of dependencies for last generated version of that asset.
+      # Multiple versions are stored since sprockets keeps the last `limit` number of assets
+      # generated present in the system.
+      #
+      # If a "history" of dependencies is present in the cache, each version of "history" will be
+      # yielded to the passed block which is responsible for loading the asset. If found, the existing
+      # history will be saved with the dependency that found a valid asset moved to the front.
+      #
+      # If no history is present, or if none of the histories could be resolved to a valid asset then,
+      # the block is yielded to and expected to return a valid asset.
+      # When this happens the dependencies for the returned asset are added to the "history", and older
+      # entries are removed if the "history" is above `limit`.
+      def fetch_asset_from_dependency_cache(unloaded, limit = 3)
+        key = unloaded.dependency_history_key
+
+        history = cache.get(key) || []
         history.each_with_index do |deps, index|
-          if asset = yield(deps)
+          expanded_deps = deps.map do |path|
+            path.start_with?("file-digest://") ? expand_from_root(path) : path
+          end
+          if asset = yield(expanded_deps)
             cache.set(key, history.rotate!(index)) if index > 0
             return asset
           end
         end
 
         asset = yield
-        deps = asset[:metadata][:dependencies]
+        deps  = asset[:metadata][:dependencies].dup.map! do |uri|
+          uri.start_with?("file-digest://") ? compress_from_root(uri) : uri
+        end
         cache.set(key, history.unshift(deps).take(limit))
         asset
       end

data/lib/sprockets/manifest.rb

@@ -1,6 +1,10 @@
 require 'json'
 require 'time'
+
+require 'concurrent'
+
 require 'sprockets/manifest_utils'
+require 'sprockets/utils/gzip'
 
 module Sprockets
   # The Manifest logs the contents of assets compiled to a single directory. It
@@ -145,6 +149,24 @@ module Sprockets
       nil
     end
 
+    # Public: Find the source of assets by paths.
+    #
+    # Returns Enumerator of assets file content.
+    def find_sources(*args)
+      return to_enum(__method__, *args) unless block_given?
+
+      if environment
+        find(*args).each do |asset|
+          yield asset.source
+        end
+      else
+        args.each do |path|
+          asset = assets[path]
+          yield File.binread(File.join(dir, asset)) if asset
+        end
+      end
+    end
+
     # Compile and write asset to directory. The asset is written to a
     # fingerprinted filename like
     # `application-2e8e9a7c6b0aafa0c9bdeec90ea30213.js`. An entry is
@@ -157,7 +179,9 @@ module Sprockets
         raise Error, "manifest requires environment for compilation"
       end
 
-      filenames = []
+      filenames              = []
+      concurrent_compressors = []
+      concurrent_writers     = []
 
       find(*args) do |asset|
         files[asset.digest_path] = {
@@ -165,21 +189,46 @@ module Sprockets
           'mtime'        => asset.mtime.iso8601,
           'size'         => asset.bytesize,
           'digest'       => asset.hexdigest,
-          'integrity'    => asset.integrity
+
+          # Deprecated: Remove beta integrity attribute in next release.
+          # Callers should DigestUtils.hexdigest_integrity_uri to compute the
+          # digest themselves.
+          'integrity'    => DigestUtils.hexdigest_integrity_uri(asset.hexdigest)
         }
         assets[asset.logical_path] = asset.digest_path
 
+        if alias_logical_path = self.class.compute_alias_logical_path(asset.logical_path)
+          assets[alias_logical_path] = asset.digest_path
+        end
+
         target = File.join(dir, asset.digest_path)
 
         if File.exist?(target)
           logger.debug "Skipping #{target}, already exists"
         else
           logger.info "Writing #{target}"
-          asset.write_to target
+          write_file = Concurrent::Future.execute { asset.write_to target }
+          concurrent_writers << write_file
         end
-
         filenames << asset.filename
+
+        next if environment.skip_gzip?
+        gzip = Utils::Gzip.new(asset)
+        next if gzip.cannot_compress?(environment.mime_types)
+
+        if File.exist?("#{target}.gz")
+          logger.debug "Skipping #{target}.gz, already exists"
+        else
+          logger.info "Writing #{target}.gz"
+          concurrent_compressors << Concurrent::Future.execute do
+            write_file.wait! if write_file
+            gzip.compress(target)
+          end
+        end
+
       end
+      concurrent_writers.each(&:wait!)
+      concurrent_compressors.each(&:wait!)
       save
 
       filenames
@@ -192,6 +241,7 @@ module Sprockets
     #
     def remove(filename)
       path = File.join(dir, filename)
+      gzip = "#{path}.gz"
       logical_path = files[filename]['logical_path']
 
       if assets[logical_path] == filename
@@ -200,6 +250,7 @@ module Sprockets
 
       files.delete(filename)
       FileUtils.rm(path) if File.exist?(path)
+      FileUtils.rm(gzip) if File.exist?(gzip)
 
       save
 
@@ -230,9 +281,9 @@ module Sprockets
           # Sort by timestamp
           Time.parse(attrs['mtime'])
         }.reverse.each_with_index.drop_while { |(_, attrs), index|
-          age = [0, Time.now - Time.parse(attrs['mtime'])].max
+          _age = [0, Time.now - Time.parse(attrs['mtime'])].max
           # Keep if under age or within the count limit
-          age < age || index < count
+          _age < age || index < count
         }.each { |(path, _), _|
            # Remove old assets
           remove(path)

data/lib/sprockets/mime.rb

@@ -111,17 +111,37 @@ module Sprockets
       def compute_extname_map
         graph = {}
 
+        engine_extname_permutation = []
+
+        4.times do |n|
+          config[:engines].keys.permutation(n).each do |engine_extnames|
+            engine_extname_permutation << engine_extnames
+          end
+        end
+
+        mime_exts_grouped_by_mime_type = {}
+        config[:mime_exts].each do |format_extname,format_type|
+          mime_exts_grouped_by_mime_type[format_type] ||= []
+          mime_exts_grouped_by_mime_type[format_type] << format_extname
+        end
+
         ([nil] + pipelines.keys.map(&:to_s)).each do |pipeline|
-          pipeline_extname = ".#{pipeline}" if pipeline
-          ([[nil, nil]] + config[:mime_exts].to_a).each do |format_extname, format_type|
-            4.times do |n|
-              config[:engines].keys.permutation(n).each do |engine_extnames|
+          pipeline_extname = pipeline ? ".#{pipeline}" : ''.freeze
+          engine_extname_permutation.each do |engine_extnames|
+            mime_exts_grouped_by_mime_type.each do |format_type, format_extnames|
+              type = format_type
+              value = [type, engine_extnames, pipeline]
+              format_extnames.each do |format_extname|
                 key = "#{pipeline_extname}#{format_extname}#{engine_extnames.join}"
-                type = format_type || config[:engine_mime_types][engine_extnames.first]
-                graph[key] = {type: type, engines: engine_extnames, pipeline: pipeline}
+                graph[key] = value
+              end
+              if format_type == config[:engine_mime_types][engine_extnames.first]
+                key =  "#{pipeline_extname}#{engine_extnames.join}"
+                graph[key] = value
               end
             end
           end
+          graph[pipeline_extname] = [nil, [], pipeline]
         end
 
         graph

data/lib/sprockets/path_utils.rb

@@ -1,5 +1,3 @@
-require 'fileutils'
-
 module Sprockets
   # Internal: File and path related utilities. Mixed into Environment.
   #
@@ -55,9 +53,13 @@ module Sprockets
     # Returns an empty `Array` if the directory does not exist.
     def entries(path)
       if File.directory?(path)
-        Dir.entries(path, :encoding => Encoding.default_internal).reject! { |entry|
-          entry =~ /^\.|~$|^\#.*\#$/
-        }.sort!
+        entries = Dir.entries(path, :encoding => Encoding.default_internal)
+        entries.reject! { |entry|
+          entry.start_with?(".".freeze) ||
+            (entry.start_with?("#".freeze) && entry.end_with?("#".freeze)) ||
+            entry.end_with?("~".freeze)
+        }
+        entries.sort!
       else
         []
       end
@@ -146,16 +148,19 @@ module Sprockets
     #
     # Returns [String extname, Object value] or nil nothing matched.
     def match_path_extname(path, extensions)
-      match, key = nil, ""
-      path_extnames(path).reverse_each do |extname|
-        key.prepend(extname)
-        if value = extensions[key]
-          match = [key.dup, value]
-        elsif match
-          break
+      basename = File.basename(path)
+
+      i = basename.index('.'.freeze)
+      while i && i < basename.length - 1
+        extname = basename[i..-1]
+        if value = extensions[extname]
+          return extname, value
         end
+
+        i = basename.index('.'.freeze, i+1)
       end
-      match
+
+      nil
     end
 
     # Internal: Returns all parents for path
@@ -274,9 +279,9 @@ module Sprockets
         yield f
       end
 
-      FileUtils.mv(tmpname, filename)
+      File.rename(tmpname, filename)
     ensure
-      FileUtils.rm(tmpname) if File.exist?(tmpname)
+      File.delete(tmpname) if File.exist?(tmpname)
     end
   end
 end

data/lib/sprockets/processing.rb

@@ -231,14 +231,24 @@ module Sprockets
         compute_transformers!
       end
 
+      def deprecate_legacy_processor_interface(interface)
+        msg = "You are using a deprecated processor interface #{ interface.inspect }.\n" +
+        "Please update your processor interface:\n" +
+        "https://github.com/rails/sprockets/blob/master/guides/extending_sprockets.md#supporting-all-versions-of-sprockets-in-processors\n"
+
+        Deprecation.new([caller[3]]).warn msg
+      end
+
       def wrap_processor(klass, proc)
         if !proc
           if klass.respond_to?(:call)
             klass
           else
+            deprecate_legacy_processor_interface(klass)
             LegacyTiltProcessor.new(klass)
           end
         elsif proc.respond_to?(:arity) && proc.arity == 2
+          deprecate_legacy_processor_interface(proc)
           LegacyProcProcessor.new(klass.to_s, proc)
         else
           proc