sprockets 2.11.3 → 3.7.2

data/lib/sprockets/manifest.rb

@@ -1,26 +1,31 @@
-require 'multi_json'
-require 'securerandom'
+require 'json'
 require 'time'
 
+require 'concurrent'
+
+require 'sprockets/manifest_utils'
+require 'sprockets/utils/gzip'
+
 module Sprockets
-  # The Manifest logs the contents of assets compiled to a single
-  # directory. It records basic attributes about the asset for fast
-  # lookup without having to compile. A pointer from each logical path
-  # indicates with fingerprinted asset is the current one.
+  # The Manifest logs the contents of assets compiled to a single directory. It
+  # records basic attributes about the asset for fast lookup without having to
+  # compile. A pointer from each logical path indicates which fingerprinted
+  # asset is the current one.
   #
-  # The JSON is part of the public API and should be considered
-  # stable. This should make it easy to read from other programming
-  # languages and processes that don't have sprockets loaded. See
-  # `#assets` and `#files` for more infomation about the structure.
+  # The JSON is part of the public API and should be considered stable. This
+  # should make it easy to read from other programming languages and processes
+  # that don't have sprockets loaded. See `#assets` and `#files` for more
+  # infomation about the structure.
   class Manifest
-    attr_reader :environment, :path, :dir
-
-    # Create new Manifest associated with an `environment`. `path` is
-    # a full path to the manifest json file. The file may or may not
-    # already exist. The dirname of the `path` will be used to write
-    # compiled assets to. Otherwise, if the path is a directory, the
-    # filename will default a random "manifest-123.json" file in that
-    # directory.
+    include ManifestUtils
+
+    attr_reader :environment
+
+    # Create new Manifest associated with an `environment`. `filename` is a full
+    # path to the manifest json file. The file may or may not already exist. The
+    # dirname of the `filename` will be used to write compiled assets to.
+    # Otherwise, if the path is a directory, the filename will default a random
+    # ".sprockets-manifest-*.json" file in that directory.
     #
     #   Manifest.new(environment, "./public/assets/manifest.json")
     #
@@ -29,48 +34,58 @@ module Sprockets
         @environment = args.shift
       end
 
-      @dir, @path = args[0], args[1]
+      @directory, @filename = args[0], args[1]
+
+      # Whether the manifest file is using the old manifest-*.json naming convention
+      @legacy_manifest = false
 
       # Expand paths
-      @dir  = File.expand_path(@dir) if @dir
-      @path = File.expand_path(@path) if @path
+      @directory = File.expand_path(@directory) if @directory
+      @filename  = File.expand_path(@filename) if @filename
 
-      # If path is given as the second arg
-      if @dir && File.extname(@dir) != ""
-        @dir, @path = nil, @dir
+      # If filename is given as the second arg
+      if @directory && File.extname(@directory) != ""
+        @directory, @filename = nil, @directory
       end
 
-      # Default dir to the directory of the path
-      @dir ||= File.dirname(@path) if @path
+      # Default dir to the directory of the filename
+      @directory ||= File.dirname(@filename) if @filename
 
-      # If directory is given w/o path, pick a random manifest.json location
-      if @dir && @path.nil?
-        # Find the first manifest.json in the directory
-        paths = Dir[File.join(@dir, "manifest*.json")]
-        if paths.any?
-          @path = paths.first
-        else
-          @path = File.join(@dir, "manifest-#{SecureRandom.hex(16)}.json")
+      # If directory is given w/o filename, pick a random manifest location
+      @rename_filename = nil
+      if @directory && @filename.nil?
+        @filename = find_directory_manifest(@directory)
+
+        # If legacy manifest name autodetected, mark to rename on save
+        if File.basename(@filename).start_with?("manifest")
+          @rename_filename = File.join(@directory, generate_manifest_path)
         end
       end
 
-      unless @dir && @path
-        raise ArgumentError, "manifest requires output path"
+      unless @directory && @filename
+        raise ArgumentError, "manifest requires output filename"
       end
 
-      data = nil
+      data = {}
 
       begin
-        if File.exist?(@path)
-          data = json_decode(File.read(@path))
+        if File.exist?(@filename)
+          data = json_decode(File.read(@filename))
         end
-      rescue MultiJson::DecodeError => e
-        logger.error "#{@path} is invalid: #{e.class} #{e.message}"
+      rescue JSON::ParserError => e
+        logger.error "#{@filename} is invalid: #{e.class} #{e.message}"
       end
 
-      @data = data.is_a?(Hash) ? data : {}
+      @data = data
     end
 
+    # Returns String path to manifest.json file.
+    attr_reader :filename
+    alias_method :path, :filename
+
+    attr_reader :directory
+    alias_method :dir, :directory
+
     # Returns internal assets mapping. Keys are logical paths which
     # map to the latest fingerprinted filename.
     #
@@ -100,6 +115,58 @@ module Sprockets
       @data['files'] ||= {}
     end
 
+    # Public: Find all assets matching pattern set in environment.
+    #
+    # Returns Enumerator of Assets.
+    def find(*args)
+      unless environment
+        raise Error, "manifest requires environment for compilation"
+      end
+
+      return to_enum(__method__, *args) unless block_given?
+
+      paths, filters = args.flatten.partition { |arg| self.class.simple_logical_path?(arg) }
+      filters = filters.map { |arg| self.class.compile_match_filter(arg) }
+
+      environment = self.environment.cached
+
+      paths.each do |path|
+        environment.find_all_linked_assets(path) do |asset|
+          yield asset
+        end
+      end
+
+      if filters.any?
+        environment.logical_paths do |logical_path, filename|
+          if filters.any? { |f| f.call(logical_path, filename) }
+            environment.find_all_linked_assets(filename) do |asset|
+              yield asset
+            end
+          end
+        end
+      end
+
+      nil
+    end
+
+    # Public: Find the source of assets by paths.
+    #
+    # Returns Enumerator of assets file content.
+    def find_sources(*args)
+      return to_enum(__method__, *args) unless block_given?
+
+      if environment
+        find(*args).each do |asset|
+          yield asset.source
+        end
+      else
+        args.each do |path|
+          asset = assets[path]
+          yield File.binread(File.join(dir, asset)) if asset
+        end
+      end
+    end
+
     # Compile and write asset to directory. The asset is written to a
     # fingerprinted filename like
     # `application-2e8e9a7c6b0aafa0c9bdeec90ea30213.js`. An entry is
@@ -112,33 +179,59 @@ module Sprockets
         raise Error, "manifest requires environment for compilation"
       end
 
-      paths = environment.each_logical_path(*args).to_a +
-        args.flatten.select { |fn| Pathname.new(fn).absolute? if fn.is_a?(String)}
+      filenames              = []
+      concurrent_compressors = []
+      concurrent_writers     = []
+
+      find(*args) do |asset|
+        files[asset.digest_path] = {
+          'logical_path' => asset.logical_path,
+          'mtime'        => asset.mtime.iso8601,
+          'size'         => asset.bytesize,
+          'digest'       => asset.hexdigest,
+
+          # Deprecated: Remove beta integrity attribute in next release.
+          # Callers should DigestUtils.hexdigest_integrity_uri to compute the
+          # digest themselves.
+          'integrity'    => DigestUtils.hexdigest_integrity_uri(asset.hexdigest)
+        }
+        assets[asset.logical_path] = asset.digest_path
+
+        if alias_logical_path = self.class.compute_alias_logical_path(asset.logical_path)
+          assets[alias_logical_path] = asset.digest_path
+        end
 
-      paths.each do |path|
-        if asset = find_asset(path)
-          files[asset.digest_path] = {
-            'logical_path' => asset.logical_path,
-            'mtime'        => asset.mtime.iso8601,
-            'size'         => asset.bytesize,
-            'digest'       => asset.digest
-          }
-          assets[asset.logical_path] = asset.digest_path
-
-          target = File.join(dir, asset.digest_path)
-
-          if File.exist?(target)
-            logger.debug "Skipping #{target}, already exists"
-          else
-            logger.info "Writing #{target}"
-            asset.write_to target
-            asset.write_to "#{target}.gz" if asset.is_a?(BundledAsset)
-          end
+        target = File.join(dir, asset.digest_path)
 
+        if File.exist?(target)
+          logger.debug "Skipping #{target}, already exists"
+        else
+          logger.info "Writing #{target}"
+          write_file = Concurrent::Future.execute { asset.write_to target }
+          concurrent_writers << write_file
         end
+        filenames << asset.filename
+
+        next if environment.skip_gzip?
+        gzip = Utils::Gzip.new(asset)
+        next if gzip.cannot_compress?(environment.mime_types)
+
+        if File.exist?("#{target}.gz")
+          logger.debug "Skipping #{target}.gz, already exists"
+        else
+          logger.info "Writing #{target}.gz"
+          concurrent_compressors << Concurrent::Future.execute do
+            write_file.wait! if write_file
+            gzip.compress(target)
+          end
+        end
+
       end
+      concurrent_writers.each(&:wait!)
+      concurrent_compressors.each(&:wait!)
       save
-      paths
+
+      filenames
     end
 
     # Removes file from directory and from manifest. `filename` must
@@ -167,79 +260,67 @@ module Sprockets
     end
 
     # Cleanup old assets in the compile directory. By default it will
-    # keep the latest version plus 2 backups.
-    def clean(keep = 2)
-      self.assets.keys.each do |logical_path|
-        # Get assets sorted by ctime, newest first
-        assets = backups_for(logical_path)
+    # keep the latest version, 2 backups and any created within the past hour.
+    #
+    # Examples
+    #
+    #   To force only 1 backup to be kept, set count=1 and age=0.
+    #
+    #   To only keep files created within the last 10 minutes, set count=0 and
+    #   age=600.
+    #
+    def clean(count = 2, age = 3600)
+      asset_versions = files.group_by { |_, attrs| attrs['logical_path'] }
 
-        # Keep the last N backups
-        assets = assets[keep..-1] || []
+      asset_versions.each do |logical_path, versions|
+        current = assets[logical_path]
 
-        # Remove old assets
-        assets.each { |path, _| remove(path) }
+        versions.reject { |path, _|
+          path == current
+        }.sort_by { |_, attrs|
+          # Sort by timestamp
+          Time.parse(attrs['mtime'])
+        }.reverse.each_with_index.drop_while { |(_, attrs), index|
+          _age = [0, Time.now - Time.parse(attrs['mtime'])].max
+          # Keep if under age or within the count limit
+          _age < age || index < count
+        }.each { |(path, _), _|
+           # Remove old assets
+          remove(path)
+        }
       end
     end
 
     # Wipe directive
     def clobber
-      FileUtils.rm_r(@dir) if File.exist?(@dir)
-      logger.info "Removed #{@dir}"
+      FileUtils.rm_r(directory) if File.exist?(directory)
+      logger.info "Removed #{directory}"
       nil
     end
 
-    protected
-      # Finds all the backup assets for a logical path. The latest
-      # version is always excluded. The return array is sorted by the
-      # assets mtime in descending order (Newest to oldest).
-      def backups_for(logical_path)
-        files.select { |filename, attrs|
-          # Matching logical paths
-          attrs['logical_path'] == logical_path &&
-            # Excluding whatever asset is the current
-            assets[logical_path] != filename
-        }.sort_by { |filename, attrs|
-          # Sort by timestamp
-          Time.parse(attrs['mtime'])
-        }.reverse
+    # Persist manfiest back to FS
+    def save
+      if @rename_filename
+        logger.info "Renaming #{@filename} to #{@rename_filename}"
+        FileUtils.mv(@filename, @rename_filename)
+        @filename = @rename_filename
+        @rename_filename = nil
       end
 
-      # Basic wrapper around Environment#find_asset. Logs compile time.
-      def find_asset(logical_path)
-        asset = nil
-        ms = benchmark do
-          asset = environment.find_asset(logical_path)
-        end
-        logger.debug "Compiled #{logical_path}  (#{ms}ms)"
-        asset
-      end
-
-      # Persist manfiest back to FS
-      def save
-        FileUtils.mkdir_p dir
-        File.open(path, 'w') do |f|
-          f.write json_encode(@data)
-        end
+      data = json_encode(@data)
+      FileUtils.mkdir_p File.dirname(@filename)
+      PathUtils.atomic_write(@filename) do |f|
+        f.write(data)
       end
+    end
 
     private
-      # Feature detect newer MultiJson API
-      if MultiJson.respond_to?(:dump)
-        def json_decode(obj)
-          MultiJson.load(obj)
-        end
-
-        def json_encode(obj)
-          MultiJson.dump(obj)
-        end
-      else
-        def json_decode(obj)
-          MultiJson.decode(obj)
-        end
+      def json_decode(obj)
+        JSON.parse(obj, create_additions: false)
+      end
 
-        def json_encode(obj)
-          MultiJson.encode(obj)
-        end
+      def json_encode(obj)
+        JSON.generate(obj)
       end
 
       def logger
@@ -251,11 +332,5 @@ module Sprockets
           logger
         end
       end
-
-      def benchmark
-        start_time = Time.now.to_f
-        yield
-        ((Time.now.to_f - start_time) * 1000).to_i
-      end
   end
 end

data/lib/sprockets/manifest_utils.rb

@@ -0,0 +1,45 @@
+require 'securerandom'
+
+module Sprockets
+  # Public: Manifest utilities.
+  module ManifestUtils
+    extend self
+
+    MANIFEST_RE = /^\.sprockets-manifest-[0-9a-f]{32}.json$/
+    LEGACY_MANIFEST_RE = /^manifest(-[0-9a-f]{32})?.json$/
+
+    # Public: Generate a new random manifest path.
+    #
+    # Manifests are not intended to be accessed publicly, but typically live
+    # alongside public assets for convenience. To avoid being served, the
+    # filename is prefixed with a "." which is usually hidden by web servers
+    # like Apache. To help in other environments that may not control this,
+    # a random hex string is appended to the filename to prevent people from
+    # guessing the location. If directory indexes are enabled on the server,
+    # all bets are off.
+    #
+    # Return String path.
+    def generate_manifest_path
+      ".sprockets-manifest-#{SecureRandom.hex(16)}.json"
+    end
+
+    # Public: Find or pick a new manifest filename for target build directory.
+    #
+    # dirname - String dirname
+    #
+    # Examples
+    #
+    #     find_directory_manifest("/app/public/assets")
+    #     # => "/app/public/assets/.sprockets-manifest-abc123.json"
+    #
+    # Returns String filename.
+    def find_directory_manifest(dirname)
+      entries = File.directory?(dirname) ? Dir.entries(dirname) : []
+      entry = entries.find { |e| e =~ MANIFEST_RE } ||
+        # Deprecated: Will be removed in 4.x
+        entries.find { |e| e =~ LEGACY_MANIFEST_RE } ||
+        generate_manifest_path
+      File.join(dirname, entry)
+    end
+  end
+end

data/lib/sprockets/mime.rb

@@ -1,49 +1,130 @@
-require 'rack/mime'
+require 'sprockets/encoding_utils'
+require 'sprockets/http_utils'
+require 'sprockets/utils'
 
 module Sprockets
   module Mime
-    # Returns a `Hash` of registered mime types registered on the
-    # environment and those part of `Rack::Mime`.
+    include HTTPUtils, Utils
+
+    # Public: Mapping of MIME type Strings to properties Hash.
     #
-    # If an `ext` is given, it will lookup the mime type for that extension.
-    def mime_types(ext = nil)
-      if ext.nil?
-        Rack::Mime::MIME_TYPES.merge(@mime_types)
-      else
-        ext = Sprockets::Utils.normalize_extension(ext)
-        @mime_types[ext] || Rack::Mime::MIME_TYPES[ext]
-      end
+    # key   - MIME Type String
+    # value - Hash
+    #   extensions - Array of extnames
+    #   charset    - Default Encoding or function to detect encoding
+    #
+    # Returns Hash.
+    def mime_types
+      config[:mime_types]
     end
 
-    # Returns a `Hash` of explicitly registered mime types.
-    def registered_mime_types
-      @mime_types.dup
+    # Internal: Mapping of MIME extension Strings to MIME type Strings.
+    #
+    # Used for internal fast lookup purposes.
+    #
+    # Examples:
+    #
+    #   mime_exts['.js'] #=> 'application/javascript'
+    #
+    # key   - MIME extension String
+    # value - MIME Type String
+    #
+    # Returns Hash.
+    def mime_exts
+      config[:mime_exts]
     end
 
-    if {}.respond_to?(:key)
-      def extension_for_mime_type(type)
-        mime_types.key(type)
+    # Public: Register a new mime type.
+    #
+    # mime_type - String MIME Type
+    # options - Hash
+    #   extensions: Array of String extnames
+    #   charset: Proc/Method that detects the charset of a file.
+    #            See EncodingUtils.
+    #
+    # Returns nothing.
+    def register_mime_type(mime_type, options = {})
+      # Legacy extension argument, will be removed from 4.x
+      if options.is_a?(String)
+        options = { extensions: [options] }
+      end
+
+      extnames = Array(options[:extensions]).map { |extname|
+        Sprockets::Utils.normalize_extension(extname)
+      }
+
+      charset = options[:charset]
+      charset ||= :default if mime_type.start_with?('text/')
+      charset = EncodingUtils::CHARSET_DETECT[charset] if charset.is_a?(Symbol)
+
+      self.computed_config = {}
+
+      self.config = hash_reassoc(config, :mime_exts) do |mime_exts|
+        extnames.each do |extname|
+          mime_exts[extname] = mime_type
+        end
+        mime_exts
       end
-    else
-      def extension_for_mime_type(type)
-        mime_types.index(type)
+
+      self.config = hash_reassoc(config, :mime_types) do |mime_types|
+        type = { extensions: extnames }
+        type[:charset] = charset if charset
+        mime_types.merge(mime_type => type)
       end
     end
 
-    # Register a new mime type.
-    def register_mime_type(mime_type, ext)
-      ext = Sprockets::Utils.normalize_extension(ext)
-      @mime_types[ext] = mime_type
+    # Internal: Get detecter function for MIME type.
+    #
+    # mime_type - String MIME type
+    #
+    # Returns Proc detector or nil if none is available.
+    def mime_type_charset_detecter(mime_type)
+      if type = config[:mime_types][mime_type]
+        if detect = type[:charset]
+          return detect
+        end
+      end
     end
 
-    if defined? Encoding
-      # Returns the correct encoding for a given mime type, while falling
-      # back on the default external encoding, if it exists.
-      def encoding_for_mime_type(type)
-        encoding = Encoding::BINARY if type =~ %r{^(image|audio|video)/}
-        encoding ||= default_external_encoding if respond_to?(:default_external_encoding)
-        encoding
+    # Public: Read file on disk with MIME type specific encoding.
+    #
+    # filename     - String path
+    # content_type - String MIME type
+    #
+    # Returns String file contents transcoded to UTF-8 or in its external
+    # encoding.
+    def read_file(filename, content_type = nil)
+      data = File.binread(filename)
+
+      if detect = mime_type_charset_detecter(content_type)
+        detect.call(data).encode(Encoding::UTF_8, :universal_newline => true)
+      else
+        data
       end
     end
+
+    private
+      def extname_map
+        self.computed_config[:_extnames] ||= compute_extname_map
+      end
+
+      def compute_extname_map
+        graph = {}
+
+        ([nil] + pipelines.keys.map(&:to_s)).each do |pipeline|
+          pipeline_extname = ".#{pipeline}" if pipeline
+          ([[nil, nil]] + config[:mime_exts].to_a).each do |format_extname, format_type|
+            4.times do |n|
+              config[:engines].keys.permutation(n).each do |engine_extnames|
+                key = "#{pipeline_extname}#{format_extname}#{engine_extnames.join}"
+                type = format_type || config[:engine_mime_types][engine_extnames.first]
+                graph[key] = {type: type, engines: engine_extnames, pipeline: pipeline}
+              end
+            end
+          end
+        end
+
+        graph
+      end
   end
 end