sprockets 2.12.5 → 3.0.0.beta.1

data/lib/sprockets/mime.rb

@@ -1,49 +1,130 @@
-require 'rack/mime'
+require 'sprockets/encoding_utils'
 
 module Sprockets
   module Mime
-    # Returns a `Hash` of registered mime types registered on the
-    # environment and those part of `Rack::Mime`.
-    #
-    # If an `ext` is given, it will lookup the mime type for that extension.
-    def mime_types(ext = nil)
-      if ext.nil?
-        Rack::Mime::MIME_TYPES.merge(@mime_types)
-      else
-        ext = Sprockets::Utils.normalize_extension(ext)
-        @mime_types[ext] || Rack::Mime::MIME_TYPES[ext]
+    include HTTPUtils
+
+    # Public: Mapping of MIME type Strings to properties Hash.
+    #
+    # key   - MIME Type String
+    # value - Hash
+    #   extensions - Array of extnames
+    #   charset    - Default Encoding or function to detect encoding
+    #
+    # Returns Hash.
+    attr_reader :mime_types
+
+    # Internal: Mapping of MIME extension Strings to MIME type Strings.
+    #
+    # Used for internal fast lookup purposes.
+    #
+    # Examples:
+    #
+    #   mime_exts['.js'] #=> 'application/javascript'
+    #
+    # key   - MIME extension String
+    # value - MIME Type String
+    #
+    # Returns Hash.
+    attr_reader :mime_exts
+
+    # Public: Register a new mime type.
+    #
+    # mime_type - String MIME Type
+    # options - Hash
+    #   extensions: Array of String extnames
+    #   charset: Proc/Method that detects the charset of a file.
+    #            See EncodingUtils.
+    #
+    # Returns nothing.
+    def register_mime_type(mime_type, options = {})
+      # Legacy extension argument, will be removed from 4.x
+      if options.is_a?(String)
+        options = { extensions: [options] }
       end
-    end
 
-    # Returns a `Hash` of explicitly registered mime types.
-    def registered_mime_types
-      @mime_types.dup
-    end
+      extnames = Array(options[:extensions]).map { |extname|
+        Sprockets::Utils.normalize_extension(extname)
+      }
 
-    if {}.respond_to?(:key)
-      def extension_for_mime_type(type)
-        mime_types.key(type)
+      charset = options[:charset]
+      charset ||= EncodingUtils::DETECT if mime_type.start_with?('text/')
+
+      mutate_config(:mime_exts) do |mime_exts|
+        extnames.each do |extname|
+          mime_exts[extname] = mime_type
+        end
+        mime_exts
       end
-    else
-      def extension_for_mime_type(type)
-        mime_types.index(type)
+
+      mutate_config(:mime_types) do |mime_types|
+        type = { extensions: extnames }
+        type[:charset] = charset if charset
+        mime_types.merge(mime_type => type)
       end
     end
 
-    # Register a new mime type.
-    def register_mime_type(mime_type, ext)
-      ext = Sprockets::Utils.normalize_extension(ext)
-      @mime_types[ext] = mime_type
+    # Public: Read file on disk with MIME type specific encoding.
+    #
+    # filename     - String path
+    # content_type - String MIME type
+    #
+    # Returns String file contents transcoded to UTF-8 or in its external
+    # encoding.
+    def read_file(filename, content_type = nil)
+      data = File.open(filename, 'rb') { |f| f.read }
+
+      if type = mime_types[content_type]
+        if charset = type[:charset]
+          data = charset.call(data).encode(Encoding::UTF_8)
+        end
+      end
+
+      data
     end
 
-    if defined? Encoding
-      # Returns the correct encoding for a given mime type, while falling
-      # back on the default external encoding, if it exists.
-      def encoding_for_mime_type(type)
-        encoding = Encoding::BINARY if type =~ %r{^(image|audio|video)/}
-        encoding ||= default_external_encoding if respond_to?(:default_external_encoding)
-        encoding
+    # Public: Mapping of supported HTTP Content/Transfer encodings
+    #
+    # key   - String name
+    # value - Method/Proc to encode data
+    #
+    # Returns Hash.
+    attr_reader :encodings
+
+    # Public: Register a new encoding.
+    #
+    # Examples
+    #
+    #   register_encoding :gzip, EncodingUtils::GZIP
+    #
+    # key    - String name
+    # encode - Method/Proc to encode data
+    #
+    # Returns nothing.
+    def register_encoding(name, encode)
+      mutate_config(:encodings) do |encodings|
+        encodings.merge(name.to_s => encode)
       end
     end
+
+    private
+      def read_input(input)
+        read_file(input[:filename], input[:content_type])
+      end
+
+      # Internal: Get a postprocessor to perform the encoding.
+      #
+      # encoding - String encoding.
+      #
+      # Returns an Array of Processors.
+      def unwrap_encoding_processors(encoding)
+        processors = []
+        if encoder = self.encodings[encoding]
+          processors << lambda do |input|
+            { data: encoder.call(input[:data]), encoding: encoding }
+          end
+        end
+        processors
+      end
   end
 end

data/lib/sprockets/path_utils.rb

@@ -0,0 +1,179 @@
+require 'fileutils'
+
+module Sprockets
+  # Internal: File and path related utilities. Mixed into Environment.
+  #
+  # Probably would be called FileUtils, but that causes namespace annoyances
+  # when code actually wants to reference ::FileUtils.
+  module PathUtils
+    extend self
+
+    # Internal: Like `File.stat`.
+    #
+    # path - String file or directory path
+    #
+    # Returns nil if the file does not exist.
+    def stat(path)
+      if File.exist?(path)
+        File.stat(path.to_s)
+      else
+        nil
+      end
+    end
+
+    # Internal: Like `File.file?`.
+    #
+    # path - String file path.
+    #
+    # Returns true path exists and is a file.
+    def file?(path)
+      if stat = self.stat(path)
+        stat.file?
+      else
+        false
+      end
+    end
+
+    # Internal: A version of `Dir.entries` that filters out `.` files and `~`
+    # swap files.
+    #
+    # path - String directory path
+    #
+    # Returns an empty `Array` if the directory does not exist.
+    def entries(path)
+      if File.directory?(path)
+        Dir.entries(path).reject { |entry| entry =~ /^\.|~$|^\#.*\#$/ }.sort
+      else
+        []
+      end
+    end
+
+    # Internal: Check if path is absolute or relative.
+    #
+    # path - String path.
+    #
+    # Returns true if path is absolute, otherwise false.
+    if File::ALT_SEPARATOR
+      # On Windows, ALT_SEPARATOR is \
+      def absolute_path?(path)
+        path[0] == File::SEPARATOR || path[0] == File::ALT_SEPARATOR
+      end
+    else
+      def absolute_path?(path)
+        path[0] == File::SEPARATOR
+      end
+    end
+
+    # Internal: Check if path is explicitly relative.
+    # Starts with "./" or "../".
+    #
+    # path - String path.
+    #
+    # Returns true if path is relative, otherwise false.
+    def relative_path?(path)
+      path =~ /^\.\.?($|\/)/ ? true : false
+    end
+
+    # Internal: Get relative path for root path and subpath.
+    #
+    # path    - String path
+    # subpath - String subpath of path
+    #
+    # Returns relative String path if subpath is a subpath of path, or nil if
+    # subpath is outside of path.
+    def split_subpath(path, subpath)
+      path = File.join(path, '')
+      if subpath.start_with?(path)
+        subpath[path.length..-1]
+      else
+        nil
+      end
+    end
+
+    # Internal: Detect root path and base for file in a set of paths.
+    #
+    # paths    - Array of String paths
+    # filename - String path of file expected to be in one of the paths.
+    #
+    # Returns [String root, String path]
+    def paths_split(paths, filename)
+      paths.each do |path|
+        if subpath = split_subpath(path, filename)
+          return path, subpath
+        end
+      end
+      nil
+    end
+
+    # Internal: Get path's extensions.
+    #
+    # path - String
+    #
+    # Returns an Array of String extnames.
+    def path_extnames(path)
+      File.basename(path).scan(/\.[^.]+/)
+    end
+
+    # Internal: Stat all the files under a directory.
+    #
+    # dir - A String directory
+    #
+    # Returns an Enumerator of [path, stat].
+    def stat_directory(dir)
+      return to_enum(__method__, dir) unless block_given?
+
+      self.entries(dir).each do |entry|
+        path = File.join(dir, entry)
+        if stat = self.stat(path)
+          yield path, stat
+        end
+      end
+
+      nil
+    end
+
+    # Internal: Recursive stat all the files under a directory.
+    #
+    # dir - A String directory
+    #
+    # Returns an Enumerator of [path, stat].
+    def stat_tree(dir, &block)
+      return to_enum(__method__, dir) unless block_given?
+
+      self.stat_directory(dir) do |path, stat|
+        yield path, stat
+
+        if stat.directory?
+          stat_tree(path, &block)
+        end
+      end
+
+      nil
+    end
+
+    # Internal: Write to a file atomically. Useful for situations where you
+    # don't want other processes or threads to see half-written files.
+    #
+    #   Utils.atomic_write('important.file') do |file|
+    #     file.write('hello')
+    #   end
+    #
+    # Returns nothing.
+    def atomic_write(filename)
+      tmpname = [
+        File.dirname(filename),
+        Thread.current.object_id,
+        Process.pid,
+        rand(1000000)
+      ].join('.')
+
+      File.open(tmpname, 'wb+') do |f|
+        yield f
+      end
+
+      FileUtils.mv(tmpname, filename)
+    ensure
+      FileUtils.rm(tmpname) if File.exist?(tmpname)
+    end
+  end
+end

data/lib/sprockets/paths.rb

@@ -4,33 +4,31 @@ module Sprockets
     #
     # All relative paths are expanded with root as its base. To be
     # useful set this to your applications root directory. (`Rails.root`)
-    def root
-      @trail.root.dup
-    end
+    attr_reader :root
 
     # Returns an `Array` of path `String`s.
     #
     # These paths will be used for asset logical path lookups.
-    #
-    # Note that a copy of the `Array` is returned so mutating will
-    # have no affect on the environment. See `append_path`,
-    # `prepend_path`, and `clear_paths`.
-    def paths
-      @trail.paths.dup
-    end
+    attr_reader :paths
 
     # Prepend a `path` to the `paths` list.
     #
     # Paths at the end of the `Array` have the least priority.
     def prepend_path(path)
-      @trail.prepend_path(path)
+      mutate_config(:paths) do |paths|
+        path = File.expand_path(path, root).dup.freeze
+        paths.unshift(path)
+      end
     end
 
     # Append a `path` to the `paths` list.
     #
     # Paths at the beginning of the `Array` have a higher priority.
     def append_path(path)
-      @trail.append_path(path)
+      mutate_config(:paths) do |paths|
+        path = File.expand_path(path, root).dup.freeze
+        paths.push(path)
+      end
     end
 
     # Clear all paths and start fresh.
@@ -39,20 +37,9 @@ module Sprockets
     # completely wipe the paths list and reappend them in the order
     # you want.
     def clear_paths
-      @trail.paths.dup.each { |path| @trail.remove_path(path) }
+      mutate_config(:paths) do |paths|
+        paths.clear
+      end
     end
-
-    # Returns an `Array` of extensions.
-    #
-    # These extensions maybe omitted from logical path searches.
-    #
-    #     # => [".js", ".css", ".coffee", ".sass", ...]
-    #
-    def extensions
-      @trail.extensions.dup
-    end
-
-    protected
-      attr_reader :trail
   end
 end

data/lib/sprockets/processing.rb

@@ -1,64 +1,43 @@
 require 'sprockets/engines'
+require 'sprockets/lazy_processor'
+require 'sprockets/legacy_proc_processor'
+require 'sprockets/legacy_tilt_processor'
 require 'sprockets/mime'
-require 'sprockets/processor'
 require 'sprockets/utils'
 
 module Sprockets
   # `Processing` is an internal mixin whose public methods are exposed on
-  # the `Environment` and `Index` classes.
+  # the `Environment` and `CachedEnvironment` classes.
   module Processing
-    # Returns an `Array` of format extension `String`s.
-    #
-    #     format_extensions
-    #     # => ['.js', '.css']
-    #
-    def format_extensions
-      @trail.extensions - @engines.keys
-    end
-
-    # Deprecated alias for `preprocessors`.
-    def processors(*args)
-      preprocessors(*args)
-    end
-
-    # Returns an `Array` of `Processor` classes. If a `mime_type`
-    # argument is supplied, the processors registered under that
-    # extension will be returned.
-    #
     # Preprocessors are ran before Postprocessors and Engine
     # processors.
+    attr_reader :preprocessors
+
+    # Internal: Find and load preprocessors by mime type.
     #
-    # All `Processor`s must follow the `Tilt::Template` interface. It is
-    # recommended to subclass `Tilt::Template`.
-    def preprocessors(mime_type = nil)
-      if mime_type
-        @preprocessors[mime_type].dup
-      else
-        deep_copy_hash(@preprocessors)
+    # mime_type - String MIME type.
+    #
+    # Returns Array of Procs.
+    def unwrap_preprocessors(mime_type)
+      preprocessors[mime_type].map do |processor|
+        unwrap_processor(processor)
       end
     end
 
-    # Returns an `Array` of `Processor` classes. If a `mime_type`
-    # argument is supplied, the processors registered under that
-    # extension will be returned.
-    #
     # Postprocessors are ran after Preprocessors and Engine processors.
+    attr_reader :postprocessors
+
+    # Internal: Find and load postprocessors by mime type.
     #
-    # All `Processor`s must follow the `Tilt::Template` interface. It is
-    # recommended to subclass `Tilt::Template`.
-    def postprocessors(mime_type = nil)
-      if mime_type
-        @postprocessors[mime_type].dup
-      else
-        deep_copy_hash(@postprocessors)
+    # mime_type - String MIME type.
+    #
+    # Returns Array of Procs.
+    def unwrap_postprocessors(mime_type)
+      postprocessors[mime_type].map do |processor|
+        unwrap_processor(processor)
       end
     end
 
-    # Deprecated alias for `register_preprocessor`.
-    def register_processor(*args, &block)
-      register_preprocessor(*args, &block)
-    end
-
     # Registers a new Preprocessor `klass` for `mime_type`.
     #
     #     register_preprocessor 'text/css', Sprockets::DirectiveProcessor
@@ -70,42 +49,28 @@ module Sprockets
     #     end
     #
     def register_preprocessor(mime_type, klass, &block)
-      if block_given?
-        name  = klass.to_s
-        klass = Class.new(Processor) do
-          @name      = name
-          @processor = block
-        end
+      mutate_hash_config(:preprocessors, mime_type) do |processors|
+        processors.push(wrap_processor(klass, block))
+        processors
       end
-
-      @preprocessors[mime_type].push(klass)
     end
 
     # Registers a new Postprocessor `klass` for `mime_type`.
     #
-    #     register_postprocessor 'text/css', Sprockets::CharsetNormalizer
+    #     register_postprocessor 'application/javascript', Sprockets::DirectiveProcessor
     #
     # A block can be passed for to create a shorthand processor.
     #
-    #     register_postprocessor 'text/css', :my_processor do |context, data|
+    #     register_postprocessor 'application/javascript', :my_processor do |context, data|
     #       data.gsub(...)
     #     end
     #
-    def register_postprocessor(mime_type, klass, &block)
-      if block_given?
-        name  = klass.to_s
-        klass = Class.new(Processor) do
-          @name      = name
-          @processor = block
-        end
+    def register_postprocessor(mime_type, klass, proc = nil, &block)
+      proc ||= block
+      mutate_hash_config(:postprocessors, mime_type) do |processors|
+        processors.push(wrap_processor(klass, proc))
+        processors
       end
-
-      @postprocessors[mime_type].push(klass)
-    end
-
-    # Deprecated alias for `unregister_preprocessor`.
-    def unregister_processor(*args)
-      unregister_preprocessor(*args)
     end
 
     # Remove Preprocessor `klass` for `mime_type`.
@@ -114,13 +79,15 @@ module Sprockets
     #
     def unregister_preprocessor(mime_type, klass)
       if klass.is_a?(String) || klass.is_a?(Symbol)
-        klass = @preprocessors[mime_type].detect { |cls|
-          cls.respond_to?(:name) &&
-            cls.name == "Sprockets::Processor (#{klass})"
+        klass = preprocessors[mime_type].detect { |cls|
+          cls.respond_to?(:name) && cls.name == "Sprockets::LegacyProcProcessor (#{klass})"
         }
       end
 
-      @preprocessors[mime_type].delete(klass)
+      mutate_hash_config(:preprocessors, mime_type) do |processors|
+        processors.delete(klass)
+        processors
+      end
     end
 
     # Remove Postprocessor `klass` for `mime_type`.
@@ -129,78 +96,202 @@ module Sprockets
     #
     def unregister_postprocessor(mime_type, klass)
       if klass.is_a?(String) || klass.is_a?(Symbol)
-        klass = @postprocessors[mime_type].detect { |cls|
-          cls.respond_to?(:name) &&
-            cls.name == "Sprockets::Processor (#{klass})"
+        klass = postprocessors[mime_type].detect { |cls|
+          cls.respond_to?(:name) && cls.name == "Sprockets::LegacyProcProcessor (#{klass})"
         }
       end
 
-      @postprocessors[mime_type].delete(klass)
+      mutate_hash_config(:postprocessors, mime_type) do |processors|
+        processors.delete(klass)
+        processors
+      end
     end
 
-    # Returns an `Array` of `Processor` classes. If a `mime_type`
-    # argument is supplied, the processors registered under that
-    # extension will be returned.
-    #
     # Bundle Processors are ran on concatenated assets rather than
     # individual files.
+    attr_reader :bundle_processors
+
+    # Internal: Find and load bundle processors by mime type.
     #
-    # All `Processor`s must follow the `Tilt::Template` interface. It is
-    # recommended to subclass `Tilt::Template`.
-    def bundle_processors(mime_type = nil)
-      if mime_type
-        @bundle_processors[mime_type].dup
-      else
-        deep_copy_hash(@bundle_processors)
+    # mime_type - String MIME type.
+    #
+    # Returns Array of Procs.
+    def unwrap_bundle_processors(mime_type)
+      bundle_processors[mime_type].map do |processor|
+        unwrap_processor(processor)
       end
     end
 
     # Registers a new Bundle Processor `klass` for `mime_type`.
     #
-    #     register_bundle_processor  'text/css', Sprockets::CharsetNormalizer
+    #     register_bundle_processor  'application/javascript', Sprockets::DirectiveProcessor
     #
     # A block can be passed for to create a shorthand processor.
     #
-    #     register_bundle_processor 'text/css', :my_processor do |context, data|
+    #     register_bundle_processor 'application/javascript', :my_processor do |context, data|
     #       data.gsub(...)
     #     end
     #
     def register_bundle_processor(mime_type, klass, &block)
-      if block_given?
-        name  = klass.to_s
-        klass = Class.new(Processor) do
-          @name      = name
-          @processor = block
-        end
+      mutate_hash_config(:bundle_processors, mime_type) do |processors|
+        processors.push(wrap_processor(klass, block))
+        processors
       end
-
-      @bundle_processors[mime_type].push(klass)
     end
 
     # Remove Bundle Processor `klass` for `mime_type`.
     #
-    #     unregister_bundle_processor 'text/css', Sprockets::CharsetNormalizer
+    #     unregister_bundle_processor 'application/javascript', Sprockets::DirectiveProcessor
     #
     def unregister_bundle_processor(mime_type, klass)
       if klass.is_a?(String) || klass.is_a?(Symbol)
-        klass = @bundle_processors[mime_type].detect { |cls|
-          cls.respond_to?(:name) &&
-            cls.name == "Sprockets::Processor (#{klass})"
+        klass = bundle_processors[mime_type].detect { |cls|
+          cls.respond_to?(:name) && cls.name == "Sprockets::LegacyProcProcessor (#{klass})"
         }
       end
 
-      @bundle_processors[mime_type].delete(klass)
+      mutate_hash_config(:bundle_processors, mime_type) do |processors|
+        processors.delete(klass)
+        processors
+      end
     end
 
-    private
-      def add_engine_to_trail(ext, klass)
-        @trail.append_extension(ext.to_s)
+    # Internal: Run processors on filename and data.
+    #
+    # Returns Hash.
+    def process(processors, uri, filename, load_path, name, content_type)
+      data, metadata = nil, {}
+
+      input = {
+        environment: self,
+        cache: cache,
+        uri: uri,
+        filename: filename,
+        load_path: load_path,
+        name: name,
+        content_type: content_type,
+        metadata: metadata
+      }
 
-        if klass.respond_to?(:default_mime_type) && klass.default_mime_type
-          if format_ext = extension_for_mime_type(klass.default_mime_type)
-            @trail.alias_extension(ext.to_s, format_ext)
+      processors.each do |processor|
+        begin
+          result = processor.call(input.merge(data: data, metadata: metadata))
+          case result
+          when NilClass
+            # noop
+          when Hash
+            data = result[:data]
+            metadata = metadata.merge(result)
+            metadata.delete(:data)
+          when String
+            data = result
+          else
+            raise Error, "invalid processor return type: #{result.class}"
           end
         end
       end
+
+      {
+        source: data,
+        charset: data.encoding.name.downcase,
+        length: data.bytesize,
+        digest: digest_class.hexdigest(data),
+        metadata: metadata
+      }
+    end
+
+    # Internal: Two dimensional Hash of reducer functions for a given mime type
+    # and asset metadata key.
+    attr_reader :bundle_reducers
+
+    # Public: Register bundle reducer function.
+    #
+    # Examples
+    #
+    #   Sprockets.register_bundle_reducer 'application/javascript', :jshint_errors, [], :+
+    #
+    #   Sprockets.register_bundle_reducer 'text/css', :selector_count, 0 { |total, count|
+    #     total + count
+    #   }
+    #
+    # mime_type - String MIME Type. Use '*/*' applies to all types.
+    # key       - Symbol metadata key
+    # initial   - Initial memo to pass to the reduce funciton (default: nil)
+    # block     - Proc accepting the memo accumulator and current value
+    #
+    # Returns nothing.
+    def register_bundle_reducer(mime_type, key, *args, &block)
+      case args.size
+      when 0
+        reducer = block
+      when 1
+        if block_given?
+          initial = args[0]
+          reducer = block
+        else
+          initial = nil
+          reducer = args[0].to_proc
+        end
+      when 2
+        initial = args[0]
+        reducer = args[1].to_proc
+      else
+        raise ArgumentError, "wrong number of arguments (#{args.size} for 0..2)"
+      end
+
+      mutate_hash_config(:bundle_reducers, mime_type) do |reducers|
+        reducers.merge(key => [initial, reducer])
+      end
+    end
+
+    # Internal: Gather all bundle reducer functions for MIME type.
+    #
+    # mime_type - String MIME type
+    #
+    # Returns an Array of [initial, reducer_proc] pairs.
+    def unwrap_bundle_reducers(mime_type)
+      self.bundle_reducers['*/*'].merge(self.bundle_reducers[mime_type])
+    end
+
+    # Internal: Run bundle reducers on set of Assets producing a reduced
+    # metadata Hash.
+    #
+    # assets - Array of Assets
+    # reducers - Array of [initial, reducer_proc] pairs
+    #
+    # Returns reduced asset metadata Hash.
+    def process_bundle_reducers(assets, reducers)
+      initial = {}
+      reducers.each do |k, (v, _)|
+        initial[k] = v if v
+      end
+
+      assets.reduce(initial) do |h, asset|
+        reducers.each do |k, (_, block)|
+          value = k == :data ? asset.source : asset.metadata[k]
+          h[k]  = h.key?(k) ? block.call(h[k], value) : value
+        end
+        h
+      end
+    end
+
+    private
+      def wrap_processor(klass, proc)
+        if !proc
+          if klass.class == Sprockets::LazyProcessor || klass.respond_to?(:call)
+            klass
+          else
+            LegacyTiltProcessor.new(klass)
+          end
+        elsif proc.respond_to?(:arity) && proc.arity == 2
+          LegacyProcProcessor.new(klass.to_s, proc)
+        else
+          proc
+        end
+      end
+
+      def unwrap_processor(processor)
+        processor.respond_to?(:unwrap) ? processor.unwrap : processor
+      end
   end
 end