carrierwave 1.2.1 → 2.1.1

data/lib/carrierwave/sanitized_file.rb

@@ -1,12 +1,7 @@
 require 'pathname'
 require 'active_support/core_ext/string/multibyte'
-
-begin
-  # Use mime/types/columnar if available, for reduced memory usage
-  require 'mime/types/columnar'
-rescue LoadError
-  require 'mime/types'
-end
+require 'mini_mime'
+require 'mimemagic'
 
 module CarrierWave
 
@@ -20,7 +15,7 @@ module CarrierWave
   #
   class SanitizedFile
 
-    attr_accessor :file
+    attr_reader :file
 
     class << self
       attr_writer :sanitize_regexp
@@ -32,6 +27,7 @@ module CarrierWave
 
     def initialize(file)
       self.file = file
+      @content = nil
     end
 
     ##
@@ -113,12 +109,11 @@ module CarrierWave
     # [String, nil] the path where the file is located.
     #
     def path
-      unless @file.blank?
-        if is_path?
-          File.expand_path(@file)
-        elsif @file.respond_to?(:path) and not @file.path.blank?
-          File.expand_path(@file.path)
-        end
+      return if @file.blank?
+      if is_path?
+        File.expand_path(@file)
+      elsif @file.respond_to?(:path) && !@file.path.blank?
+        File.expand_path(@file.path)
       end
     end
 
@@ -264,12 +259,10 @@ module CarrierWave
     # [String] the content type of the file
     #
     def content_type
-      return @content_type if @content_type
-      if @file.respond_to?(:content_type) and @file.content_type
-        @content_type = @file.content_type.to_s.chomp
-      elsif path
-        @content_type = ::MIME::Types.type_for(path).first.to_s
-      end
+      @content_type ||=
+        existing_content_type ||
+        mime_magic_content_type ||
+        mini_mime_content_type
     end
 
     ##
@@ -312,7 +305,7 @@ module CarrierWave
     def mkdir!(path, directory_permissions)
       options = {}
       options[:mode] = directory_permissions if directory_permissions
-      FileUtils.mkdir_p(File.dirname(path), options) unless File.exist?(File.dirname(path))
+      FileUtils.mkdir_p(File.dirname(path), **options) unless File.exist?(File.dirname(path))
     end
 
     def chmod!(path, permissions)
@@ -325,10 +318,32 @@ module CarrierWave
       name = File.basename(name)
       name = name.gsub(sanitize_regexp,"_")
       name = "_#{name}" if name =~ /\A\.+\z/
-      name = "unnamed" if name.size == 0
+      name = "unnamed" if name.size.zero?
       return name.mb_chars.to_s
     end
 
+    def existing_content_type
+      if @file.respond_to?(:content_type) && @file.content_type
+        @file.content_type.to_s.chomp
+      end
+    end
+
+    def mime_magic_content_type
+      if path
+        File.open(path) do |file|
+          MimeMagic.by_magic(file).try(:type) || 'invalid/invalid'
+        end
+      end
+    rescue Errno::ENOENT
+      nil
+    end
+
+    def mini_mime_content_type
+      return unless path
+      mime_type = ::MiniMime.lookup_by_filename(path)
+      @content_type = (mime_type && mime_type.content_type).to_s
+    end
+
     def split_extension(filename)
       # regular expressions to try for identifying extensions
       extension_matchers = [

data/lib/carrierwave/storage/file.rb

@@ -7,6 +7,10 @@ module CarrierWave
     # pretty much it.
     #
     class File < Abstract
+      def initialize(*)
+        super
+        @cache_called = nil
+      end
 
       ##
       # Move the file to the uploader's store path.
@@ -62,7 +66,7 @@ module CarrierWave
       #
       def cache!(new_file)
         new_file.move_to(::File.expand_path(uploader.cache_path, uploader.root), uploader.permissions, uploader.directory_permissions, true)
-      rescue Errno::EMLINK => e
+      rescue Errno::EMLINK, Errno::ENOSPC => e
         raise(e) if @cache_called
         @cache_called = true
 
@@ -106,8 +110,8 @@ module CarrierWave
 
       def clean_cache!(seconds)
         Dir.glob(::File.expand_path(::File.join(uploader.cache_dir, '*'), CarrierWave.root)).each do |dir|
-          # generate_cache_id returns key formated TIMEINT-PID-COUNTER-RND
-          time = dir.scan(/(\d+)-\d+-\d+-\d+/).first.map(&:to_i)
+          # generate_cache_id returns key formated TIMEINT-PID(-COUNTER)-RND
+          time = dir.scan(/(\d+)-\d+-\d+(?:-\d+)?/).first.map(&:to_i)
           time = Time.at(*time)
           if time < (Time.now.utc - seconds)
             FileUtils.rm_rf(dir)

data/lib/carrierwave/storage/fog.rb

@@ -60,6 +60,14 @@ module CarrierWave
         def connection_cache
           @connection_cache ||= {}
         end
+
+        def eager_load
+          # see #1198. This will hopefully no longer be necessary in future release of fog
+          fog_credentials = CarrierWave::Uploader::Base.fog_credentials
+          if fog_credentials.present?
+            CarrierWave::Storage::Fog.connection_cache[fog_credentials] ||= ::Fog::Storage.new(fog_credentials)
+          end
+        end
       end
 
       ##
@@ -138,8 +146,8 @@ module CarrierWave
           :key    => uploader.fog_directory,
           :public => uploader.fog_public
         ).files.all(:prefix => uploader.cache_dir).each do |file|
-          # generate_cache_id returns key formated TIMEINT-PID-COUNTER-RND
-          time = file.key.scan(/(\d+)-\d+-\d+-\d+/).first.map { |t| t.to_i }
+          # generate_cache_id returns key formated TIMEINT-PID(-COUNTER)-RND
+          time = file.key.scan(/(\d+)-\d+-\d+(?:-\d+)?/).first.map { |t| t.to_i }
           time = Time.at(*time)
           file.destroy if time < (Time.now.utc - seconds)
         end
@@ -153,6 +161,8 @@ module CarrierWave
       end
 
       class File
+        DEFAULT_S3_REGION = 'us-east-1'
+
         include CarrierWave::Utilities::Uri
 
         ##
@@ -177,7 +187,7 @@ module CarrierWave
 
         ##
         # Return a temporary authenticated url to a private file, if available
-        # Only supported for AWS, Rackspace and Google providers
+        # Only supported for AWS, Rackspace, Google, AzureRM and Aliyun providers
         #
         # === Returns
         #
@@ -186,18 +196,25 @@ module CarrierWave
         # [NilClass] no authenticated url available
         #
         def authenticated_url(options = {})
-          if ['AWS', 'Google', 'Rackspace', 'OpenStack'].include?(@uploader.fog_credentials[:provider])
+          if ['AWS', 'Google', 'Rackspace', 'OpenStack', 'AzureRM', 'Aliyun', 'backblaze'].include?(@uploader.fog_credentials[:provider])
             # avoid a get by using local references
             local_directory = connection.directories.new(:key => @uploader.fog_directory)
             local_file = local_directory.files.new(:key => path)
-            expire_at = ::Fog::Time.now + @uploader.fog_authenticated_url_expiration
+            expire_at = options[:expire_at] || ::Fog::Time.now + @uploader.fog_authenticated_url_expiration
             case @uploader.fog_credentials[:provider]
-              when 'AWS'
-                local_file.url(expire_at, options)
-              when 'Rackspace'
+              when 'AWS', 'Google'
+                # Older versions of fog-google do not support options as a parameter
+                if url_options_supported?(local_file)
+                  local_file.url(expire_at, options)
+                else
+                  warn "Options hash not supported in #{local_file.class}. You may need to upgrade your Fog provider."
+                  local_file.url(expire_at)
+                end
+              when 'Rackspace', 'OpenStack'
                 connection.get_object_https_url(@uploader.fog_directory, path, expire_at, options)
-              when 'OpenStack'
-                connection.get_object_https_url(@uploader.fog_directory, path, expire_at)
+              when 'Aliyun'
+                expire_at = expire_at - Time.now
+                local_file.url(expire_at)
               else
                 local_file.url(expire_at)
             end
@@ -212,7 +229,7 @@ module CarrierWave
         # [String] value of content-type
         #
         def content_type
-          @content_type || !file.nil? && file.content_type
+          @content_type || file.try(:content_type)
         end
 
         ##
@@ -235,7 +252,9 @@ module CarrierWave
         #
         def delete
           # avoid a get by just using local reference
-          directory.files.new(:key => path).destroy
+          directory.files.new(:key => path).destroy.tap do |result|
+            @file = nil if result
+          end
         end
 
         ##
@@ -266,7 +285,7 @@ module CarrierWave
         end
 
         def initialize(uploader, base, path)
-          @uploader, @base, @path = uploader, base, path
+          @uploader, @base, @path, @content_type = uploader, base, path, nil
         end
 
         ##
@@ -276,6 +295,16 @@ module CarrierWave
         #
         # [String] contents of file
         def read
+          file_body = file.body
+
+          return if file_body.nil?
+          return file_body unless file_body.is_a?(::File)
+
+          # Fog::Storage::XXX::File#body could return the source file which was upoloaded to the remote server.
+          read_source_file(file_body) if ::File.exist?(file_body.path)
+
+          # If the source file doesn't exist, the remote content is read
+          @file = nil # rubocop:disable Gitlab/ModuleWithInstanceVariables
           file.body
         end
 
@@ -313,7 +342,7 @@ module CarrierWave
             fog_file = new_file.to_file
             @content_type ||= new_file.content_type
             @file = directory.files.create({
-              :body         => (fog_file ? fog_file : new_file).read,
+              :body         => fog_file ? fog_file : new_file.read,
               :content_type => @content_type,
               :key          => path,
               :public       => @uploader.fog_public
@@ -342,20 +371,30 @@ module CarrierWave
             end
           else
             # AWS/Google optimized for speed over correctness
-            case @uploader.fog_credentials[:provider].to_s
+            case fog_provider
             when 'AWS'
               # check if some endpoint is set in fog_credentials
               if @uploader.fog_credentials.has_key?(:endpoint)
                 "#{@uploader.fog_credentials[:endpoint]}/#{@uploader.fog_directory}/#{encoded_path}"
               else
                 protocol = @uploader.fog_use_ssl_for_aws ? "https" : "http"
+
+                subdomain_regex = /^(?:[a-z]|\d(?!\d{0,2}(?:\d{1,3}){3}$))(?:[a-z0-9\.]|(?![\-])|\-(?![\.])){1,61}[a-z0-9]$/
+                valid_subdomain = @uploader.fog_directory.to_s =~ subdomain_regex && !(protocol == 'https' && @uploader.fog_directory =~ /\./)
+
                 # if directory is a valid subdomain, use that style for access
-                if @uploader.fog_directory.to_s =~ /^(?:[a-z]|\d(?!\d{0,2}(?:\d{1,3}){3}$))(?:[a-z0-9\.]|(?![\-])|\-(?![\.])){1,61}[a-z0-9]$/
+                if valid_subdomain
                   s3_subdomain = @uploader.fog_aws_accelerate ? "s3-accelerate" : "s3"
                   "#{protocol}://#{@uploader.fog_directory}.#{s3_subdomain}.amazonaws.com/#{encoded_path}"
-                else
-                  # directory is not a valid subdomain, so use path style for access
-                  "#{protocol}://s3.amazonaws.com/#{@uploader.fog_directory}/#{encoded_path}"
+                else # directory is not a valid subdomain, so use path style for access
+                  region = @uploader.fog_credentials[:region].to_s
+                  host   = case region
+                           when DEFAULT_S3_REGION, ''
+                             's3.amazonaws.com'
+                           else
+                             "s3.#{region}.amazonaws.com"
+                           end
+                  "#{protocol}://#{host}/#{@uploader.fog_directory}/#{encoded_path}"
                 end
               end
             when 'Google'
@@ -456,7 +495,31 @@ module CarrierWave
         end
 
         def acl_header
-          {'x-amz-acl' => @uploader.fog_public ? 'public-read' : 'private'}
+          if fog_provider == 'AWS'
+            { 'x-amz-acl' => @uploader.fog_public ? 'public-read' : 'private' }
+          else
+            {}
+          end
+        end
+
+        def fog_provider
+          @uploader.fog_credentials[:provider].to_s
+        end
+
+        def read_source_file(file_body)
+          return unless ::File.exist?(file_body.path)
+
+          begin
+            file_body = ::File.open(file_body.path) if file_body.closed? # Reopen if it's already closed
+            file_body.read
+          ensure
+            file_body.close
+          end
+        end
+
+        def url_options_supported?(local_file)
+          parameters = local_file.method(:url).parameters
+          parameters.count == 2 && parameters[1].include?(:options)
         end
       end
 

data/lib/carrierwave/storage.rb

@@ -1,2 +1,3 @@
 require "carrierwave/storage/abstract"
 require "carrierwave/storage/file"
+require "carrierwave/storage/fog"

data/lib/carrierwave/uploader/cache.rb

@@ -1,3 +1,5 @@
+require 'securerandom'
+
 module CarrierWave
 
   class FormNotMultipart < UploadError
@@ -23,9 +25,9 @@ module CarrierWave
   #
   def self.generate_cache_id
     [Time.now.utc.to_i,
-      Process.pid,
-      '%04d' % (CarrierWave::CacheCounter.increment % 1000),
-      '%04d' % rand(9999)
+      SecureRandom.random_number(1_000_000_000_000_000),
+      '%04d' % (CarrierWave::CacheCounter.increment % 10_000),
+      '%04d' % SecureRandom.random_number(10_000)
     ].map(&:to_s).join('-')
   end
 
@@ -36,6 +38,16 @@ module CarrierWave
       include CarrierWave::Uploader::Callbacks
       include CarrierWave::Uploader::Configuration
 
+      included do
+        prepend Module.new {
+          def initialize(*)
+            super
+            @staged = false
+          end
+        }
+        attr_accessor :staged
+      end
+
       module ClassMethods
 
         ##
@@ -52,7 +64,7 @@ module CarrierWave
         # It's recommended that you keep cache files in one place only.
         #
         def clean_cached_files!(seconds=60*60*24)
-          cache_storage.new(CarrierWave::Uploader::Base.new).clean_cache!(seconds)
+          (cache_storage || storage).new(CarrierWave::Uploader::Base.new).clean_cache!(seconds)
         end
       end
 
@@ -78,14 +90,8 @@ module CarrierWave
       end
 
       def sanitized_file
-        _content = file.read
-        if _content.is_a?(File) # could be if storage is Fog
-          sanitized = CarrierWave::Storage::Fog.new(self).retrieve!(File.basename(_content.path))
-        else
-          sanitized = SanitizedFile.new :tempfile => StringIO.new(_content),
-            :filename => File.basename(path), :content_type => file.content_type
-        end
-        sanitized
+        ActiveSupport::Deprecation.warn('#sanitized_file is deprecated, use #file instead.')
+        file
       end
 
       ##
@@ -96,7 +102,7 @@ module CarrierWave
       # [String] a cache name, in the format TIMEINT-PID-COUNTER-RND/filename.txt
       #
       def cache_name
-        File.join(cache_id, full_original_filename) if cache_id and original_filename
+        File.join(cache_id, full_original_filename) if cache_id && original_filename
       end
 
       ##
@@ -115,7 +121,7 @@ module CarrierWave
       #
       # [CarrierWave::FormNotMultipart] if the assigned parameter is a string
       #
-      def cache!(new_file = sanitized_file)
+      def cache!(new_file = file)
         new_file = CarrierWave::SanitizedFile.new(new_file)
         return if new_file.empty?
 
@@ -123,6 +129,7 @@ module CarrierWave
 
         self.cache_id = CarrierWave.generate_cache_id unless cache_id
 
+        @staged = true
         @filename = new_file.filename
         self.original_filename = new_file.filename
 
@@ -156,6 +163,7 @@ module CarrierWave
       def retrieve_from_cache!(cache_name)
         with_callbacks(:retrieve_from_cache, cache_name) do
           self.cache_id, self.original_filename = cache_name.to_s.split('/', 2)
+          @staged = true
           @filename = original_filename
           @file = cache_storage.retrieve_from_cache!(full_filename(original_filename))
         end
@@ -200,7 +208,7 @@ module CarrierWave
       end
 
       def cache_storage
-        @cache_storage ||= self.class.cache_storage.new(self)
+        @cache_storage ||= (self.class.cache_storage || self.class.storage).new(self)
       end
     end # Cache
   end # Uploader

data/lib/carrierwave/uploader/configuration.rb

@@ -1,3 +1,5 @@
+require 'carrierwave/downloader/base'
+
 module CarrierWave
 
   module Uploader
@@ -21,9 +23,10 @@ module CarrierWave
         add_config :move_to_cache
         add_config :move_to_store
         add_config :remove_previously_stored_files_after_update
+        add_config :downloader
 
         # fog
-        add_config :fog_provider
+        add_deprecated_config :fog_provider
         add_config :fog_attributes
         add_config :fog_credentials
         add_config :fog_directory
@@ -107,8 +110,8 @@ module CarrierWave
         #     cache_storage CarrierWave::Storage::File
         #     cache_storage MyCustomStorageEngine
         #
-        def cache_storage(storage = nil)
-          if storage
+        def cache_storage(storage = false)
+          unless storage == false
             self._cache_storage = storage.is_a?(Symbol) ? eval(storage_engines[storage]) : storage
           end
           _cache_storage
@@ -117,16 +120,10 @@ module CarrierWave
 
         def add_config(name)
           class_eval <<-RUBY, __FILE__, __LINE__ + 1
-            def self.eager_load_fog(fog_credentials)
-              # see #1198. This will hopefully no longer be necessary after fog 2.0
-              require self.fog_provider
-              require 'carrierwave/storage/fog'
-              Fog::Storage.new(fog_credentials) if fog_credentials.present?
-            end
+            @#{name} = nil
 
             def self.#{name}(value=nil)
               @#{name} = value if value
-              eager_load_fog(value) if value && '#{name}' == 'fog_credentials'
               return @#{name} if self.object_id == #{self.object_id} || defined?(@#{name})
               name = superclass.#{name}
               return nil if name.nil? && !instance_variable_defined?(:@#{name})
@@ -134,12 +131,10 @@ module CarrierWave
             end
 
             def self.#{name}=(value)
-              eager_load_fog(value) if '#{name}' == 'fog_credentials' && value.present?
               @#{name} = value
             end
 
             def #{name}=(value)
-              self.class.eager_load_fog(value) if '#{name}' == 'fog_credentials' && value.present?
               @#{name} = value
             end
 
@@ -155,6 +150,26 @@ module CarrierWave
           RUBY
         end
 
+        def add_deprecated_config(name)
+          class_eval <<-RUBY, __FILE__, __LINE__ + 1
+            def self.#{name}(value=nil)
+              ActiveSupport::Deprecation.warn "##{name} is deprecated and has no effect"
+            end
+
+            def self.#{name}=(value)
+              ActiveSupport::Deprecation.warn "##{name} is deprecated and has no effect"
+            end
+
+            def #{name}=(value)
+              ActiveSupport::Deprecation.warn "##{name} is deprecated and has no effect"
+            end
+
+            def #{name}
+              ActiveSupport::Deprecation.warn "##{name} is deprecated and has no effect"
+            end
+          RUBY
+        end
+
         def configure
           yield self
         end
@@ -171,8 +186,7 @@ module CarrierWave
               :fog  => "CarrierWave::Storage::Fog"
             }
             config.storage = :file
-            config.cache_storage = :file
-            config.fog_provider = 'fog'
+            config.cache_storage = nil
             config.fog_attributes = {}
             config.fog_credentials = {}
             config.fog_public = true
@@ -185,6 +199,7 @@ module CarrierWave
             config.move_to_cache = false
             config.move_to_store = false
             config.remove_previously_stored_files_after_update = true
+            config.downloader = CarrierWave::Downloader::Base
             config.ignore_integrity_errors = true
             config.ignore_processing_errors = true
             config.ignore_download_errors = true

data/lib/carrierwave/uploader/content_type_whitelist.rb

@@ -35,7 +35,7 @@ module CarrierWave
       def check_content_type_whitelist!(new_file)
         content_type = new_file.content_type
         if content_type_whitelist && !whitelisted_content_type?(content_type)
-          raise CarrierWave::IntegrityError, I18n.translate(:"errors.messages.content_type_whitelist_error", content_type: content_type)
+          raise CarrierWave::IntegrityError, I18n.translate(:"errors.messages.content_type_whitelist_error", content_type: content_type, allowed_types: Array(content_type_whitelist).join(", "))
         end
       end
 

data/lib/carrierwave/uploader/download.rb

@@ -1,5 +1,3 @@
-require 'open-uri'
-
 module CarrierWave
   module Uploader
     module Download
@@ -9,63 +7,8 @@ module CarrierWave
       include CarrierWave::Uploader::Configuration
       include CarrierWave::Uploader::Cache
 
-      class RemoteFile
-        def initialize(uri, remote_headers = {})
-          @uri = uri
-          @remote_headers = remote_headers
-        end
-
-        def original_filename
-          filename = filename_from_header || filename_from_uri
-          mime_type = MIME::Types[file.content_type].first
-          unless File.extname(filename).present? || mime_type.blank?
-            filename = "#{filename}.#{mime_type.extensions.first}"
-          end
-          filename
-        end
-
-        def respond_to?(*args)
-          super or file.respond_to?(*args)
-        end
-
-        def http?
-          @uri.scheme =~ /^https?$/
-        end
-
-      private
-
-        def file
-          if @file.blank?
-            headers = @remote_headers.
-              reverse_merge('User-Agent' => "CarrierWave/#{CarrierWave::VERSION}")
-
-            @file = Kernel.open(@uri.to_s, headers)
-            @file = @file.is_a?(String) ? StringIO.new(@file) : @file
-          end
-          @file
-
-        rescue StandardError => e
-          raise CarrierWave::DownloadError, "could not download file: #{e.message}"
-        end
-
-        def filename_from_header
-          if file.meta.include? 'content-disposition'
-            match = file.meta['content-disposition'].match(/filename="?([^"]+)/)
-            return match[1] unless match.nil? || match[1].empty?
-          end
-        end
-
-        def filename_from_uri
-          URI.decode(File.basename(file.base_uri.path))
-        end
-
-        def method_missing(*args, &block)
-          file.send(*args, &block)
-        end
-      end
-
       ##
-      # Caches the file by downloading it from the given URL.
+      # Caches the file by downloading it from the given URL, using downloader.
       #
       # === Parameters
       #
@@ -73,29 +16,9 @@ module CarrierWave
       # [remote_headers (Hash)] Request headers
       #
       def download!(uri, remote_headers = {})
-        processed_uri = process_uri(uri)
-        file = RemoteFile.new(processed_uri, remote_headers)
-        raise CarrierWave::DownloadError, "trying to download a file which is not served over HTTP" unless file.http?
+        file = downloader.new(self).download(uri, remote_headers)
         cache!(file)
       end
-
-      ##
-      # Processes the given URL by parsing and escaping it. Public to allow overriding.
-      #
-      # === Parameters
-      #
-      # [url (String)] The URL where the remote file is stored
-      #
-      def process_uri(uri)
-        URI.parse(uri)
-      rescue URI::InvalidURIError
-        uri_parts = uri.split('?')
-        # regexp from Ruby's URI::Parser#regexp[:UNSAFE], with [] specifically removed
-        encoded_uri = URI.encode(uri_parts.shift, /[^\-_.!~*'()a-zA-Z\d;\/?:@&=+$,]/)
-        encoded_uri << '?' << URI.encode(uri_parts.join('?')) if uri_parts.any?
-        URI.parse(encoded_uri) rescue raise CarrierWave::DownloadError, "couldn't parse URL"
-      end
-
     end # Download
   end # Uploader
 end # CarrierWave

data/lib/carrierwave/uploader/mountable.rb

@@ -33,6 +33,12 @@ module CarrierWave
         @mounted_as = mounted_as
       end
 
+      ##
+      # Returns array index of given uploader within currently mounted uploaders
+      #
+      def index
+        model.__send__(:_mounter, mounted_as).uploaders.index(self)
+      end
     end # Mountable
   end # Uploader
 end # CarrierWave

data/lib/carrierwave/uploader/proxy.rb

@@ -23,14 +23,14 @@ module CarrierWave
       alias_method :path, :current_path
 
       ##
-      # Returns a string that uniquely identifies the last stored file
+      # Returns a string that uniquely identifies the retrieved or last stored file
       #
       # === Returns
       #
       # [String] uniquely identifies a file
       #
       def identifier
-        storage.try(:identifier)
+        @identifier || storage.try(:identifier)
       end
 
       ##