carrierwave 1.3.2 → 3.0.2

data/lib/carrierwave/sanitized_file.rb

@@ -1,12 +1,6 @@
 require 'pathname'
 require 'active_support/core_ext/string/multibyte'
-
-begin
-  # Use mime/types/columnar if available, for reduced memory usage
-  require 'mime/types/columnar'
-rescue LoadError
-  require 'mime/types'
-end
+require 'marcel'
 
 module CarrierWave
 
@@ -19,6 +13,7 @@ module CarrierWave
   # It's probably needlessly comprehensive and complex. Help is appreciated.
   #
   class SanitizedFile
+    include CarrierWave::Utilities::FileName
 
     attr_reader :file
 
@@ -32,7 +27,7 @@ module CarrierWave
 
     def initialize(file)
       self.file = file
-      @content = nil
+      @content = @content_type = nil
     end
 
     ##
@@ -44,7 +39,7 @@ module CarrierWave
     #
     def original_filename
       return @original_filename if @original_filename
-      if @file and @file.respond_to?(:original_filename)
+      if @file && @file.respond_to?(:original_filename)
         @file.original_filename
       elsif path
         File.basename(path)
@@ -64,29 +59,6 @@ module CarrierWave
 
     alias_method :identifier, :filename
 
-    ##
-    # Returns the part of the filename before the extension. So if a file is called 'test.jpeg'
-    # this would return 'test'
-    #
-    # === Returns
-    #
-    # [String] the first part of the filename
-    #
-    def basename
-      split_extension(filename)[0] if filename
-    end
-
-    ##
-    # Returns the file extension
-    #
-    # === Returns
-    #
-    # [String] the extension
-    #
-    def extension
-      split_extension(filename)[1] if filename
-    end
-
     ##
     # Returns the file's size.
     #
@@ -137,7 +109,7 @@ module CarrierWave
     # [Boolean] whether the file is valid and has a non-zero size
     #
     def empty?
-      @file.nil? || self.size.nil? || (self.size.zero? && ! self.exists?)
+      @file.nil? || self.size.nil? || (self.size.zero? && !self.exists?)
     end
 
     ##
@@ -156,15 +128,21 @@ module CarrierWave
     #
     # [String] contents of the file
     #
-    def read
+    def read(*args)
       if @content
-        @content
+        if args.empty?
+          @content
+        else
+          length, outbuf = args
+          raise ArgumentError, "outbuf argument not supported since the content is already loaded" if outbuf
+          @content[0, length]
+        end
       elsif is_path?
-        File.open(@file, "rb") {|file| file.read}
+        File.open(@file, "rb") {|file| file.read(*args)}
       else
         @file.try(:rewind)
-        @content = @file.read
-        @file.try(:close) unless @file.try(:closed?)
+        @content = @file.read(*args)
+        @file.try(:close) unless @file.class.ancestors.include?(::StringIO) || @file.try(:closed?)
         @content
       end
     end
@@ -185,13 +163,10 @@ module CarrierWave
       mkdir!(new_path, directory_permissions)
       move!(new_path)
       chmod!(new_path, permissions)
-      if keep_filename
-        self.file = {:tempfile => new_path, :filename => original_filename, :content_type => content_type}
-      else
-        self.file = {:tempfile => new_path, :content_type => content_type}
-      end
+      self.file = {tempfile: new_path, filename: keep_filename ? original_filename : nil, content_type: declared_content_type}
       self
     end
+
     ##
     # Helper to move file to new path.
     #
@@ -223,7 +198,7 @@ module CarrierWave
       mkdir!(new_path, directory_permissions)
       copy!(new_path)
       chmod!(new_path, permissions)
-      self.class.new({:tempfile => new_path, :content_type => content_type})
+      self.class.new({tempfile: new_path, content_type: declared_content_type})
     end
 
     ##
@@ -264,12 +239,11 @@ module CarrierWave
     # [String] the content type of the file
     #
     def content_type
-      return @content_type if @content_type
-      if @file.respond_to?(:content_type) and @file.content_type
-        @content_type = @file.content_type.to_s.chomp
-      elsif path
-        @content_type = ::MIME::Types.type_for(path).first.to_s
-      end
+      @content_type ||=
+        identified_content_type ||
+        declared_content_type ||
+        guessed_safe_content_type ||
+        Marcel::MimeType::BINARY
     end
 
     ##
@@ -300,11 +274,11 @@ module CarrierWave
       if file.is_a?(Hash)
         @file = file["tempfile"] || file[:tempfile]
         @original_filename = file["filename"] || file[:filename]
-        @content_type = file["content_type"] || file[:content_type] || file["type"] || file[:type]
+        @declared_content_type = file["content_type"] || file[:content_type] || file["type"] || file[:type]
       else
         @file = file
         @original_filename = nil
-        @content_type = nil
+        @declared_content_type = nil
       end
     end
 
@@ -321,28 +295,48 @@ module CarrierWave
 
     # Sanitize the filename, to prevent hacking
     def sanitize(name)
+      name = name.scrub
       name = name.tr("\\", "/") # work-around for IE
       name = File.basename(name)
-      name = name.gsub(sanitize_regexp,"_")
+      name = name.gsub(sanitize_regexp, "_")
       name = "_#{name}" if name =~ /\A\.+\z/
-      name = "unnamed" if name.size == 0
-      return name.mb_chars.to_s
+      name = "unnamed" if name.size.zero?
+      name.mb_chars.to_s
     end
 
-    def split_extension(filename)
-      # regular expressions to try for identifying extensions
-      extension_matchers = [
-        /\A(.+)\.(tar\.([glx]?z|bz2))\z/, # matches "something.tar.gz"
-        /\A(.+)\.([^\.]+)\z/ # matches "something.jpg"
-      ]
-
-      extension_matchers.each do |regexp|
-        if filename =~ regexp
-          return $1, $2
+    def declared_content_type
+      @declared_content_type ||
+        if @file.respond_to?(:content_type) && @file.content_type
+          @file.content_type.to_s.chomp
         end
+    end
+
+    # Guess content type from its file extension. Limit what to be returned to prevent spoofing.
+    def guessed_safe_content_type
+      return unless path
+
+      type = Marcel::Magic.by_path(original_filename).to_s
+      type if type.start_with?('text/') || type.start_with?('application/json')
+    end
+
+    def identified_content_type
+      with_io do |io|
+        Marcel::Magic.by_magic(io).try(:type)
       end
-      return filename, "" # In case we weren't able to split the extension
+    rescue Errno::ENOENT
+      nil
     end
 
+    def with_io(&block)
+      if file.is_a?(IO)
+        begin
+          yield file
+        ensure
+          file.try(:rewind)
+        end
+      elsif path
+        File.open(path, &block)
+      end
+    end
   end # SanitizedFile
 end # CarrierWave

data/lib/carrierwave/storage/abstract.rb

@@ -14,7 +14,7 @@ module CarrierWave
       end
 
       def identifier
-        uploader.filename
+        uploader.deduplicated_filename
       end
 
       def store!(file)
@@ -24,19 +24,19 @@ module CarrierWave
       end
 
       def cache!(new_file)
-        raise NotImplementedError.new("Need to implement #cache! if you want to use #{self.class.name} as a cache storage.")
+        raise NotImplementedError, "Need to implement #cache! if you want to use #{self.class.name} as a cache storage."
       end
 
       def retrieve_from_cache!(identifier)
-        raise NotImplementedError.new("Need to implement #retrieve_from_cache! if you want to use #{self.class.name} as a cache storage.")
+        raise NotImplementedError, "Need to implement #retrieve_from_cache! if you want to use #{self.class.name} as a cache storage."
       end
 
       def delete_dir!(path)
-        raise NotImplementedError.new("Need to implement #delete_dir! if you want to use #{self.class.name} as a cache storage.")
+        raise NotImplementedError, "Need to implement #delete_dir! if you want to use #{self.class.name} as a cache storage."
       end
 
       def clean_cache!(seconds)
-        raise NotImplementedError.new("Need to implement #clean_cache! if you want to use #{self.class.name} as a cache storage.")
+        raise NotImplementedError, "Need to implement #clean_cache! if you want to use #{self.class.name} as a cache storage."
       end
     end # Abstract
   end # Storage

data/lib/carrierwave/storage/file.rb

@@ -17,7 +17,7 @@ module CarrierWave
       #
       # By default, store!() uses copy_to(), which operates by copying the file
       # from the cache to the store, then deleting the file from the cache.
-      # If move_to_store() is overriden to return true, then store!() uses move_to(),
+      # If move_to_store() is overridden to return true, then store!() uses move_to(),
       # which simply moves the file from cache to store.  Useful for large files.
       #
       # === Parameters
@@ -109,10 +109,11 @@ module CarrierWave
       end
 
       def clean_cache!(seconds)
-        Dir.glob(::File.expand_path(::File.join(uploader.cache_dir, '*'), CarrierWave.root)).each do |dir|
-          # generate_cache_id returns key formated TIMEINT-PID-COUNTER-RND
-          time = dir.scan(/(\d+)-\d+-\d+-\d+/).first.map(&:to_i)
-          time = Time.at(*time)
+        Dir.glob(::File.expand_path(::File.join(uploader.cache_dir, '*'), uploader.root)).each do |dir|
+          # generate_cache_id returns key formatted TIMEINT-PID(-COUNTER)-RND
+          matched = dir.scan(/(\d+)-\d+-\d+(?:-\d+)?/).first
+          next unless matched
+          time = Time.at(matched[0].to_i)
           if time < (Time.now.utc - seconds)
             FileUtils.rm_rf(dir)
           end

data/lib/carrierwave/storage/fog.rb

@@ -30,7 +30,7 @@ module CarrierWave
     #
     # Google credentials contain the following keys:
     # [:google_storage_access_key_id]
-    # [:google_storage_secrete_access_key]
+    # [:google_storage_secret_access_key]
     #
     #
     # Local credentials contain the following keys:
@@ -60,6 +60,14 @@ module CarrierWave
         def connection_cache
           @connection_cache ||= {}
         end
+
+        def eager_load
+          # see #1198. This will hopefully no longer be necessary in future release of fog
+          fog_credentials = CarrierWave::Uploader::Base.fog_credentials
+          if fog_credentials.present?
+            CarrierWave::Storage::Fog.connection_cache[fog_credentials] ||= ::Fog::Storage.new(fog_credentials)
+          end
+        end
       end
 
       ##
@@ -138,9 +146,10 @@ module CarrierWave
           :key    => uploader.fog_directory,
           :public => uploader.fog_public
         ).files.all(:prefix => uploader.cache_dir).each do |file|
-          # generate_cache_id returns key formated TIMEINT-PID-COUNTER-RND
-          time = file.key.scan(/(\d+)-\d+-\d+-\d+/).first.map { |t| t.to_i }
-          time = Time.at(*time)
+          # generate_cache_id returns key formatted TIMEINT-PID(-COUNTER)-RND
+          matched = file.key.match(/(\d+)-\d+-\d+(?:-\d+)?/)
+          next unless matched
+          time = Time.at(matched[1].to_i)
           file.destroy if time < (Time.now.utc - seconds)
         end
       end
@@ -153,7 +162,10 @@ module CarrierWave
       end
 
       class File
+        DEFAULT_S3_REGION = 'us-east-1'.freeze
+
         include CarrierWave::Utilities::Uri
+        include CarrierWave::Utilities::FileName
 
         ##
         # Current local path to file
@@ -177,7 +189,7 @@ module CarrierWave
 
         ##
         # Return a temporary authenticated url to a private file, if available
-        # Only supported for AWS, Rackspace, Google and AzureRM providers
+        # Only supported for AWS, Rackspace, Google, AzureRM and Aliyun providers
         #
         # === Returns
         #
@@ -186,24 +198,27 @@ module CarrierWave
         # [NilClass] no authenticated url available
         #
         def authenticated_url(options = {})
-          if ['AWS', 'Google', 'Rackspace', 'OpenStack', 'AzureRM'].include?(@uploader.fog_credentials[:provider])
+          if ['AWS', 'Google', 'Rackspace', 'OpenStack', 'AzureRM', 'Aliyun', 'backblaze'].include?(fog_provider)
             # avoid a get by using local references
             local_directory = connection.directories.new(:key => @uploader.fog_directory)
             local_file = local_directory.files.new(:key => path)
-            expire_at = ::Fog::Time.now + @uploader.fog_authenticated_url_expiration
-            case @uploader.fog_credentials[:provider]
-              when 'AWS', 'Google'
-                # Older versions of fog-google do not support options as a parameter
-                if url_options_supported?(local_file)
-                  local_file.url(expire_at, options)
-                else
-                  warn "Options hash not supported in #{local_file.class}. You may need to upgrade your Fog provider."
-                  local_file.url(expire_at)
-                end
-              when 'Rackspace', 'OpenStack'
-                connection.get_object_https_url(@uploader.fog_directory, path, expire_at, options)
+            expire_at = options[:expire_at] || ::Fog::Time.now.since(@uploader.fog_authenticated_url_expiration.to_i)
+            case fog_provider
+            when 'AWS', 'Google'
+              # Older versions of fog-google do not support options as a parameter
+              if url_options_supported?(local_file)
+                local_file.url(expire_at, options)
               else
+                warn "Options hash not supported in #{local_file.class}. You may need to upgrade your Fog provider."
                 local_file.url(expire_at)
+              end
+            when 'Rackspace', 'OpenStack'
+              connection.get_object_https_url(@uploader.fog_directory, path, expire_at, options)
+            when 'Aliyun'
+              expire_at -= Time.now
+              local_file.url(expire_at)
+            else
+              local_file.url(expire_at)
             end
           end
         end
@@ -216,7 +231,7 @@ module CarrierWave
         # [String] value of content-type
         #
         def content_type
-          @content_type || !file.nil? && file.content_type
+          @content_type || file.try(:content_type)
         end
 
         ##
@@ -239,19 +254,9 @@ module CarrierWave
         #
         def delete
           # avoid a get by just using local reference
-          directory.files.new(:key => path).destroy
-        end
-
-        ##
-        # Return extension of file
-        #
-        # === Returns
-        #
-        # [String] extension of file or nil if the file has no extension
-        #
-        def extension
-          path_elements = path.split('.')
-          path_elements.last if path_elements.size > 1
+          directory.files.new(:key => path).destroy.tap do |result|
+            @file = nil if result
+          end
         end
 
         ##
@@ -280,16 +285,16 @@ module CarrierWave
         #
         # [String] contents of file
         def read
-          file_body = file.body
+          file_body = file&.body
 
           return if file_body.nil?
           return file_body unless file_body.is_a?(::File)
 
-          # Fog::Storage::XXX::File#body could return the source file which was upoloaded to the remote server.
-          read_source_file(file_body) if ::File.exist?(file_body.path)
+          # Fog::Storage::XXX::File#body could return the source file which was uploaded to the remote server.
+          return read_source_file if ::File.exist?(file_body.path)
 
           # If the source file doesn't exist, the remote content is read
-          @file = nil # rubocop:disable Gitlab/ModuleWithInstanceVariables
+          @file = nil
           file.body
         end
 
@@ -327,7 +332,7 @@ module CarrierWave
             fog_file = new_file.to_file
             @content_type ||= new_file.content_type
             @file = directory.files.create({
-              :body         => fog_file ? fog_file : new_file.read,
+              :body         => fog_file || new_file.read,
               :content_type => @content_type,
               :key          => path,
               :public       => @uploader.fog_public
@@ -348,7 +353,7 @@ module CarrierWave
         #
         def public_url
           encoded_path = encode_path(path)
-          if host = @uploader.asset_host
+          if (host = @uploader.asset_host)
             if host.respond_to? :call
               "#{host.call(self)}/#{encoded_path}"
             else
@@ -365,15 +370,22 @@ module CarrierWave
                 protocol = @uploader.fog_use_ssl_for_aws ? "https" : "http"
 
                 subdomain_regex = /^(?:[a-z]|\d(?!\d{0,2}(?:\d{1,3}){3}$))(?:[a-z0-9\.]|(?![\-])|\-(?![\.])){1,61}[a-z0-9]$/
-                valid_subdomain = @uploader.fog_directory.to_s =~ subdomain_regex && !(protocol == 'https' && @uploader.fog_directory =~ /\./)
-
-                # if directory is a valid subdomain, use that style for access
-                if valid_subdomain
-                  s3_subdomain = @uploader.fog_aws_accelerate ? "s3-accelerate" : "s3"
-                  "#{protocol}://#{@uploader.fog_directory}.#{s3_subdomain}.amazonaws.com/#{encoded_path}"
-                else
-                  # directory is not a valid subdomain, so use path style for access
-                  "#{protocol}://s3.amazonaws.com/#{@uploader.fog_directory}/#{encoded_path}"
+                # To use the virtual-hosted style, the bucket name needs to be representable as a subdomain
+                use_virtual_hosted_style = @uploader.fog_directory.to_s =~ subdomain_regex && !(protocol == 'https' && @uploader.fog_directory =~ /\./)
+
+                region = @uploader.fog_credentials[:region].to_s
+                regional_host = case region
+                                when DEFAULT_S3_REGION, ''
+                                  's3.amazonaws.com'
+                                else
+                                  "s3.#{region}.amazonaws.com"
+                                end
+
+                if use_virtual_hosted_style
+                  regional_host = 's3-accelerate.amazonaws.com' if @uploader.fog_aws_accelerate
+                  "#{protocol}://#{@uploader.fog_directory}.#{regional_host}/#{encoded_path}"
+                else # directory is not a valid subdomain, so use path style for access
+                  "#{protocol}://#{regional_host}/#{@uploader.fog_directory}/#{encoded_path}"
                 end
               end
             when 'Google'
@@ -387,7 +399,7 @@ module CarrierWave
         end
 
         ##
-        # Return url to file, if avaliable
+        # Return url to file, if available
         #
         # === Returns
         #
@@ -413,7 +425,7 @@ module CarrierWave
         # [NilClass] no file name available
         #
         def filename(options = {})
-          return unless file_url = url(options)
+          return unless (file_url = url(options))
           CGI.unescape(file_url.split('?').first).gsub(/.*\/(.*?$)/, '\1')
         end
 
@@ -429,10 +441,29 @@ module CarrierWave
         # @return [CarrierWave::Storage::Fog::File] the location where the file will be stored.
         #
         def copy_to(new_path)
-          connection.copy_object(@uploader.fog_directory, file.key, @uploader.fog_directory, new_path, acl_header)
+          file.copy(@uploader.fog_directory, new_path, copy_options)
           CarrierWave::Storage::Fog::File.new(@uploader, @base, new_path)
         end
 
+        ##
+        # Return the local file
+        #
+        # === Returns
+        #
+        # [File] The local file as Ruby's File class
+        #   or
+        # [NilClass] When there's no file, or the file is remotely stored
+        #
+        def to_file
+          return nil unless file.body.is_a? ::File
+
+          if file.body.closed?
+            ::File.open(file.body.path) # Reopen if it's already closed
+          else
+            file.body
+          end
+        end
+
       private
 
         ##
@@ -454,12 +485,10 @@ module CarrierWave
         # [Fog::#{provider}::Directory] containing directory
         #
         def directory
-          @directory ||= begin
-            connection.directories.new(
-              :key    => @uploader.fog_directory,
-              :public => @uploader.fog_public
-            )
-          end
+          @directory ||= connection.directories.new(
+            :key    => @uploader.fog_directory,
+            :public => @uploader.fog_public
+          )
         end
 
         ##
@@ -473,9 +502,19 @@ module CarrierWave
           @file ||= directory.files.head(path)
         end
 
+        def copy_options
+          options = {}
+          options.merge!(acl_header) if acl_header.present?
+          options[fog_provider == "Google" ? :content_type : 'Content-Type'] ||= content_type if content_type
+          options.merge(@uploader.fog_attributes)
+        end
+
         def acl_header
-          if fog_provider == 'AWS'
+          case fog_provider
+          when 'AWS'
             { 'x-amz-acl' => @uploader.fog_public ? 'public-read' : 'private' }
+          when "Google"
+            @uploader.fog_public ? { destination_predefined_acl: "publicRead" } : {}
           else
             {}
           end
@@ -485,14 +524,14 @@ module CarrierWave
           @uploader.fog_credentials[:provider].to_s
         end
 
-        def read_source_file(file_body)
-          return unless ::File.exist?(file_body.path)
+        def read_source_file
+          source_file = to_file
+          return unless source_file
 
           begin
-            file_body = ::File.open(file_body.path) if file_body.closed? # Reopen if it's already closed
-            file_body.read
+            source_file.read
           ensure
-            file_body.close
+            source_file.close
           end
         end
 

data/lib/carrierwave/storage.rb

@@ -1,2 +1,3 @@
 require "carrierwave/storage/abstract"
 require "carrierwave/storage/file"
+require "carrierwave/storage/fog"

data/lib/carrierwave/test/matchers.rb

@@ -45,11 +45,11 @@ module CarrierWave
         def matches?(actual)
           @actual = actual
           # Satisfy expectation here. Return false or raise an error if it's not met.
-          (File.stat(@actual.path).mode & 0777) == @expected
+          (File.stat(@actual.path).mode & 0o777) == @expected
         end
 
         def failure_message
-          "expected #{@actual.current_path.inspect} to have permissions #{@expected.to_s(8)}, but they were #{(File.stat(@actual.path).mode & 0777).to_s(8)}"
+          "expected #{@actual.current_path.inspect} to have permissions #{@expected.to_s(8)}, but they were #{(File.stat(@actual.path).mode & 0o777).to_s(8)}"
         end
 
         def failure_message_when_negated
@@ -76,11 +76,11 @@ module CarrierWave
         def matches?(actual)
           @actual = actual
           # Satisfy expectation here. Return false or raise an error if it's not met.
-          (File.stat(File.dirname @actual.path).mode & 0777) == @expected
+          (File.stat(File.dirname(@actual.path)).mode & 0o777) == @expected
         end
 
         def failure_message
-          "expected #{File.dirname @actual.current_path.inspect} to have permissions #{@expected.to_s(8)}, but they were #{(File.stat(@actual.path).mode & 0777).to_s(8)}"
+          "expected #{File.dirname @actual.current_path.inspect} to have permissions #{@expected.to_s(8)}, but they were #{(File.stat(@actual.path).mode & 0o777).to_s(8)}"
         end
 
         def failure_message_when_negated
@@ -341,9 +341,11 @@ module CarrierWave
               begin
                 require 'rmagick'
               rescue LoadError
-                require 'RMagick'
-              rescue LoadError
-                puts "WARNING: Failed to require rmagick, image processing may fail!"
+                begin
+                  require 'RMagick'
+                rescue LoadError
+                  puts "WARNING: Failed to require rmagick, image processing may fail!"
+                end
               end
             end
             MagickWrapper.new(filename)
@@ -353,6 +355,7 @@ module CarrierWave
 
       class MagickWrapper # :nodoc:
         attr_reader :image
+
         def width
           image.columns
         end
@@ -372,6 +375,7 @@ module CarrierWave
 
       class MiniMagickWrapper # :nodoc:
         attr_reader :image
+
         def width
           image[:width]
         end