carrierwave 3.1.4 → 4.0.0.rc

data/lib/carrierwave/uploader/configuration.rb

@@ -28,6 +28,7 @@ module CarrierWave
 
         # fog
         add_deprecated_config :fog_provider
+        add_config :fog_acl
         add_config :fog_attributes
         add_config :fog_credentials
         add_config :fog_directory
@@ -45,10 +46,12 @@ module CarrierWave
         add_config :validate_processing
         add_config :validate_download
         add_config :mount_on
+        add_config :metadata_column
         add_config :cache_only
         add_config :download_retry_count
         add_config :download_retry_wait_time
         add_config :skip_ssrf_protection
+        add_config :ssrf_filter_options
 
         # set default values
         reset_config
@@ -220,6 +223,7 @@ module CarrierWave
             config.download_retry_count = 0
             config.download_retry_wait_time = 5
             config.skip_ssrf_protection = false
+            config.ssrf_filter_options = {}
           end
         end
       end

data/lib/carrierwave/uploader/direct_upload.rb

@@ -0,0 +1,51 @@
+module CarrierWave
+  module Uploader
+    module DirectUpload
+      extend ActiveSupport::Concern
+
+      include CarrierWave::Uploader::Callbacks
+      include CarrierWave::Uploader::Configuration
+      include CarrierWave::Uploader::Cache
+
+      ##
+      # Returns what a client needs to put a file into the cache itself, without the
+      # bytes passing through the application. Assign the returned #cache_name to
+      # #{column}_cache once the client is done, as with any cached file.
+      #
+      # === Parameters
+      #
+      # [filename (String)] the name the file is to be cached under
+      # [expires_in (Integer)] seconds the upload stays possible, defaults to
+      #                        fog_authenticated_url_expiration
+      # [content_type (String)] the type the client has to send, if it is to be fixed
+      #
+      # === Returns
+      #
+      # [CarrierWave::DirectUpload] where and how to upload, and the name to hand back
+      #
+      def direct_upload(filename:, expires_in: nil, content_type: nil)
+        raise CarrierWave::InvalidParameter, "invalid filename" if filename.blank?
+
+        filename = CarrierWave::SanitizedFile.new(filename).filename
+
+        upload = nil
+        with_callbacks(:direct_upload, filename) do
+          self.cache_id = CarrierWave.generate_cache_id
+          self.original_filename = filename
+          headers = content_type ? { 'Content-Type' => content_type } : {}
+
+          upload = ::CarrierWave::DirectUpload.new(
+            url: cache_storage.direct_upload_url(
+              cache_path,
+              expires_in: expires_in || fog_authenticated_url_expiration,
+              headers: headers.dup # the storage may add its own
+            ),
+            headers: headers,
+            cache_name: cache_name
+          )
+        end
+        upload
+      end
+    end # DirectUpload
+  end # Uploader
+end # CarrierWave

data/lib/carrierwave/uploader/metadata.rb

@@ -0,0 +1,100 @@
+require "active_support/core_ext/hash/keys"
+
+module CarrierWave
+  module Uploader
+    module Metadata
+      extend ActiveSupport::Concern
+
+      include CarrierWave::Uploader::Versions
+
+      ##
+      # Facts about the file which were determined when it was stored, as recorded in
+      # the column given to the +metadata+ option of the mount. They are the answer to
+      # questions the storage cannot answer, like whether a conditional version was
+      # created, and they save asking it the ones it can.
+      #
+      # Returns an empty Hash when nothing was recorded, in which case the facts are
+      # derived from the file and the uploader definition as they have always been.
+      #
+      # === Returns
+      #
+      # [Hash] the recorded facts
+      #
+      def metadata
+        return @metadata || {} unless parent_version
+
+        parent_version.metadata.dig('versions', self.class.version_names.last.to_s) || {}
+      end
+
+      def metadata=(metadata)
+        @metadata = metadata&.deep_stringify_keys
+      end
+
+      ##
+      # Collects the facts to record about the file being stored. Override to record
+      # your own, merging them into the result of +super+.
+      #
+      # === Returns
+      #
+      # [Hash] the facts to record
+      #
+      def build_metadata
+        {
+          'filename' => full_filename(identifier),
+          'size' => size,
+          'content_type' => content_type,
+          'versions' => active_versions.map { |name, version| [name.to_s, version.build_metadata] }.to_h
+        }
+      end
+
+      ##
+      # === Returns
+      #
+      # [Boolean] Whether what is stored gets recorded, which the mount decides
+      #
+      def metadata_recorded?
+        model.class.respond_to?(:uploader_option) &&
+          model.class.uploaders.has_key?(mounted_as) &&
+          !!model.class.uploader_option(mounted_as, :metadata_column)
+      end
+
+      def size
+        metadata['size'] || super
+      end
+
+      def content_type
+        metadata['content_type'] || super
+      end
+
+      ##
+      # The conditions decide which versions to create, so which ones were created is
+      # settled once the file is stored. Consult the record instead of asking them
+      # again, which is not only costly but can give a different answer.
+      #
+      def version_active?(name)
+        recorded_versions = metadata['versions']
+        return super unless recorded_versions
+
+        versions.has_key?(name.to_sym) && recorded_versions.has_key?(name.to_s)
+      end
+
+      def cache!(*)
+        @metadata = nil
+        super
+      end
+
+      def recreate_versions!(*)
+        @metadata = nil
+        super
+      end
+
+    private
+
+      def full_filename(for_file)
+        return super unless for_file == identifier
+
+        metadata['filename'] || super
+      end
+    end # Metadata
+  end # Uploader
+end # CarrierWave

data/lib/carrierwave/uploader/processing.rb

@@ -10,6 +10,7 @@ module CarrierWave
         self.processors = []
 
         before :cache, :process!
+        before :direct_upload, :refuse_direct_upload_of_processed!
       end
 
       module ClassMethods
@@ -67,13 +68,17 @@ module CarrierWave
           new_processors.each do |processor, processor_args|
             self.processors += [[processor, processor_args, condition, condition_type]]
 
-            if processor == :convert
-              # Treat :convert specially, since it should trigger the file extension change
-              force_extension processor_args
-              if condition
-                warn "Use of 'process convert: format' with conditionals has an issue and doesn't work correctly. See https://github.com/carrierwaveuploader/carrierwave/issues/2723 for details. "
-              end
+            next unless processor == :convert
+
+            # Treat :convert specially, since it should trigger the file extension change.
+            # That has to be worked out again when the file is retrieved, where whether
+            # the condition held cannot be known, so a conditional one is refused.
+            if condition
+              raise ArgumentError, "`process convert:` cannot be given `:if` or `:unless`. The file extension follows the conversion, and it has to be worked out again when the file is retrieved, where whether the condition held is not known. " \
+                                   "Convert unconditionally, or put the conversion in a version of its own, or convert in a processor of your own and have #filename answer the extension the file ended up with."
             end
+
+            force_extension processor_args
           end
         end
       end # ClassMethods
@@ -116,6 +121,13 @@ module CarrierWave
 
     private
 
+      # Processing needs the file, which a direct upload never hands to the application
+      def refuse_direct_upload_of_processed!(filename)
+        return if !enable_processing || self.class.processors.empty?
+
+        raise "#{filename.inspect} can't be uploaded directly, as the processing of #{self.class} would be skipped"
+      end
+
       def forcing_extension(filename)
         if force_extension && filename
           Pathname.new(filename).sub_ext(".#{force_extension.to_s.delete_prefix('.')}").to_s

data/lib/carrierwave/uploader/proxy.rb

@@ -5,10 +5,26 @@ module CarrierWave
       ##
       # === Returns
       #
-      # [Boolean] Whether the uploaded file is blank
+      # [Boolean] Whether a file is assigned. Use #exists? to ask the storage whether
+      #   the file is actually there.
       #
       def blank?
-        file.blank?
+        return true unless file
+        # A cache name can point at a cache which is gone, so it is verified. It is
+        # local and thus cheap, unlike asking a remote store.
+        return file.empty? if cached?
+
+        false
+      end
+
+      ##
+      # === Returns
+      #
+      # [Boolean] Whether the file exists in the storage. Asking costs a request when
+      #   the storage is a remote one.
+      #
+      def exists?
+        !!file&.exists?
       end
 
       ##

data/lib/carrierwave/uploader/store.rb

@@ -94,6 +94,7 @@ module CarrierWave
         cache!(new_file) if new_file && !cached?
         if !cache_only && @file && @cache_id
           with_callbacks(:store, new_file) do
+            ensure_the_filename_can_be_worked_out_again
             new_file = storage.store!(@file)
             if delete_tmp_file_after_storage
               @file.delete unless move_to_store
@@ -116,8 +117,31 @@ module CarrierWave
       #
       def retrieve_from_store!(identifier)
         with_callbacks(:retrieve_from_store, identifier) do
-          @file = storage.retrieve!(identifier)
+          # Has to come first, as the storage derives the path from what was recorded
+          # for the identifier
           @identifier = identifier
+          @file = storage.retrieve!(identifier)
+        end
+      end
+
+      ##
+      # Takes the file already stored with the given identifier as its own, for when
+      # something other than CarrierWave put it there.
+      #
+      # === Parameters
+      #
+      # [identifier (String)] identifies the file in the store
+      #
+      # === Raises
+      #
+      # [CarrierWave::InvalidParameter] if the identifier points out of the store dir
+      #
+      def adopt!(identifier)
+        # The file gets removed when replaced, so it must not point out of the store dir
+        raise CarrierWave::InvalidParameter, "invalid identifier" if identifier.start_with?('/') || identifier.split('/').include?('..')
+
+        with_callbacks(:adopt, identifier) do
+          retrieve_from_store!(identifier)
         end
       end
 
@@ -143,6 +167,29 @@ module CarrierWave
 
     private
 
+      ##
+      # A later request gets the identifier and nothing else, so the name the file is
+      # stored under has to follow from it. Ask a fresh uploader what it would work out,
+      # and refuse before anything is written rather than leave the file somewhere it
+      # would never be found. Versions are left out: as long as the original is found,
+      # they can be made again. So is a mount which records what it stored, which is
+      # spared having to work the name out at all.
+      #
+      def ensure_the_filename_can_be_worked_out_again
+        if filename.blank?
+          raise CarrierWave::FilenameNotReproducible, "The file has no name to be stored under. " \
+            "#filename has to give one, as a cached file doesn't have to carry a name of its own."
+        end
+        return if parent_version || metadata_recorded?
+
+        for_file = deduplicated_filename
+        would_look_for = self.class.new(model, mounted_as).send(:full_filename, for_file)
+        return if would_look_for == full_filename(for_file)
+
+        raise CarrierWave::FilenameNotReproducible, "The file would be stored as #{full_filename(for_file).inspect}, but a later request given only the identifier #{for_file.inspect} would look for #{would_look_for.inspect}. " \
+          "#full_filename has to follow from the identifier, as that is all such a request gets."
+      end
+
       def full_filename(for_file)
         forcing_extension(for_file)
       end

data/lib/carrierwave/uploader/url.rb

@@ -15,6 +15,9 @@ module CarrierWave
       # [String] the location where this file is accessible via a url
       #
       def url(options = {})
+        # The URL is to be fetched by a browser, which the file has to survive this request for
+        materialize_cache! if cached?
+
         if file.respond_to?(:url)
           tmp_url = file.method(:url).arity.zero? ? file.url : file.url(options)
           return tmp_url if tmp_url.present?

data/lib/carrierwave/uploader/versions.rb

@@ -9,6 +9,7 @@ module CarrierWave
           @options = {}
           @blocks = []
           @klass = nil
+          @mutex = Mutex.new
         end
 
         def configure(options, &block)
@@ -18,55 +19,68 @@ module CarrierWave
         end
 
         def build(superclass)
+          # Double-checked locking pattern for thread-safe lazy initialization.
+          # Without synchronization, concurrent threads can observe a partially
+          # initialized @klass where the class is created but version_options
+          # is not yet set, causing NoMethodError in version_active?.
           return @klass if @klass
-          @klass = Class.new(superclass)
-          superclass.const_set("VersionUploader#{@name.to_s.camelize}", @klass)
-
-          @klass.version_names += [@name]
-          @klass.versions = {}
-          @klass.processors = []
-          @klass.version_options = @options
-          @klass.class_eval <<-RUBY, __FILE__, __LINE__ + 1
-            # Define the enable_processing method for versions so they get the
-            # value from the parent class unless explicitly overwritten
-            def self.enable_processing(value=nil)
-              self.enable_processing = value if value
-              if defined?(@enable_processing) && !@enable_processing.nil?
-                @enable_processing
-              else
-                superclass.enable_processing
+
+          @mutex.synchronize do
+            return @klass if @klass
+
+            klass = Class.new(superclass)
+            superclass.const_set("VersionUploader#{@name.to_s.camelize}", klass)
+
+            klass.version_names += [@name]
+            klass.versions = {}
+            klass.processors = []
+            klass.version_options = @options
+            klass.class_eval <<-RUBY, __FILE__, __LINE__ + 1
+              # Define the enable_processing method for versions so they get the
+              # value from the parent class unless explicitly overwritten
+              def self.enable_processing(value=nil)
+                self.enable_processing = value if value
+                if defined?(@enable_processing) && !@enable_processing.nil?
+                  @enable_processing
+                else
+                  superclass.enable_processing
+                end
               end
-            end
 
-            # Regardless of what is set in the parent uploader, do not enforce the
-            # move_to_cache config option on versions because it moves the original
-            # file to the version's target file.
-            #
-            # If you want to enforce this setting on versions, override this method
-            # in each version:
-            #
-            # version :thumb do
-            #   def move_to_cache
-            #     true
-            #   end
-            # end
-            #
-            def move_to_cache
-              false
-            end
+              # Regardless of what is set in the parent uploader, do not enforce the
+              # move_to_cache config option on versions because it moves the original
+              # file to the version's target file.
+              #
+              # If you want to enforce this setting on versions, override this method
+              # in each version:
+              #
+              # version :thumb do
+              #   def move_to_cache
+              #     true
+              #   end
+              # end
+              #
+              def move_to_cache
+                false
+              end
 
-            # Need to rely on the parent version's identifier, as versions don't have its own one.
-            def identifier
-              parent_version.identifier
-            end
-          RUBY
-          @blocks.each { |block| @klass.class_eval(&block) }
-          @klass
+              # Need to rely on the parent version's identifier, as versions don't have its own one.
+              def identifier
+                parent_version.identifier
+              end
+            RUBY
+            @blocks.each { |block| klass.class_eval(&block) }
+
+            # Only assign to @klass after full initialization to ensure
+            # other threads never see a partially initialized class.
+            @klass = klass
+          end
         end
 
         def deep_dup
           other = dup
           other.instance_variable_set(:@blocks, @blocks.dup)
+          other.instance_variable_set(:@mutex, Mutex.new)
           other
         end
 
@@ -97,15 +111,18 @@ module CarrierWave
         attr_accessor :parent_version
 
         after :cache, :cache_versions!
+        after :materialize_cache, :materialize_versions_cache!
         after :store, :store_versions!
         after :remove, :remove_versions!
         after :retrieve_from_cache, :retrieve_versions_from_cache!
         after :retrieve_from_store, :retrieve_versions_from_store!
+        after :adopt, :adopt_versions!
+        before :direct_upload, :refuse_direct_upload_of_versions!
 
         prepend Module.new {
           def initialize(*)
             super
-            @versions = nil
+            @versions = @deferred_version_retrieval = nil
           end
         }
       end
@@ -170,11 +187,16 @@ module CarrierWave
       # [Hash{Symbol => CarrierWave::Uploader}] a list of uploader instances
       #
       def versions
-        return @versions if @versions
-        @versions = {}
-        self.class.versions.each do |name, version|
-          @versions[name] = version.build(self.class).new(model, mounted_as)
-          @versions[name].parent_version = self
+        unless @versions
+          @versions = {}
+          self.class.versions.each do |name, version|
+            @versions[name] = version.build(self.class).new(model, mounted_as)
+            @versions[name].parent_version = self
+          end
+        end
+        if (retrieval = @deferred_version_retrieval)
+          @deferred_version_retrieval = nil
+          active_versions.each_value { |v| v.public_send(*retrieval) }
         end
         @versions
       end
@@ -231,7 +253,7 @@ module CarrierWave
       # that the cached files are generated by the single upload attempt.
       #
       def cache!(*args)
-        self.cache_id = parent_version.cache_id if parent_version
+        self.cache_id = parent_version.cache_id if parent_version&.cached?
 
         super
       end
@@ -324,13 +346,18 @@ module CarrierWave
       end
 
       def full_original_filename
-        [version_name, super].compact.join('_')
+        # nil stays nil, as a cached file doesn't have to carry a name of its own
+        [version_name, super].compact.join('_').presence
       end
 
       def cache_versions!(new_file)
         derived_versions.each_value { |v| v.cache!(new_file) }
       end
 
+      def materialize_versions_cache!
+        versions.each_value(&:materialize_cache!)
+      end
+
       def store_versions!(new_file)
         active_versions.each_value { |v| v.store!(new_file) }
       end
@@ -339,12 +366,28 @@ module CarrierWave
         versions.each_value { |v| v.remove! }
       end
 
+      # Retrieval is deferred until #versions is accessed, as evaluating the versions'
+      # conditions can be costly and is pointless when no version is used
       def retrieve_versions_from_cache!(cache_name)
-        active_versions.each_value { |v| v.retrieve_from_cache!(cache_name) }
+        @deferred_version_retrieval = [:retrieve_from_cache!, cache_name]
       end
 
       def retrieve_versions_from_store!(identifier)
-        active_versions.each_value { |v| v.retrieve_from_store!(identifier) }
+        @deferred_version_retrieval = [:retrieve_from_store!, identifier]
+      end
+
+      # Versions are created from a file the application has, which a direct upload never gives it
+      def refuse_direct_upload_of_versions!(filename)
+        return if versions.empty?
+
+        raise "#{filename.inspect} can't be uploaded directly, as nothing would create the versions #{versions.keys.join(', ')}"
+      end
+
+      # Versions are created only through the cache, which an adopted file never goes through
+      def adopt_versions!(identifier)
+        return if active_versions.empty?
+
+        raise "#{identifier.inspect} can't be adopted, as nothing would create the versions #{active_versions.keys.join(', ')}"
       end
 
     end # Versions

data/lib/carrierwave/uploader.rb

@@ -5,6 +5,7 @@ require "carrierwave/uploader/url"
 require "carrierwave/uploader/mountable"
 require "carrierwave/uploader/cache"
 require "carrierwave/uploader/store"
+require "carrierwave/uploader/direct_upload"
 require "carrierwave/uploader/download"
 require "carrierwave/uploader/remove"
 require "carrierwave/uploader/extension_allowlist"
@@ -15,6 +16,7 @@ require "carrierwave/uploader/file_size"
 require "carrierwave/uploader/dimension"
 require "carrierwave/uploader/processing"
 require "carrierwave/uploader/versions"
+require "carrierwave/uploader/metadata"
 require "carrierwave/uploader/default_url"
 
 require "carrierwave/uploader/serialization"
@@ -51,6 +53,7 @@ module CarrierWave
       include CarrierWave::Uploader::Mountable
       include CarrierWave::Uploader::Cache
       include CarrierWave::Uploader::Store
+      include CarrierWave::Uploader::DirectUpload
       include CarrierWave::Uploader::Download
       include CarrierWave::Uploader::Remove
       include CarrierWave::Uploader::ExtensionAllowlist
@@ -61,6 +64,7 @@ module CarrierWave
       include CarrierWave::Uploader::Dimension
       include CarrierWave::Uploader::Processing
       include CarrierWave::Uploader::Versions
+      include CarrierWave::Uploader::Metadata
       include CarrierWave::Uploader::DefaultUrl
       include CarrierWave::Uploader::Serialization
     end # Base

data/lib/carrierwave/utilities/uri.rb

@@ -3,23 +3,48 @@ require 'uri'
 module CarrierWave
   module Utilities
     module Uri
+      # For encoding raw data into a URI component, where '%' is data and becomes '%25'.
       # based on Ruby < 2.0's URI.encode
       PATH_SAFE = URI::RFC2396_REGEXP::PATTERN::UNRESERVED + '\/'
       PATH_UNSAFE = Regexp.new("[^#{PATH_SAFE}]", false)
-      NON_ASCII = /[^[:ascii:]]/.freeze
 
-    private
+      # For repairing a string which is already a URI, following RFC 3986 per component.
+      UNRESERVED = 'A-Za-z0-9\-._~'.freeze
+      SUB_DELIMS = "!$&'()*+,;=".freeze
+      PCHAR = "#{UNRESERVED}#{SUB_DELIMS}:@".freeze
+      SANITIZE_PATH = Regexp.new("[#{PCHAR}/]").freeze
+      # Also allows [ ] { } | \ ^, which browsers and URI.parse accept in a query
+      SANITIZE_QUERY = Regexp.new("[#{PCHAR}/?\\[\\]{}|\\\\^]").freeze
+      SANITIZE_FRAGMENT = Regexp.new("[#{PCHAR}/?]").freeze
 
+      PERCENT_ENCODED = /%[0-9a-fA-F]{2}/.freeze
+
+      # URI::DEFAULT_PARSER#escape warns as obsolete, and URI::RFC2396_PARSER is missing in older uri gems
+      RFC2396_PARSER = URI::RFC2396_Parser.new
+
+    module_function
+
+      # Not idempotent, as '%' is escaped to '%25' every time
       def encode_path(path)
-        URI::DEFAULT_PARSER.escape(path, PATH_UNSAFE)
+        RFC2396_PARSER.escape(path, PATH_UNSAFE)
+      end
+
+      # Only for strings not sent over the wire, like a filename to be shown to the user.
+      # CGI.unescape is for form encoding and would turn '+' into a space.
+      def decode_path(str)
+        RFC2396_PARSER.unescape(str)
       end
 
-      def encode_non_ascii(str)
-        URI::DEFAULT_PARSER.escape(str, NON_ASCII)
+      # Escapes only what cannot appear in the component, leaving existing %XX alone.
+      # Decoding first would make %2F and '/', %2B and '+' indistinguishable.
+      def sanitize_component(str, allowed)
+        str.gsub(/(#{PERCENT_ENCODED})|(.)/m) do
+          $1 || ($2.match?(allowed) ? $2 : escape_octets($2))
+        end
       end
 
-      def decode_uri(str)
-        URI::DEFAULT_PARSER.unescape(str)
+      def escape_octets(str)
+        str.b.bytes.map { |byte| format('%%%02X', byte) }.join
       end
     end # Uri
   end # Utilities

data/lib/carrierwave/version.rb

@@ -1,3 +1,3 @@
 module CarrierWave
-  VERSION = "3.1.4".freeze
+  VERSION = "4.0.0.rc".freeze
 end

data/lib/carrierwave.rb

@@ -98,6 +98,7 @@ end
 require "carrierwave/utilities"
 require "carrierwave/error"
 require "carrierwave/sanitized_file"
+require "carrierwave/direct_upload"
 require "carrierwave/mounter"
 require "carrierwave/mount"
 require "carrierwave/processing"