carrierwave 3.1.3 → 4.0.0.beta

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 20e93cc014326f32bc9123d74648c047a9eeaf634a421981df01ece46dc142cc
-  data.tar.gz: e86f6c907282bc12306f916c31084f34a20e815a4b64291971eefbf366f7ec0c
+  metadata.gz: 44d2471aedac47f453dcf2e0b0b3a36657ff89a1cd521875528a118cdfc465f4
+  data.tar.gz: 4de4a19f9c7b959d8bfd3658c8ee672b03227a5c74d96ec86cbca64ab692d726
 SHA512:
-  metadata.gz: 0b1ade5c07dccfc124750e5c066e5a8dff02c52a918e3af496c1de35a42e836d1d1465fe7f48283e3a088214d843b107102fc7a87e0a68d148884d07e0378529
-  data.tar.gz: 9597174ba08ceb10b8ab91e76e67fd1d37a54c97dac99cb745408da716297bbd960c81b809ec003d6b9695dbe0c2d363c1e958e25a43602b5049f6c2f25bd1a0
+  metadata.gz: 5f15fe1cc5ca10c246398c23bfa6d882494434dde71f40c4d044d949fae30e0929ade0d928ae951b028d2c9821538ae3d1e18a71c43f17e7a5256ecc19f4b8ea
+  data.tar.gz: 675258a32e180790066ea27d8d12aad87e78c607e51125afc0c24fef5eab42588f7a2dba4d54967ee28b52ae7d4921bf408801ee6ab504ebac05d91ef907cdbd

data/README.md

@@ -378,6 +378,32 @@ class MyUploader < CarrierWave::Uploader::Base
 end
 ```
 
+`process convert: format` cannot be given a condition. The file extension follows the
+conversion, and it has to be worked out again when the file is retrieved, where whether
+the condition held is not known. Convert unconditionally, or give the format a version
+of its own:
+
+```ruby
+version :webp, if: :convert_to_webp? do
+  process convert: :webp   # unconditional within the version
+end
+```
+
+If you need to convert on a condition and keep it on the uploader itself, take the
+naming over so that it still follows from the identifier:
+
+```ruby
+process :to_jpeg, if: :heic?
+
+def to_jpeg
+  minimagick! { |builder| builder.convert('jpg') }
+end
+
+def full_filename(for_file)
+  for_file.sub(/\.heic\z/i, '.jpg')
+end
+```
+
 ### Nested versions
 
 It is possible to nest versions within versions:
@@ -424,6 +450,13 @@ end
 ```
 
 The `model` variable points to the instance object the uploader is attached to.
+The condition is evaluated when the version is accessed, so it isn't called at all
+as long as you only use the original file.
+
+Note that `present?` on a version only tells whether a file is assigned, not whether
+that version was actually created. Use `uploader.thumb.exists?` to ask the storage,
+which costs a request when the storage is a remote one, or
+[record what was stored](#recording-what-was-stored) to have the answer at hand.
 
 ### Create versions from existing versions
 
@@ -466,7 +499,44 @@ end
 
 Please note that `#full_filename` mustn't be constructed based on a dynamic value
 that can change from the time of store and time of retrieval, since it will result in
-being unable to retrieve a file previously stored.
+being unable to retrieve a file previously stored. Storing refuses to leave a file
+somewhere a later request would not look, so this is caught rather than silent.
+
+## Recording what was stored
+
+By default CarrierWave persists the identifier alone, and works out everything else on
+retrieval by re-running the uploader's definition and asking the storage. Which versions
+a conditional creates is decided when the file is stored, so re-deciding it later can
+give a different answer than what is actually there, and asking a remote storage for
+things like the size costs a request per record.
+
+Give the mount a column to record it in, and it stops guessing:
+
+```ruby
+class Event < ActiveRecord::Base
+  mount_uploader :image, ImageUploader, metadata_column: :image_metadata
+end
+```
+
+```ruby
+add_column :events, :image_metadata, :json
+```
+
+The column can be a `json`/`jsonb` one, a serialized one, or plain text holding JSON.
+For `mount_uploaders` it holds one set of facts per file.
+
+Records stored before the column was added have nothing recorded, and keep working as
+they always have, so no backfill is needed.
+
+Override `#build_metadata` to record your own facts:
+
+```ruby
+class ImageUploader < CarrierWave::Uploader::Base
+  def build_metadata
+    super.merge('width' => width, 'height' => height)
+  end
+end
+```
 
 ## Making uploads work across form redisplays
 
@@ -500,6 +570,11 @@ case of images, a small thumbnail would be a good indicator:
 <% end %>
 ```
 
+When the cache storage is a remote one, uploading the cached file to it is deferred until
+it turns out to be needed beyond the current request, which reading `avatar_cache` or
+`avatar_url` tells CarrierWave. If you carry the cache name over by other means, call
+`user.avatar.materialize_cache!` before handing it out.
+
 ## Removing uploaded files
 
 If you want to remove a previously uploaded file on a mounted uploader, you can

data/lib/carrierwave/downloader/base.rb

@@ -1,6 +1,6 @@
 require 'open-uri'
 require 'ssrf_filter'
-require 'addressable'
+require 'addressable/uri'
 require 'carrierwave/downloader/remote_file'
 
 module CarrierWave
@@ -8,6 +8,14 @@ module CarrierWave
     class Base
       include CarrierWave::Utilities::Uri
 
+      # Leftovers from pasting, removed before parsing as in the first steps of
+      # https://url.spec.whatwg.org/#concept-basic-url-parser
+      LEADING_TRAILING_JUNK = /\A[\x00-\x20]+|[\x00-\x20]+\z/.freeze
+      EMBEDDED_JUNK = /[\t\n\r]/.freeze
+      # Matches a host name with an optional port, but not a scheme like 'http://' or 'data:'
+      HOSTISH = /\A[[:alnum:]][[:alnum:]\-._]*(?::\d+)?(?:[\/?#]|\z)/.freeze
+      DEFAULT_SCHEME = 'https'.freeze
+
       attr_reader :uploader
 
       def initialize(uploader)
@@ -29,7 +37,7 @@ module CarrierWave
         uri = process_uri(url.to_s)
         begin
           if skip_ssrf_protection?(uri)
-            response = OpenURI.open_uri(process_uri(url.to_s), headers)
+            response = OpenURI.open_uri(uri, headers)
           else
             request = nil
             if ::SsrfFilter::VERSION.to_f < 1.1
@@ -59,22 +67,42 @@ module CarrierWave
       ##
       # Processes the given URL by parsing it, and escaping if necessary. Public to allow overriding.
       #
+      # Never decodes, as that would make %2F and '/', %2B and '+' indistinguishable.
+      #
       # === Parameters
       #
       # [url (String)] The URL where the remote file is stored
       #
       def process_uri(source)
-        uri = Addressable::URI.parse(source)
+        uri = Addressable::URI.parse(normalize_input(source))
         uri.host = uri.normalized_host
-        # Perform decode first, as the path is likely to be already encoded
-        uri.path = encode_path(decode_uri(uri.path)) if uri.path =~ CarrierWave::Utilities::Uri::PATH_UNSAFE
-        uri.query = encode_non_ascii(uri.query) if uri.query
-        uri.fragment = encode_non_ascii(uri.fragment) if uri.fragment
+        uri.path = sanitize_component(uri.path, SANITIZE_PATH) if uri.path
+        uri.query = sanitize_component(uri.query, SANITIZE_QUERY) if uri.query
+        uri.fragment = sanitize_component(uri.fragment, SANITIZE_FRAGMENT) if uri.fragment
         URI.parse(uri.to_s)
       rescue URI::InvalidURIError, Addressable::URI::InvalidURIError
         raise CarrierWave::DownloadError, "couldn't parse URL: #{source}"
       end
 
+      ##
+      # Cleans up a URL as pasted by an end user, since `remote_#{column}_url=` is meant to be
+      # fed from a text field. Public to allow overriding.
+      #
+      # === Parameters
+      #
+      # [source (String)] The URL given by the user
+      #
+      def normalize_input(source)
+        source = source.to_s.gsub(LEADING_TRAILING_JUNK, '').gsub(EMBEDDED_JUNK, '')
+
+        return "#{DEFAULT_SCHEME}:#{source}" if source.start_with?('//')
+        # Don't promote a local path like '/etc/passwd' into a URL
+        return source if source.start_with?('/')
+        return "#{DEFAULT_SCHEME}://#{source}" if source.match?(HOSTISH)
+
+        source
+      end
+
       ##
       # If this returns true, SSRF protection will be bypassed.
       # You can override this if you want to allow accessing specific local URIs that are not SSRF exploitable.

data/lib/carrierwave/downloader/remote_file.rb

@@ -1,6 +1,8 @@
 module CarrierWave
   module Downloader
     class RemoteFile
+      include CarrierWave::Utilities::Uri
+
       attr_reader :file, :uri
 
       def initialize(file)
@@ -53,7 +55,7 @@ module CarrierWave
       end
 
       def filename_from_uri
-        CGI.unescape(File.basename(uri.path))
+        decode_path(File.basename(uri.path))
       end
 
       def method_missing(*args, &block)

data/lib/carrierwave/error.rb

@@ -5,4 +5,5 @@ module CarrierWave
   class ProcessingError < UploadError; end
   class DownloadError < UploadError; end
   class UnknownStorageError < StandardError; end
+  class FilenameNotReproducible < StandardError; end
 end

data/lib/carrierwave/locale/en.yml

@@ -9,6 +9,7 @@ en:
       content_type_allowlist_error: "You are not allowed to upload %{content_type} files, allowed types: %{allowed_types}"
       content_type_denylist_error: "You are not allowed to upload %{content_type} files"
       processing_error: "Failed to manipulate, maybe it is not an image?"
+      unsupported_image_format_error: "Failed to manipulate, file format is not supported"
       min_size_error: "File size should be greater than %{min_size}"
       max_size_error: "File size should be less than %{max_size}"
       min_width_error: "Image width should be greater than %{min_width}px"

data/lib/carrierwave/mount.rb

@@ -102,6 +102,7 @@ module CarrierWave
     # === Options
     #
     # [:mount_on => Symbol] if the name of the column to be serialized to differs you can override it using this option
+    # [:metadata_column => Symbol] the column to record what was stored in, so that it does not have to be worked out again
     # [:ignore_integrity_errors => Boolean] if set to true, integrity errors will result in caching failing silently
     # [:ignore_processing_errors => Boolean] if set to true, processing errors will result in caching failing silently
     #
@@ -247,6 +248,7 @@ module CarrierWave
     # === Options
     #
     # [:mount_on => Symbol] if the name of the column to be serialized to differs you can override it using this option
+    # [:metadata_column => Symbol] the column to record what was stored in, so that it does not have to be worked out again
     # [:ignore_integrity_errors => Boolean] if set to true, integrity errors will result in caching failing silently
     # [:ignore_processing_errors => Boolean] if set to true, processing errors will result in caching failing silently
     #

data/lib/carrierwave/mounter.rb

@@ -1,3 +1,6 @@
+require "json"
+require "active_support/core_ext/array/wrap"
+
 module CarrierWave
 
   # this is an internal class, used by CarrierWave::Mount so that
@@ -8,6 +11,10 @@ module CarrierWave
         uploaders.first&.identifier
       end
 
+      def metadata
+        uploaders.first&.build_metadata
+      end
+
       def temporary_identifier
         temporary_identifiers.first
       end
@@ -18,6 +25,10 @@ module CarrierWave
         uploaders.map(&:identifier).presence
       end
 
+      def metadata
+        uploaders.map(&:build_metadata).presence
+      end
+
       def temporary_identifier
         temporary_identifiers.presence
       end
@@ -63,11 +74,28 @@ module CarrierWave
       [record.read_uploader(serialization_column)].flatten.reject(&:blank?)
     end
 
+    ##
+    # The stored metadata, as many as there are identifiers. Whatever the column gives
+    # is accepted, so that it can be a json column, a serialized one, or plain text.
+    #
+    def read_metadata
+      return [] unless metadata_column
+
+      value = record.read_uploader(metadata_column)
+      value = JSON.parse(value) rescue nil if value.is_a?(String)
+      Array.wrap(value)
+    end
+
     def uploaders
-      @uploaders ||= read_identifiers.map do |identifier|
-        uploader = blank_uploader
-        uploader.retrieve_from_store!(identifier)
-        uploader
+      @uploaders ||= begin
+        metadata = read_metadata
+        read_identifiers.each_with_index.map do |identifier, index|
+          uploader = blank_uploader
+          # Has to come first, as retrieval derives the path from what was recorded
+          uploader.metadata = metadata[index]
+          uploader.retrieve_from_store!(identifier)
+          uploader
+        end
       end
     end
 
@@ -101,6 +129,8 @@ module CarrierWave
     end
 
     def cache_names
+      # The names are carried over to the next request, which the files have to survive
+      uploaders.each(&:materialize_cache!)
       uploaders.map(&:cache_name).compact
     end
 
@@ -157,6 +187,7 @@ module CarrierWave
       @added_uploaders += additions
 
       record.write_uploader(serialization_column, identifier)
+      record.write_uploader(metadata_column, metadata) if metadata_column
     end
 
     def urls(*args)
@@ -197,6 +228,10 @@ module CarrierWave
       option(:mount_on) || column
     end
 
+    def metadata_column
+      option(:metadata_column)
+    end
+
     def remove_previous
       current_paths = uploaders.map(&:path)
       @removed_uploaders

data/lib/carrierwave/orm/activerecord.rb

@@ -52,6 +52,9 @@ module CarrierWave
           @_mounters[:"#{column}"] = nil
           # The attribute needs to be cleared to prevent it from picked up as identifier
           write_uploader(_mounter(:#{column}).serialization_column, nil)
+          if (metadata_column = _mounter(:#{column}).metadata_column)
+            write_uploader(metadata_column, nil)
+          end
           _mounter(:"#{column}").cache(old_uploaders)
         end
 

data/lib/carrierwave/processing/rmagick.rb

@@ -383,6 +383,11 @@ module CarrierWave
         frame = yield(*[frame, index, options].take(block.arity)) if block_given?
         frames << frame if frame
       end
+
+      if frames.none?
+        raise CarrierWave::ProcessingError, I18n.translate(:"errors.messages.unsupported_image_format_error")
+      end
+
       frames.append(true) if block_given?
 
       write_block = create_info_block(options[:write])

data/lib/carrierwave/sanitized_file.rb

@@ -317,10 +317,18 @@ module CarrierWave
     def declared_content_type
       @declared_content_type ||
         if @file.respond_to?(:content_type) && @file.content_type
-          Marcel::MimeType.for(declared_type: @file.content_type.to_s.chomp)
+          Marcel::MimeType.for(declared_type: first_declared_content_type(@file.content_type))
         end
     end
 
+    # Clients occasionally send more than one MIME type in a single header
+    # (e.g. "image/png; text/html" or "image/png, text/html"). Marcel 2 validates
+    # the declared type against the RFC grammar and rejects such values, so pick
+    # the first media type ourselves before handing it over.
+    def first_declared_content_type(content_type)
+      content_type.to_s.strip.split(/[;,\s]/, 2).first
+    end
+
     # Guess content type from its file extension. Limit what to be returned to prevent spoofing.
     def guessed_safe_content_type
       return unless path

data/lib/carrierwave/storage/abstract.rb

@@ -31,6 +31,11 @@ module CarrierWave
         raise NotImplementedError, "Need to implement #retrieve_from_cache! if you want to use #{self.class.name} as a cache storage."
       end
 
+      # Storage engines which defer writing to the cache have to flush it here
+      def materialize_cache!(file)
+        file
+      end
+
       def delete_dir!(path)
         raise NotImplementedError, "Need to implement #delete_dir! if you want to use #{self.class.name} as a cache storage."
       end

data/lib/carrierwave/storage/fog.rb

@@ -105,7 +105,8 @@ module CarrierWave
       end
 
       ##
-      # Stores given file to cache directory.
+      # Stages given file to be cached, deferring the upload until #materialize_cache!,
+      # as a cached file usually goes straight to the store within the same request.
       #
       # === Parameters
       #
@@ -113,12 +114,35 @@ module CarrierWave
       #
       # === Returns
       #
-      # [CarrierWave::SanitizedFile] a sanitized file
+      # [CarrierWave::SanitizedFile] the staged file
+      #   or
+      # [CarrierWave::Storage::Fog::File] the uploaded file, with cache_only
       #
       def cache!(new_file)
-        f = CarrierWave::Storage::Fog::File.new(uploader, self, uploader.cache_path)
-        f.store(new_file)
-        f
+        # With cache_only there's no store to move to, so the cache is the final location
+        return upload_to_cache(new_file) if uploader.cache_only
+
+        local_storage.cache!(new_file)
+      end
+
+      ##
+      # Uploads the staged file to the cache directory.
+      #
+      # === Parameters
+      #
+      # [file (CarrierWave::SanitizedFile)] the staged file
+      #
+      # === Returns
+      #
+      # [CarrierWave::Storage::Fog::File] the uploaded file
+      #
+      def materialize_cache!(file)
+        return file if file.is_a?(CarrierWave::Storage::Fog::File)
+
+        upload_to_cache(file).tap do
+          file.delete
+          local_storage.delete_dir!(uploader.cache_path(nil))
+        end
       end
 
       ##
@@ -140,10 +164,12 @@ module CarrierWave
       # Deletes a cache dir
       #
       def delete_dir!(path)
-        # do nothing, because there's no such things as 'empty directory'
+        # Only the local cache has dirs, as there's no such things as 'empty directory'
+        local_storage.delete_dir!(path)
       end
 
       def clean_cache!(seconds)
+        local_storage.clean_cache!(seconds)
         connection.directories.new(
           :key    => uploader.fog_directory,
           :public => uploader.fog_public
@@ -296,7 +322,7 @@ module CarrierWave
           return read_source_file if ::File.exist?(file_body.path)
 
           # If the source file doesn't exist, the remote content is read
-          @file = nil
+          remove_instance_variable(:@file)
           file.body
         end
 
@@ -308,7 +334,7 @@ module CarrierWave
         # [Integer] size of file body
         #
         def size
-          file.nil? ? 0 : file.content_length
+          file&.content_length || 0
         end
 
         ##
@@ -439,7 +465,7 @@ module CarrierWave
         #
         def filename(options = {})
           return unless (file_url = url(options))
-          CGI.unescape(file_url.split('?').first).gsub(/.*\/(.*?$)/, '\1')
+          decode_path(file_url.split('?').first).gsub(/.*\/(.*?$)/, '\1')
         end
 
         ##
@@ -512,7 +538,9 @@ module CarrierWave
         # [Fog::#{provider}::File] file data from remote service
         #
         def file
-          @file ||= directory.files.head(path)
+          return @file if defined?(@file)
+
+          @file = directory.files.head(path)
         end
 
         def copy_options
@@ -554,6 +582,18 @@ module CarrierWave
         end
       end
 
+    private
+
+      def upload_to_cache(new_file)
+        f = CarrierWave::Storage::Fog::File.new(uploader, self, uploader.cache_path)
+        f.store(new_file)
+        f
+      end
+
+      def local_storage
+        @local_storage ||= CarrierWave::Storage::File.new(uploader)
+      end
+
     end # Fog
 
   end # Storage

data/lib/carrierwave/test/matchers.rb

@@ -336,6 +336,8 @@ module CarrierWave
         def self.load_image(filename)
           if defined? ::MiniMagick
             MiniMagickWrapper.new(filename)
+          elsif defined? ::Vips
+            VipsWrapper.new(filename)
           else
             unless defined? ::Magick
               begin
@@ -393,6 +395,28 @@ module CarrierWave
         end
       end
 
+      class VipsWrapper # :nodoc:
+        attr_reader :image
+
+        def width
+          image.width
+        end
+
+        def height
+          image.height
+        end
+
+        def format
+          # This returns the name of the vips loader (e.g. pngload) as Vips
+          # doesn't do content detection.
+          image.get("vips-loader").delete_suffix("load")
+        end
+
+        def initialize(filename)
+          @image = ::Vips::Image.new_from_file(filename)
+        end
+      end
+
     end # Matchers
   end # Test
 end # CarrierWave

data/lib/carrierwave/uploader/cache.rb

@@ -151,6 +151,16 @@ module CarrierWave
         end
       end
 
+      ##
+      # Makes sure the cached file is written to the cache storage, which may have
+      # deferred it. Needed when the file has to outlive the current request.
+      #
+      def materialize_cache!
+        with_callbacks(:materialize_cache) do
+          @file = cache_storage.materialize_cache!(@file) if @file
+        end
+      end
+
       ##
       # Retrieves the file with the given cache_name from the cache.
       #

data/lib/carrierwave/uploader/configuration.rb

@@ -45,6 +45,7 @@ module CarrierWave
         add_config :validate_processing
         add_config :validate_download
         add_config :mount_on
+        add_config :metadata_column
         add_config :cache_only
         add_config :download_retry_count
         add_config :download_retry_wait_time

data/lib/carrierwave/uploader/metadata.rb

@@ -0,0 +1,100 @@
+require "active_support/core_ext/hash/keys"
+
+module CarrierWave
+  module Uploader
+    module Metadata
+      extend ActiveSupport::Concern
+
+      include CarrierWave::Uploader::Versions
+
+      ##
+      # Facts about the file which were determined when it was stored, as recorded in
+      # the column given to the +metadata+ option of the mount. They are the answer to
+      # questions the storage cannot answer, like whether a conditional version was
+      # created, and they save asking it the ones it can.
+      #
+      # Returns an empty Hash when nothing was recorded, in which case the facts are
+      # derived from the file and the uploader definition as they have always been.
+      #
+      # === Returns
+      #
+      # [Hash] the recorded facts
+      #
+      def metadata
+        return @metadata || {} unless parent_version
+
+        parent_version.metadata.dig('versions', self.class.version_names.last.to_s) || {}
+      end
+
+      def metadata=(metadata)
+        @metadata = metadata&.deep_stringify_keys
+      end
+
+      ##
+      # Collects the facts to record about the file being stored. Override to record
+      # your own, merging them into the result of +super+.
+      #
+      # === Returns
+      #
+      # [Hash] the facts to record
+      #
+      def build_metadata
+        {
+          'filename' => full_filename(identifier),
+          'size' => size,
+          'content_type' => content_type,
+          'versions' => active_versions.map { |name, version| [name.to_s, version.build_metadata] }.to_h
+        }
+      end
+
+      ##
+      # === Returns
+      #
+      # [Boolean] Whether what is stored gets recorded, which the mount decides
+      #
+      def metadata_recorded?
+        model.class.respond_to?(:uploader_option) &&
+          model.class.uploaders.has_key?(mounted_as) &&
+          !!model.class.uploader_option(mounted_as, :metadata_column)
+      end
+
+      def size
+        metadata['size'] || super
+      end
+
+      def content_type
+        metadata['content_type'] || super
+      end
+
+      ##
+      # The conditions decide which versions to create, so which ones were created is
+      # settled once the file is stored. Consult the record instead of asking them
+      # again, which is not only costly but can give a different answer.
+      #
+      def version_active?(name)
+        recorded_versions = metadata['versions']
+        return super unless recorded_versions
+
+        versions.has_key?(name.to_sym) && recorded_versions.has_key?(name.to_s)
+      end
+
+      def cache!(*)
+        @metadata = nil
+        super
+      end
+
+      def recreate_versions!(*)
+        @metadata = nil
+        super
+      end
+
+    private
+
+      def full_filename(for_file)
+        return super unless for_file == identifier
+
+        metadata['filename'] || super
+      end
+    end # Metadata
+  end # Uploader
+end # CarrierWave

data/lib/carrierwave/uploader/processing.rb

@@ -67,13 +67,17 @@ module CarrierWave
           new_processors.each do |processor, processor_args|
             self.processors += [[processor, processor_args, condition, condition_type]]
 
-            if processor == :convert
-              # Treat :convert specially, since it should trigger the file extension change
-              force_extension processor_args
-              if condition
-                warn "Use of 'process convert: format' with conditionals has an issue and doesn't work correctly. See https://github.com/carrierwaveuploader/carrierwave/issues/2723 for details. "
-              end
+            next unless processor == :convert
+
+            # Treat :convert specially, since it should trigger the file extension change.
+            # That has to be worked out again when the file is retrieved, where whether
+            # the condition held cannot be known, so a conditional one is refused.
+            if condition
+              raise ArgumentError, "`process convert:` cannot be given `:if` or `:unless`. The file extension follows the conversion, and it has to be worked out again when the file is retrieved, where whether the condition held is not known. " \
+                                   "Convert unconditionally, or put the conversion in a version of its own, or take the naming over by overriding #full_filename."
             end
+
+            force_extension processor_args
           end
         end
       end # ClassMethods

data/lib/carrierwave/uploader/proxy.rb

@@ -5,10 +5,26 @@ module CarrierWave
       ##
       # === Returns
       #
-      # [Boolean] Whether the uploaded file is blank
+      # [Boolean] Whether a file is assigned. Use #exists? to ask the storage whether
+      #   the file is actually there.
       #
       def blank?
-        file.blank?
+        return true unless file
+        # A cache name can point at a cache which is gone, so it is verified. It is
+        # local and thus cheap, unlike asking a remote store.
+        return file.empty? if cached?
+
+        false
+      end
+
+      ##
+      # === Returns
+      #
+      # [Boolean] Whether the file exists in the storage. Asking costs a request when
+      #   the storage is a remote one.
+      #
+      def exists?
+        !!file&.exists?
       end
 
       ##

data/lib/carrierwave/uploader/store.rb

@@ -94,6 +94,7 @@ module CarrierWave
         cache!(new_file) if new_file && !cached?
         if !cache_only && @file && @cache_id
           with_callbacks(:store, new_file) do
+            ensure_the_filename_can_be_worked_out_again
             new_file = storage.store!(@file)
             if delete_tmp_file_after_storage
               @file.delete unless move_to_store
@@ -116,8 +117,10 @@ module CarrierWave
       #
       def retrieve_from_store!(identifier)
         with_callbacks(:retrieve_from_store, identifier) do
-          @file = storage.retrieve!(identifier)
+          # Has to come first, as the storage derives the path from what was recorded
+          # for the identifier
           @identifier = identifier
+          @file = storage.retrieve!(identifier)
         end
       end
 
@@ -143,6 +146,25 @@ module CarrierWave
 
     private
 
+      ##
+      # A later request gets the identifier and nothing else, so the name the file is
+      # stored under has to follow from it. Ask a fresh uploader what it would work out,
+      # and refuse before anything is written rather than leave the file somewhere it
+      # would never be found. Versions are left out: as long as the original is found,
+      # they can be made again. So is a mount which records what it stored, which is
+      # spared having to work the name out at all.
+      #
+      def ensure_the_filename_can_be_worked_out_again
+        return if parent_version || metadata_recorded?
+
+        for_file = deduplicated_filename
+        would_look_for = self.class.new(model, mounted_as).send(:full_filename, for_file)
+        return if would_look_for == full_filename(for_file)
+
+        raise CarrierWave::FilenameNotReproducible, "The file would be stored as #{full_filename(for_file).inspect}, but a later request given only the identifier #{for_file.inspect} would look for #{would_look_for.inspect}. " \
+          "#full_filename has to follow from the identifier, as that is all such a request gets."
+      end
+
       def full_filename(for_file)
         forcing_extension(for_file)
       end

data/lib/carrierwave/uploader/url.rb

@@ -15,6 +15,9 @@ module CarrierWave
       # [String] the location where this file is accessible via a url
       #
       def url(options = {})
+        # The URL is to be fetched by a browser, which the file has to survive this request for
+        materialize_cache! if cached?
+
         if file.respond_to?(:url)
           tmp_url = file.method(:url).arity.zero? ? file.url : file.url(options)
           return tmp_url if tmp_url.present?

data/lib/carrierwave/uploader/versions.rb

@@ -9,6 +9,7 @@ module CarrierWave
           @options = {}
           @blocks = []
           @klass = nil
+          @mutex = Mutex.new
         end
 
         def configure(options, &block)
@@ -18,55 +19,68 @@ module CarrierWave
         end
 
         def build(superclass)
+          # Double-checked locking pattern for thread-safe lazy initialization.
+          # Without synchronization, concurrent threads can observe a partially
+          # initialized @klass where the class is created but version_options
+          # is not yet set, causing NoMethodError in version_active?.
           return @klass if @klass
-          @klass = Class.new(superclass)
-          superclass.const_set("VersionUploader#{@name.to_s.camelize}", @klass)
-
-          @klass.version_names += [@name]
-          @klass.versions = {}
-          @klass.processors = []
-          @klass.version_options = @options
-          @klass.class_eval <<-RUBY, __FILE__, __LINE__ + 1
-            # Define the enable_processing method for versions so they get the
-            # value from the parent class unless explicitly overwritten
-            def self.enable_processing(value=nil)
-              self.enable_processing = value if value
-              if defined?(@enable_processing) && !@enable_processing.nil?
-                @enable_processing
-              else
-                superclass.enable_processing
+
+          @mutex.synchronize do
+            return @klass if @klass
+
+            klass = Class.new(superclass)
+            superclass.const_set("VersionUploader#{@name.to_s.camelize}", klass)
+
+            klass.version_names += [@name]
+            klass.versions = {}
+            klass.processors = []
+            klass.version_options = @options
+            klass.class_eval <<-RUBY, __FILE__, __LINE__ + 1
+              # Define the enable_processing method for versions so they get the
+              # value from the parent class unless explicitly overwritten
+              def self.enable_processing(value=nil)
+                self.enable_processing = value if value
+                if defined?(@enable_processing) && !@enable_processing.nil?
+                  @enable_processing
+                else
+                  superclass.enable_processing
+                end
               end
-            end
 
-            # Regardless of what is set in the parent uploader, do not enforce the
-            # move_to_cache config option on versions because it moves the original
-            # file to the version's target file.
-            #
-            # If you want to enforce this setting on versions, override this method
-            # in each version:
-            #
-            # version :thumb do
-            #   def move_to_cache
-            #     true
-            #   end
-            # end
-            #
-            def move_to_cache
-              false
-            end
+              # Regardless of what is set in the parent uploader, do not enforce the
+              # move_to_cache config option on versions because it moves the original
+              # file to the version's target file.
+              #
+              # If you want to enforce this setting on versions, override this method
+              # in each version:
+              #
+              # version :thumb do
+              #   def move_to_cache
+              #     true
+              #   end
+              # end
+              #
+              def move_to_cache
+                false
+              end
 
-            # Need to rely on the parent version's identifier, as versions don't have its own one.
-            def identifier
-              parent_version.identifier
-            end
-          RUBY
-          @blocks.each { |block| @klass.class_eval(&block) }
-          @klass
+              # Need to rely on the parent version's identifier, as versions don't have its own one.
+              def identifier
+                parent_version.identifier
+              end
+            RUBY
+            @blocks.each { |block| klass.class_eval(&block) }
+
+            # Only assign to @klass after full initialization to ensure
+            # other threads never see a partially initialized class.
+            @klass = klass
+          end
         end
 
         def deep_dup
           other = dup
           other.instance_variable_set(:@blocks, @blocks.dup)
+          other.instance_variable_set(:@mutex, Mutex.new)
           other
         end
 
@@ -97,6 +111,7 @@ module CarrierWave
         attr_accessor :parent_version
 
         after :cache, :cache_versions!
+        after :materialize_cache, :materialize_versions_cache!
         after :store, :store_versions!
         after :remove, :remove_versions!
         after :retrieve_from_cache, :retrieve_versions_from_cache!
@@ -105,7 +120,7 @@ module CarrierWave
         prepend Module.new {
           def initialize(*)
             super
-            @versions = nil
+            @versions = @deferred_version_retrieval = nil
           end
         }
       end
@@ -170,11 +185,16 @@ module CarrierWave
       # [Hash{Symbol => CarrierWave::Uploader}] a list of uploader instances
       #
       def versions
-        return @versions if @versions
-        @versions = {}
-        self.class.versions.each do |name, version|
-          @versions[name] = version.build(self.class).new(model, mounted_as)
-          @versions[name].parent_version = self
+        unless @versions
+          @versions = {}
+          self.class.versions.each do |name, version|
+            @versions[name] = version.build(self.class).new(model, mounted_as)
+            @versions[name].parent_version = self
+          end
+        end
+        if (retrieval = @deferred_version_retrieval)
+          @deferred_version_retrieval = nil
+          active_versions.each_value { |v| v.public_send(*retrieval) }
         end
         @versions
       end
@@ -331,6 +351,10 @@ module CarrierWave
         derived_versions.each_value { |v| v.cache!(new_file) }
       end
 
+      def materialize_versions_cache!
+        versions.each_value(&:materialize_cache!)
+      end
+
       def store_versions!(new_file)
         active_versions.each_value { |v| v.store!(new_file) }
       end
@@ -339,12 +363,14 @@ module CarrierWave
         versions.each_value { |v| v.remove! }
       end
 
+      # Retrieval is deferred until #versions is accessed, as evaluating the versions'
+      # conditions can be costly and is pointless when no version is used
       def retrieve_versions_from_cache!(cache_name)
-        active_versions.each_value { |v| v.retrieve_from_cache!(cache_name) }
+        @deferred_version_retrieval = [:retrieve_from_cache!, cache_name]
       end
 
       def retrieve_versions_from_store!(identifier)
-        active_versions.each_value { |v| v.retrieve_from_store!(identifier) }
+        @deferred_version_retrieval = [:retrieve_from_store!, identifier]
       end
 
     end # Versions

data/lib/carrierwave/uploader.rb

@@ -15,6 +15,7 @@ require "carrierwave/uploader/file_size"
 require "carrierwave/uploader/dimension"
 require "carrierwave/uploader/processing"
 require "carrierwave/uploader/versions"
+require "carrierwave/uploader/metadata"
 require "carrierwave/uploader/default_url"
 
 require "carrierwave/uploader/serialization"
@@ -61,6 +62,7 @@ module CarrierWave
       include CarrierWave::Uploader::Dimension
       include CarrierWave::Uploader::Processing
       include CarrierWave::Uploader::Versions
+      include CarrierWave::Uploader::Metadata
       include CarrierWave::Uploader::DefaultUrl
       include CarrierWave::Uploader::Serialization
     end # Base

data/lib/carrierwave/utilities/uri.rb

@@ -3,23 +3,45 @@ require 'uri'
 module CarrierWave
   module Utilities
     module Uri
+      # For encoding raw data into a URI component, where '%' is data and becomes '%25'.
       # based on Ruby < 2.0's URI.encode
       PATH_SAFE = URI::RFC2396_REGEXP::PATTERN::UNRESERVED + '\/'
       PATH_UNSAFE = Regexp.new("[^#{PATH_SAFE}]", false)
-      NON_ASCII = /[^[:ascii:]]/.freeze
 
-    private
+      # For repairing a string which is already a URI, following RFC 3986 per component.
+      UNRESERVED = 'A-Za-z0-9\-._~'.freeze
+      SUB_DELIMS = "!$&'()*+,;=".freeze
+      PCHAR = "#{UNRESERVED}#{SUB_DELIMS}:@".freeze
+      SANITIZE_PATH = Regexp.new("[#{PCHAR}/]").freeze
+      # Also allows [ ] { } | \ ^, which browsers and URI.parse accept in a query
+      SANITIZE_QUERY = Regexp.new("[#{PCHAR}/?\\[\\]{}|\\\\^]").freeze
+      SANITIZE_FRAGMENT = Regexp.new("[#{PCHAR}/?]").freeze
 
+      PERCENT_ENCODED = /%[0-9a-fA-F]{2}/.freeze
+
+    module_function
+
+      # Not idempotent, as '%' is escaped to '%25' every time
       def encode_path(path)
         URI::DEFAULT_PARSER.escape(path, PATH_UNSAFE)
       end
 
-      def encode_non_ascii(str)
-        URI::DEFAULT_PARSER.escape(str, NON_ASCII)
+      # Only for strings not sent over the wire, like a filename to be shown to the user.
+      # CGI.unescape is for form encoding and would turn '+' into a space.
+      def decode_path(str)
+        URI::DEFAULT_PARSER.unescape(str)
+      end
+
+      # Escapes only what cannot appear in the component, leaving existing %XX alone.
+      # Decoding first would make %2F and '/', %2B and '+' indistinguishable.
+      def sanitize_component(str, allowed)
+        str.gsub(/(#{PERCENT_ENCODED})|(.)/m) do
+          $1 || ($2.match?(allowed) ? $2 : escape_octets($2))
+        end
       end
 
-      def decode_uri(str)
-        URI::DEFAULT_PARSER.unescape(str)
+      def escape_octets(str)
+        str.b.bytes.map { |byte| format('%%%02X', byte) }.join
       end
     end # Uri
   end # Utilities

data/lib/carrierwave/version.rb

@@ -1,3 +1,3 @@
 module CarrierWave
-  VERSION = "3.1.3".freeze
+  VERSION = "4.0.0.beta".freeze
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: carrierwave
 version: !ruby/object:Gem::Version
-  version: 3.1.3
+  version: 4.0.0.beta
 platform: ruby
 authors:
 - Jonas Nicklas
@@ -15,56 +15,68 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 6.0.0
+        version: 7.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 6.0.0
+        version: 7.0.0
 - !ruby/object:Gem::Dependency
   name: activemodel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 6.0.0
+        version: 7.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 6.0.0
+        version: 7.0.0
 - !ruby/object:Gem::Dependency
   name: image_processing
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3'
 - !ruby/object:Gem::Dependency
   name: marcel
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3'
 - !ruby/object:Gem::Dependency
   name: addressable
   requirement: !ruby/object:Gem::Requirement
@@ -239,6 +251,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby-vips
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rmagick
   requirement: !ruby/object:Gem::Requirement
@@ -350,6 +376,7 @@ files:
 - lib/carrierwave/uploader/extension_allowlist.rb
 - lib/carrierwave/uploader/extension_denylist.rb
 - lib/carrierwave/uploader/file_size.rb
+- lib/carrierwave/uploader/metadata.rb
 - lib/carrierwave/uploader/mountable.rb
 - lib/carrierwave/uploader/processing.rb
 - lib/carrierwave/uploader/proxy.rb
@@ -378,14 +405,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.5.0
+      version: 2.7.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.9
+rubygems_version: 4.0.19
 specification_version: 4
 summary: Ruby file upload library
 test_files: []