shrine 2.12.0 → 2.13.0

data/lib/shrine/plugins/determine_mime_type.rb

@@ -179,9 +179,14 @@ class Shrine
 
             raise Error, "file command failed to spawn: #{stderr.read}" if status.nil?
             raise Error, "file command failed: #{stderr.read}" unless status.success?
+
             $stderr.print(stderr.read)
 
-            stdout.read.strip
+            output = stdout.read.strip
+
+            raise Error, "file command failed: #{output}" if output.include?("cannot open")
+
+            output
           end
         rescue Errno::ENOENT
           raise Error, "file command-line tool is not installed"

data/lib/shrine/plugins/download_endpoint.rb

@@ -12,6 +12,9 @@ class Shrine
     # from your storage isn't accessible over URL (e.g. database storages) or
     # if you want to authenticate your downloads. It requires the [Roda] gem.
     #
+    #     # Gemfile
+    #     gem "roda" # dependency of the download_endpoint plugin
+    #
     # You can configure the plugin with the path prefix which the endpoint will
     # be mounted on.
     #

data/lib/shrine/plugins/infer_extension.rb

@@ -5,21 +5,29 @@ class Shrine
     # The `infer_extension` plugin allows deducing the appropriate file
     # extension for the upload location based on the MIME type of the file.
     # This is useful when using `data_uri` and `remote_url` plugins, where the
-    # file extension is not or might not be known.
+    # file extension might not be known.
     #
     #     plugin :infer_extension
     #
-    # The upload location will gain the inferred extension only if couldn't be
-    # determined from the filename. By default `MIME::Types` will be used for
-    # inferring the extension, but you can also choose a different inferrer:
+    # Ordinarily, the upload location will gain the inferred extension only if
+    # it couldn't be determined from the filename. However, you can pass
+    # `force: true` to force the inferred extension to be used rather than an
+    # extension from the original filename. This can be used to canonicalize
+    # extensions (jpg, jpeg => jpeg), or replace an incorrect original
+    # extension.
+    #
+    #     plugin :infer_extension, force: true
+    #
+    # By default `MIME::Types` will be used for inferring the extension, but
+    # you can also choose a different inferrer:
     #
     #     plugin :infer_extension, inferrer: :mini_mime
     #
     # The following inferrers are accepted:
     #
     # :mime_types
-    # : (Default). Uses the [mime-types] gem to infer the appropriate extension from MIME
-    #   type.
+    # : (Default). Uses the [mime-types] gem to infer the appropriate extension
+    #   from MIME type.
     #
     # :mini_mime
     # : Uses the [mini_mime] gem to infer the appropriate extension from MIME
@@ -45,12 +53,13 @@ class Shrine
     # [mini_mime]: https://github.com/discourse/mini_mime
     module InferExtension
       def self.configure(uploader, opts = {})
-        uploader.opts[:extension_inferrer] = opts.fetch(:inferrer, uploader.opts.fetch(:infer_extension_inferrer, :mime_types))
+        uploader.opts[:infer_extension_inferrer] = opts.fetch(:inferrer, uploader.opts.fetch(:infer_extension_inferrer, :mime_types))
+        uploader.opts[:infer_extension_force] = opts.fetch(:force, uploader.opts.fetch(:infer_extension_force, false))
       end
 
       module ClassMethods
         def infer_extension(mime_type)
-          inferrer = opts[:extension_inferrer]
+          inferrer = opts[:infer_extension_inferrer]
           inferrer = extension_inferrer(inferrer) if inferrer.is_a?(Symbol)
           args     = [mime_type, extension_inferrers].take(inferrer.arity.abs)
 
@@ -72,8 +81,14 @@ class Shrine
         def generate_location(io, context = {})
           mime_type = (context[:metadata] || {})["mime_type"]
 
-          location  = super
-          location += infer_extension(mime_type) if File.extname(location).empty?
+          location = super
+          current_extension = File.extname(location)
+
+          if current_extension.empty? || opts[:infer_extension_force]
+            inferred_extension = infer_extension(mime_type)
+            location = location.chomp(current_extension) << inferred_extension unless inferred_extension.empty?
+          end
+
           location
         end
 

data/lib/shrine/plugins/module_include.rb

@@ -19,7 +19,7 @@ class Shrine
     #       def included(model)
     #         super
     #
-    #         module_eval <<-RUBY, __FILE__, __LINE + 1
+    #         module_eval <<-RUBY, __FILE__, __LINE__ + 1
     #           def #{@name}_size(version)
     #             if #{@name}.is_a?(Hash)
     #               #{@name}[version].size

data/lib/shrine/plugins/presign_endpoint.rb

@@ -72,17 +72,16 @@ class Shrine
     # ## Options
     #
     # Some storages accept additional presign options, which you can pass in via
-    # `:presign_options`:
+    # `:presign_options`, here is an example for S3 storage:
     #
     #     plugin :presign_endpoint, presign_options: -> (request) do
     #       filename     = request.params["filename"]
-    #       extension    = File.extname(filename)
-    #       content_type = Rack::Mime.mime_type(extension)
+    #       type         = request.params["type"]
     #
     #       {
-    #         content_length_range: 0..(10*1024*1024),                     # limit filesize to 10MB
-    #         content_disposition: "attachment; filename=\"#{filename}\"", # download with original filename
-    #         content_type:        content_type,                           # set correct content type
+    #         content_length_range: 0..(10*1024*1024),                 # limit filesize to 10MB
+    #         content_disposition: "inline; filename=\"#{filename}\"", # download with original filename
+    #         content_type:        type,                               # set correct content type
     #       }
     #     end
     #
@@ -149,6 +148,9 @@ class Shrine
       # `#presign` on the specified storage, and returns that information in
       # JSON format.
       class App
+        CONTENT_TYPE_JSON = "application/json; charset=utf-8"
+        CONTENT_TYPE_TEXT = "text/plain"
+
         # Writes given options to instance variables.
         def initialize(options)
           options.each do |name, value|
@@ -240,7 +242,7 @@ class Shrine
           if @rack_response
             response = @rack_response.call(object, request)
           else
-            response = [200, {"Content-Type" => "application/json"}, [object.to_json]]
+            response = [200, {"Content-Type" => CONTENT_TYPE_JSON}, [object.to_json]]
           end
 
           # prevent browsers from caching the response
@@ -251,7 +253,7 @@ class Shrine
 
         # Used for early returning an error response.
         def error!(status, message)
-          throw :halt, [status, {"Content-Type" => "text/plain"}, [message]]
+          throw :halt, [status, {"Content-Type" => CONTENT_TYPE_TEXT}, [message]]
         end
 
         # Returns the uploader around the specified storage.

data/lib/shrine/plugins/rack_file.rb

@@ -47,6 +47,14 @@ class Shrine
       module ClassMethods
         # Accepts a Rack uploaded file hash and wraps it in an IO object.
         def rack_file(hash)
+          if hash[:filename]
+            # Rack can sometimes return the filename binary encoded, so we force
+            # the encoding to utf-8
+            hash = hash.merge(
+              filename: hash[:filename].dup.force_encoding(Encoding::UTF_8)
+            )
+          end
+
           UploadedFile.new(hash)
         end
       end

data/lib/shrine/plugins/store_dimensions.rb

@@ -64,7 +64,7 @@ class Shrine
     #
     # [fastimage]: https://github.com/sdsykes/fastimage
     # [mini_magick]: https://github.com/minimagick/minimagick
-    # [ruby-vips]: https://github.com/jcupitt/ruby-vips
+    # [ruby-vips]: https://github.com/libvips/ruby-vips
     module StoreDimensions
       def self.configure(uploader, opts = {})
         uploader.opts[:dimensions_analyzer] = opts.fetch(:analyzer, uploader.opts.fetch(:dimensions_analyzer, :fastimage))

data/lib/shrine/plugins/upload_endpoint.rb

@@ -94,7 +94,7 @@ class Shrine
     # You can also customize the upload itself via the `:upload` option:
     #
     #     plugin :upload_endpoint, upload: -> (io, context, request) do
-    #       # perform uploading and return the Shrine::UploadedFile
+    #       Shrine.new(:cache).upload(io, context)
     #     end
     #
     # ## Response
@@ -150,6 +150,9 @@ class Shrine
       # calls `#upload` with the uploaded file, and returns the uploaded file
       # information in JSON format.
       class App
+        CONTENT_TYPE_JSON = "application/json; charset=utf-8"
+        CONTENT_TYPE_TEXT = "text/plain"
+
         # Writes given options to instance variables.
         def initialize(options)
           options.each do |name, value|
@@ -196,7 +199,8 @@ class Shrine
         def get_io(request)
           file = request.params["file"]
 
-          error!(400, "Upload Not Found") unless file.is_a?(Hash) && file[:tempfile]
+          error!(400, "Upload Not Found") if file.nil?
+          error!(400, "Upload Not Valid") unless file.is_a?(Hash) && file[:tempfile]
           error!(413, "Upload Too Large") if @max_size && file[:tempfile].size > @max_size
 
           verify_checksum!(file[:tempfile], request.env["HTTP_CONTENT_MD5"]) if request.env["HTTP_CONTENT_MD5"]
@@ -232,7 +236,7 @@ class Shrine
           if @rack_response
             @rack_response.call(object, request)
           else
-            [200, {"Content-Type" => "application/json"}, [object.to_json]]
+            [200, {"Content-Type" => CONTENT_TYPE_JSON}, [object.to_json]]
           end
         end
 
@@ -246,7 +250,7 @@ class Shrine
 
         # Used for early returning an error response.
         def error!(status, message)
-          throw :halt, [status, {"Content-Type" => "text/plain"}, [message]]
+          throw :halt, [status, {"Content-Type" => CONTENT_TYPE_TEXT}, [message]]
         end
 
         # Returns the uploader around the specified storage.

data/lib/shrine/plugins/versions.rb

@@ -234,8 +234,7 @@ class Shrine
           end
         end
 
-        # Deletes each file individually, but uses S3's multi delete
-        # capabilities.
+        # Deletes each file individually
         def _delete(uploaded_file, context)
           if (hash = uploaded_file).is_a?(Hash)
             hash.each do |name, value|

data/lib/shrine/storage/file_system.rb

@@ -165,8 +165,9 @@ class Shrine
       # Delets the file, and by default deletes the containing directory if
       # it's empty.
       def delete(id)
-        path(id).delete
-        clean(path(id)) if clean?
+        path = path(id)
+        path.delete
+        clean(path) if clean?
       rescue Errno::ENOENT
       end
 
@@ -241,8 +242,9 @@ class Shrine
 
       # Creates all intermediate directories for that location.
       def path!(id)
-        FileUtils.mkdir_p(path(id).dirname, mode: directory_permissions)
-        path(id)
+        path = path(id)
+        FileUtils.mkdir_p(path.dirname, mode: directory_permissions)
+        path
       end
 
       def relative_path(id)

data/lib/shrine/storage/s3.rb

@@ -37,7 +37,7 @@ class Shrine
     #       region: "eu-west-1",
     #     )
     #
-    # The core features of this storage requires the following AWS permissions:
+    # The core features of this storage require the following AWS permissions:
     # `s3:ListBucket`, `s3:PutObject`, `s3:GetObject`, and `s3:DeleteObject`.
     # If you have additional upload options configured such as setting object
     # ACLs, then additional permissions may be required.
@@ -54,6 +54,18 @@ class Shrine
     #
     #     s3.object("key") #=> #<Aws::S3::Object>
     #
+    # ## Public uploads
+    #
+    # By default, uploaded S3 objects will have private visibility, meaning
+    # they can only be accessed via signed expiring URLs generated using your
+    # private S3 credentials. If you would like to generate public URLs, you
+    # can tell S3 storage to make uploads public:
+    #
+    #     s3 = Shrine::Storage::S3.new(public: true, **s3_options)
+    #
+    #     s3.upload(io, "key") # uploads with "public-read" ACL
+    #     s3.url("key")        # returns public (unsigned) object URL
+    #
     # ## Prefix
     #
     # The `:prefix` option can be specified for uploading all files inside
@@ -103,22 +115,44 @@ class Shrine
     #
     # All other options are forwarded to the aws-sdk-s3 gem:
     #
-    #     s3.url(expires_in: 15)
-    #     s3.url(virtual_host: true)
+    #     s3.url(expires_in: 15, response_content_disposition: "...")
     #
-    # ## CDN
+    # ## URL Host
     #
-    # If you're using a CDN with S3 like Amazon CloudFront, you can specify
-    # the `:host` option to `#url`:
+    # If you want your S3 object URLs to be generated with a different URL host
+    # (e.g. a CDN), you can specify the `:host` option to `#url`:
     #
     #     s3.url("image.jpg", host: "http://abc123.cloudfront.net")
     #     #=> "http://abc123.cloudfront.net/image.jpg"
     #
-    # You have the `:host` option passed automatically for every URL with the
+    # The host URL can include a path prefix, but it needs to end with a slash:
+    #
+    #     s3.url("image.jpg", host: "https://your-s3-host.com/prefix/") # needs to end with a slash
+    #     #=> "http://your-s3-host.com/prefix/image.jpg"
+    #
+    # To have the `:host` option passed automatically for every URL, use the
     # `default_url_options` plugin.
     #
     #     plugin :default_url_options, store: { host: "http://abc123.cloudfront.net" }
     #
+    # If you would like to [serve private content via CloudFront], you need to
+    # sign the object URLs with a special signer, such as
+    # [`Aws::CloudFront::UrlSigner`] provided by the `aws-sdk-cloudfront` gem.
+    # The S3 storage initializer accepts a `:signer` block, which you can use
+    # to call your signer:
+    #
+    #
+    #     require "aws-sdk-cloudfront"
+    #
+    #     signer = Aws::CloudFront::UrlSigner.new(
+    #       key_pair_id:      "cf-keypair-id",
+    #       private_key_path: "./cf_private_key.pem"
+    #     )
+    #
+    #     Shrine::Storage::S3.new(signer: signer.method(:signed_url))
+    #     # or
+    #     Shrine::Storage::S3.new(signer: -> (url, **options) { signer.signed_url(url, **options) })
+    #
     # ## Accelerate endpoint
     #
     # To use Amazon S3's [Transfer Acceleration] feature, you can change the
@@ -177,10 +211,12 @@ class Shrine
     # [aws-sdk-s3]: https://github.com/aws/aws-sdk-ruby/tree/master/gems/aws-sdk-s3
     # [Transfer Acceleration]: http://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html
     # [object lifecycle]: http://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Object.html#put-instance_method
+    # [serve private content via CloudFront]: https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/PrivateContent.html
+    # [`Aws::CloudFront::UrlSigner`]: https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/CloudFront/UrlSigner.html
     class S3
       MIN_PART_SIZE = 5 * 1024 * 1024 # 5MB
 
-      attr_reader :client, :bucket, :prefix, :host, :upload_options
+      attr_reader :client, :bucket, :prefix, :host, :upload_options, :signer, :public
 
       # Initializes a storage for uploading to S3. All options are forwarded to
       # [`Aws::S3::Client#initialize`], except the following:
@@ -213,7 +249,7 @@ class Shrine
       # [`Aws::S3::Bucket#presigned_post`]: http://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Object.html#presigned_post-instance_method
       # [`Aws::S3::Client#initialize`]: http://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html#initialize-instance_method
       # [configuring AWS SDK]: https://docs.aws.amazon.com/sdk-for-ruby/v3/developer-guide/setup-config.html
-      def initialize(bucket:, prefix: nil, host: nil, upload_options: {}, multipart_threshold: {}, **s3_options)
+      def initialize(bucket:, prefix: nil, host: nil, upload_options: {}, multipart_threshold: {}, signer: nil, public: nil, **s3_options)
         Shrine.deprecation("The :host option to Shrine::Storage::S3#initialize is deprecated and will be removed in Shrine 3. Pass :host to S3#url instead, you can also use default_url_options plugin.") if host
         resource = Aws::S3::Resource.new(**s3_options)
 
@@ -229,6 +265,8 @@ class Shrine
         @host = host
         @upload_options = upload_options
         @multipart_threshold = multipart_threshold
+        @signer = signer
+        @public = public
       end
 
       # Returns an `Aws::S3::Resource` object.
@@ -250,9 +288,10 @@ class Shrine
         options = {}
         options[:content_type] = content_type if content_type
         options[:content_disposition] = "inline; filename=\"#{filename}\"" if filename
+        options[:acl] = "public-read" if public
 
-        options.update(@upload_options)
-        options.update(upload_options)
+        options.merge!(@upload_options)
+        options.merge!(upload_options)
 
         options[:content_disposition] = encode_content_disposition(options[:content_disposition]) if options[:content_disposition]
 
@@ -306,11 +345,6 @@ class Shrine
 
       # Returns the presigned URL to the file.
       #
-      # :public
-      # :  Controls whether the URL is signed (`false`) or unsigned (`true`).
-      #    Note that for unsigned URLs the S3 bucket need to be modified to allow
-      #    public URLs. Defaults to `false`.
-      #
       # :host
       # :  This option replaces the host part of the returned URL, and is
       #    typically useful for setting CDN hosts (e.g.
@@ -326,11 +360,11 @@ class Shrine
       #
       # [`Aws::S3::Object#presigned_url`]: http://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Object.html#presigned_url-instance_method
       # [`Aws::S3::Object#public_url`]: http://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Object.html#public_url-instance_method
-      def url(id, download: nil, public: nil, host: self.host, **options)
+      def url(id, download: nil, public: self.public, host: self.host, **options)
         options[:response_content_disposition] ||= "attachment" if download
         options[:response_content_disposition] = encode_content_disposition(options[:response_content_disposition]) if options[:response_content_disposition]
 
-        if public
+        if public || signer
           url = object(id).public_url(**options)
         else
           url = object(id).presigned_url(:get, **options)
@@ -338,8 +372,12 @@ class Shrine
 
         if host
           uri = URI.parse(url)
-          uri.path = uri.path.match(/^\/#{bucket.name}/).post_match unless uri.host.include?(bucket.name) || client.config.force_path_style
-          url = URI.join(host, uri.request_uri).to_s
+          uri.path = uri.path.match(/^\/#{bucket.name}/).post_match unless uri.host.include?(bucket.name)
+          url = URI.join(host, uri.request_uri[1..-1]).to_s
+        end
+
+        if signer
+          url = signer.call(url, **options)
         end
 
         url
@@ -353,8 +391,13 @@ class Shrine
       #
       # [`Aws::S3::Object#presigned_post`]: http://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Object.html#presigned_post-instance_method
       # [`Aws::S3::Object#presigned_url`]: https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Object.html#presigned_url-instance_method
-      def presign(id, method: :post, **options)
-        options = @upload_options.merge(options)
+      def presign(id, method: :post, **presign_options)
+        options = {}
+        options[:acl] = "public-read" if public
+
+        options.merge!(@upload_options)
+        options.merge!(presign_options)
+
         options[:content_disposition] = encode_content_disposition(options[:content_disposition]) if options[:content_disposition]
 
         if method == :post
@@ -427,33 +470,43 @@ class Shrine
 
       # Uploads the file to S3. Uses multipart upload for large files.
       def put(io, id, **options)
-        if io.respond_to?(:path)
-          path = io.path
-        elsif io.is_a?(UploadedFile) && defined?(Storage::FileSystem) && io.storage.is_a?(Storage::FileSystem)
-          path = io.storage.path(io.id).to_s
-        end
+        bytes_uploaded = nil
 
-        if path
+        if (path = extract_path(io))
           # use `upload_file` for files because it can do multipart upload
           options = { multipart_threshold: @multipart_threshold[:upload] }.merge!(options)
           object(id).upload_file(path, **options)
-          File.size(path)
+          bytes_uploaded = File.size(path)
         else
           io.to_io if io.is_a?(UploadedFile) # open if not already opened
 
-          if io.respond_to?(:size) && io.size
+          if io.respond_to?(:size) && io.size && (io.size <= @multipart_threshold[:upload] || !object(id).respond_to?(:upload_stream))
             object(id).put(body: io, **options)
-            io.size
+            bytes_uploaded = io.size
+          elsif object(id).respond_to?(:upload_stream)
+            # `upload_stream` uses multipart upload
+            object(id).upload_stream(tempfile: true, **options) do |write_stream|
+              bytes_uploaded = IO.copy_stream(io, write_stream)
+            end
           else
-            # IO has unknown size, so we have to use multipart upload
-            multipart_put(io, id, **options)
+            Shrine.deprecation "Uploading a file of unknown size with aws-sdk-s3 older than 1.14 is deprecated and will be removed in Shrine 3. Update to aws-sdk-s3 1.14 or higher."
+
+            Tempfile.create("shrine-s3", binmode: true) do |file|
+              bytes_uploaded = IO.copy_stream(io, file.path)
+              object(id).upload_file(file.path, **options)
+            end
           end
         end
+
+        bytes_uploaded
       end
 
-      # Uploads the file to S3 using multipart upload.
-      def multipart_put(io, id, **options)
-        MultipartUploader.new(object(id)).upload(io, **options)
+      def extract_path(io)
+        if io.respond_to?(:path)
+          io.path
+        elsif io.is_a?(UploadedFile) && defined?(Storage::FileSystem) && io.storage.is_a?(Storage::FileSystem)
+          io.storage.path(io.id).to_s
+        end
       end
 
       # The file is copyable if it's on S3 and on the same Amazon account.
@@ -480,52 +533,6 @@ class Shrine
           CGI.escape(filename).gsub("+", " ")
         end
       end
-
-      # Uploads IO objects of unknown size using the multipart API.
-      class MultipartUploader
-        def initialize(object)
-          @object = object
-        end
-
-        # Initiates multipart upload, uploads IO content into multiple parts,
-        # and completes the multipart upload. If an exception is raised, the
-        # multipart upload is automatically aborted.
-        def upload(io, **options)
-          multipart_upload = @object.initiate_multipart_upload(**options)
-
-          parts = upload_parts(multipart_upload, io)
-          bytes_uploaded = parts.inject(0) { |size, part| size + part.delete(:size) }
-
-          multipart_upload.complete(multipart_upload: { parts: parts })
-
-          bytes_uploaded
-        rescue
-          multipart_upload.abort if multipart_upload
-          raise
-        end
-
-        # Uploads parts until the IO object has reached EOF.
-        def upload_parts(multipart_upload, io)
-          1.step.inject([]) do |parts, part_number|
-            parts << upload_part(multipart_upload, io, part_number)
-            break parts if io.eof?
-            parts
-          end
-        end
-
-        # Uploads at most 5MB of IO content into a single multipart part.
-        def upload_part(multipart_upload, io, part_number)
-          Tempfile.create("shrine-s3-part-#{part_number}", binmode: true) do |body|
-            IO.copy_stream(io, body, MIN_PART_SIZE)
-            body.rewind
-
-            multipart_part = multipart_upload.part(part_number)
-            response       = multipart_part.upload(body: body)
-
-            { part_number: part_number, size: body.size, etag: response.etag }
-          end
-        end
-      end
     end
   end
 end