shrine 2.19.3 → 3.6.0

data/lib/shrine/storage/s3.rb

@@ -1,34 +1,27 @@
 # frozen_string_literal: true
 
+gem "aws-sdk-s3", "~> 1.14"
+
 require "shrine"
-begin
-  require "aws-sdk-s3"
-  if Gem::Version.new(Aws::S3::GEM_VERSION) < Gem::Version.new("1.2.0")
-    raise "Shrine::Storage::S3 requires aws-sdk-s3 version 1.2.0 or above"
-  elsif Gem::Version.new(Aws::S3::GEM_VERSION) < Gem::Version.new("1.14.0")
-    Shrine.deprecation("Using aws-sdk-s3 < 1.14 is deprecated and support for it will be removed in Shrine 3. Update to aws-sdk-s3 version 1.14 or higher")
-  end
-rescue LoadError => exception
-  begin
-    require "aws-sdk"
-    Shrine.deprecation("Using aws-sdk 2.x is deprecated and support for it will be removed in Shrine 3. Use the new aws-sdk-s3 gem instead.")
-    Aws.eager_autoload!(services: ["S3"])
-  rescue LoadError
-    raise exception
-  end
-end
+require "aws-sdk-s3"
 
 require "down/chunked_io"
 require "content_disposition"
 
 require "uri"
-require "cgi"
 require "tempfile"
 
 class Shrine
   module Storage
     class S3
-      attr_reader :client, :bucket, :prefix, :host, :upload_options, :signer, :public
+      attr_reader :client, :bucket, :prefix, :upload_options, :copy_options, :signer, :public
+
+      MAX_MULTIPART_PARTS = 10_000
+      MIN_PART_SIZE       = 5*1024*1024
+
+      MULTIPART_THRESHOLD = { upload: 15*1024*1024, copy: 100*1024*1024 }
+
+      COPY_OPTIONS = { tagging_directive: "REPLACE" }
 
       # Initializes a storage for uploading to S3. All options are forwarded to
       # [`Aws::S3::Client#initialize`], except the following:
@@ -50,12 +43,20 @@ class Shrine
       #   be passed to [`Aws::S3::Object#put`], [`Aws::S3::Object#copy_from`]
       #   and [`Aws::S3::Bucket#presigned_post`].
       #
+      # :copy_options
+      # : Additional options that will be used for copying files, they will
+      #   be passed to [`Aws::S3::Object#copy_from`].
+      #
       # :multipart_threshold
       # : If the input file is larger than the specified size, a parallelized
       #   multipart will be used for the upload/copy. Defaults to
       #   `{upload: 15*1024*1024, copy: 100*1024*1024}` (15MB for upload
       #   requests, 100MB for copy requests).
       #
+      # :max_multipart_parts
+      # : Limits the number of parts if parellized multipart upload/copy is used.
+      # Defaults to 10_000.
+      #
       # In addition to specifying the `:bucket`, you'll also need to provide
       # AWS credentials. The most common way is to provide them directly via
       # `:access_key_id`, `:secret_access_key`, and `:region` options. But you
@@ -67,33 +68,20 @@ class Shrine
       # [`Aws::S3::Bucket#presigned_post`]: http://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Object.html#presigned_post-instance_method
       # [`Aws::S3::Client#initialize`]: http://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html#initialize-instance_method
       # [configuring AWS SDK]: https://docs.aws.amazon.com/sdk-for-ruby/v3/developer-guide/setup-config.html
-      def initialize(bucket:, client: nil, prefix: nil, host: nil, upload_options: {}, multipart_threshold: {}, signer: nil, public: nil, **s3_options)
+      def initialize(bucket:, client: nil, prefix: nil, upload_options: {}, multipart_threshold: {}, max_multipart_parts: nil, signer: nil, public: nil, copy_options: COPY_OPTIONS, **s3_options)
         raise ArgumentError, "the :bucket option is nil" unless bucket
 
-        Shrine.deprecation("The :host option to Shrine::Storage::S3#initialize is deprecated and will be removed in Shrine 3. Pass :host to S3#url instead, you can also use default_url_options plugin.") if host
-
-        if multipart_threshold.is_a?(Integer)
-          Shrine.deprecation("Accepting the :multipart_threshold S3 option as an integer is deprecated, use a hash with :upload and :copy keys instead, e.g. {upload: 15*1024*1024, copy: 150*1024*1024}")
-          multipart_threshold = { upload: multipart_threshold }
-        end
-        multipart_threshold = { upload: 15*1024*1024, copy: 100*1024*1024 }.merge(multipart_threshold)
-
         @client = client || Aws::S3::Client.new(**s3_options)
         @bucket = Aws::S3::Bucket.new(name: bucket, client: @client)
         @prefix = prefix
-        @host = host
         @upload_options = upload_options
-        @multipart_threshold = multipart_threshold
+        @copy_options = copy_options
+        @multipart_threshold = MULTIPART_THRESHOLD.merge(multipart_threshold)
+        @max_multipart_parts = max_multipart_parts || MAX_MULTIPART_PARTS
         @signer = signer
         @public = public
       end
 
-      # Returns an `Aws::S3::Resource` object.
-      def s3
-        Shrine.deprecation("Shrine::Storage::S3#s3 that returns an Aws::S3::Resource is deprecated, use Shrine::Storage::S3#client which returns an Aws::S3::Client object.")
-        Aws::S3::Resource.new(client: @client)
-      end
-
       # If the file is an UploadedFile from S3, issues a COPY command, otherwise
       # uploads the file. For files larger than `:multipart_threshold` a
       # multipart upload/copy will be used for better performance and more
@@ -112,8 +100,6 @@ class Shrine
         options.merge!(@upload_options)
         options.merge!(upload_options)
 
-        options[:content_disposition] = encode_content_disposition(options[:content_disposition]) if options[:content_disposition]
-
         if copyable?(io)
           copy(io, id, **options)
         else
@@ -124,22 +110,18 @@ class Shrine
       # Returns a `Down::ChunkedIO` object that downloads S3 object content
       # on-demand. By default, read content will be cached onto disk so that
       # it can be rewinded, but if you don't need that you can pass
-      # `rewindable: false`.
+      # `rewindable: false`. A required character encoding can be passed in
+      # `encoding`; the default is `Encoding::BINARY` via `Down::ChunkedIO`.
       #
       # Any additional options are forwarded to [`Aws::S3::Object#get`].
       #
       # [`Aws::S3::Object#get`]: http://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Object.html#get-instance_method
-      def open(id, rewindable: true, **options)
-        object = object(id)
-
-        load_data(object, **options)
+      def open(id, rewindable: true, encoding: nil, **options)
+        chunks, length = get(id, **options)
 
-        Down::ChunkedIO.new(
-          chunks:     object.enum_for(:get, **options),
-          rewindable: rewindable,
-          size:       object.content_length,
-          data:       { object: object },
-        )
+        Down::ChunkedIO.new(chunks: chunks, rewindable: rewindable, size: length, encoding: encoding)
+      rescue Aws::S3::Errors::NoSuchKey
+        raise Shrine::FileNotFound, "file #{id.inspect} not found on storage"
       end
 
       # Returns true file exists on S3.
@@ -163,13 +145,7 @@ class Shrine
       #
       # [`Aws::S3::Object#presigned_url`]: http://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Object.html#presigned_url-instance_method
       # [`Aws::S3::Object#public_url`]: http://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Object.html#public_url-instance_method
-      def url(id, download: nil, public: self.public, host: self.host, **options)
-        if download
-          Shrine.deprecation("The :download option in Shrine::Storage::S3#url is deprecated and will be removed in Shrine 3. Use the :response_content_disposition option directly, e.g. `response_content_disposition: \"attachment\"`.")
-          options[:response_content_disposition] ||= "attachment"
-        end
-        options[:response_content_disposition] = encode_content_disposition(options[:response_content_disposition]) if options[:response_content_disposition]
-
+      def url(id, public: self.public, host: nil, **options)
         if public || signer
           url = object(id).public_url(**options)
         else
@@ -217,27 +193,7 @@ class Shrine
         options.merge!(@upload_options)
         options.merge!(presign_options)
 
-        options[:content_disposition] = encode_content_disposition(options[:content_disposition]) if options[:content_disposition]
-
-        if method == :post
-          presigned_post = object(id).presigned_post(options)
-
-          Struct.new(:method, :url, :fields).new(method, presigned_post.url, presigned_post.fields)
-        else
-          url = object(id).presigned_url(method, options)
-
-          # When any of these options are specified, the corresponding request
-          # headers must be included in the upload request.
-          headers = {}
-          headers["Content-Length"]      = options[:content_length]      if options[:content_length]
-          headers["Content-Type"]        = options[:content_type]        if options[:content_type]
-          headers["Content-Disposition"] = options[:content_disposition] if options[:content_disposition]
-          headers["Content-Encoding"]    = options[:content_encoding]    if options[:content_encoding]
-          headers["Content-Language"]    = options[:content_language]    if options[:content_language]
-          headers["Content-MD5"]         = options[:content_md5]         if options[:content_md5]
-
-          { method: method, url: url, headers: headers }
-        end
+        send(:"presign_#{method}", id, options)
       end
 
       # Deletes the file from the storage.
@@ -245,6 +201,16 @@ class Shrine
         object(id).delete
       end
 
+      # Deletes objects at keys starting with the specified prefix.
+      #
+      #    s3.delete_prefixed("somekey/derivatives/")
+      def delete_prefixed(delete_prefix)
+        # We need to make sure to combine with storage prefix, and
+        # that it ends in '/' cause S3 can be squirrely about matching interior.
+        delete_prefix = delete_prefix.chomp("/") + "/"
+        bucket.objects(prefix: [*prefix, delete_prefix].join("/")).batch_delete!
+      end
+
       # If block is given, deletes all objects from the storage for which the
       # block evaluates to true. Otherwise deletes all objects from the storage.
       #
@@ -260,76 +226,109 @@ class Shrine
 
       # Returns an `Aws::S3::Object` for the given id.
       def object(id)
-        bucket.object([*prefix, id].join("/"))
+        bucket.object(object_key(id))
       end
 
-      # Catches the deprecated `#download` and `#stream` methods.
-      def method_missing(name, *args, &block)
-        case name
-        when :stream   then deprecated_stream(*args, &block)
-        when :download then deprecated_download(*args, &block)
-        else
-          super
+      private
+
+      # Uploads the file to S3. Uses multipart upload for large files.
+      def put(io, id, **options)
+        if io.respond_to?(:size) && io.size && io.size <= @multipart_threshold[:upload]
+          object(id).put(body: io, **options)
+        else # multipart upload
+          object(id).upload_stream(part_size: part_size(io), **options) do |write_stream|
+            IO.copy_stream(io, write_stream)
+          end
         end
       end
 
-      private
-
       # Copies an existing S3 object to a new location. Uses multipart copy for
       # large files.
-      def copy(io, id, **options)
-        # pass :content_length on multipart copy to avoid an additional HEAD request
-        options = { multipart_copy: true, content_length: io.size }.merge!(options) if io.size && io.size >= @multipart_threshold[:copy]
+      def copy(io, id, **copy_options)
+        # don't inherit source object metadata or AWS tags
+        options = {
+          metadata_directive: "REPLACE",
+        }
+
+        if io.size && io.size >= @multipart_threshold[:copy]
+          # pass :content_length on multipart copy to avoid an additional HEAD request
+          options.merge!(multipart_copy: true, content_length: io.size)
+        end
+
+        options.merge!(@copy_options)
+        options.merge!(copy_options)
+
         object(id).copy_from(io.storage.object(io.id), **options)
       end
 
-      # Uploads the file to S3. Uses multipart upload for large files.
-      def put(io, id, **options)
-        if (path = extract_path(io))
-          # use `upload_file` for files because it can do multipart upload
-          options = { multipart_threshold: @multipart_threshold[:upload] }.merge!(options)
-          object(id).upload_file(path, **options)
-        else
-          io.to_io if io.is_a?(UploadedFile) # open if not already opened
-
-          if io.respond_to?(:size) && io.size && (io.size <= @multipart_threshold[:upload] || !object(id).respond_to?(:upload_stream))
-            object(id).put(body: io, **options)
-          elsif object(id).respond_to?(:upload_stream)
-            # `upload_stream` uses multipart upload
-            object(id).upload_stream(tempfile: true, **options) do |write_stream|
-              IO.copy_stream(io, write_stream)
-            end
-          else
-            Tempfile.create("shrine-s3", binmode: true) do |file|
-              IO.copy_stream(io, file.path)
-              object(id).upload_file(file.path, **options)
-            end
-          end
-        end
+      # Generates parameters for a POST upload request.
+      def presign_post(id, options)
+        presigned_post = object(id).presigned_post(options)
+
+        { method: :post, url: presigned_post.url, fields: presigned_post.fields }
       end
 
-      # Aws::S3::Object#load doesn't support passing options to #head_object,
-      # so we call #head_object ourselves and assign the response data
-      def load_data(object, **options)
-        # filter out #get_object options that are not valid #head_object options
-        options = options.select do |key, value|
-          client.config.api.operation(:head_object).input.shape.member?(key)
-        end
+      # Generates parameters for a PUT upload request.
+      def presign_put(id, options)
+        url = object(id).presigned_url(:put, options)
+
+        # When any of these options are specified, the corresponding request
+        # headers must be included in the upload request.
+        headers = {}
+        headers["Content-Length"]      = options[:content_length]      if options[:content_length]
+        headers["Content-Type"]        = options[:content_type]        if options[:content_type]
+        headers["Content-Disposition"] = options[:content_disposition] if options[:content_disposition]
+        headers["Content-Encoding"]    = options[:content_encoding]    if options[:content_encoding]
+        headers["Content-Language"]    = options[:content_language]    if options[:content_language]
+        headers["Content-MD5"]         = options[:content_md5]         if options[:content_md5]
+
+        { method: :put, url: url, headers: headers }
+      end
 
-        response = client.head_object(
-          bucket: bucket.name,
-          key: object.key,
-          **options
-        )
+      # Determins the part size that should be used when uploading the given IO
+      # object via multipart upload.
+      def part_size(io)
+        return unless io.respond_to?(:size) && io.size
 
-        object.instance_variable_set(:@data, response.data)
+        if io.size <= MIN_PART_SIZE * @max_multipart_parts # <= 50 GB
+          MIN_PART_SIZE
+        else # > 50 GB
+          (io.size.to_f / @max_multipart_parts).ceil
+        end
       end
 
-      def extract_path(io)
-        if io.respond_to?(:path)
-          io.path
-        elsif io.is_a?(UploadedFile) && defined?(Storage::FileSystem) && io.storage.is_a?(Storage::FileSystem)
-          io.storage.path(io.id).to_s
+      # Aws::S3::Object#get doesn't allow us to get the content length of the
+      # object before all content is downloaded, so we hack our way around it.
+      # This way get the content length without an additional HEAD request.
+      if Gem::Version.new(Aws::CORE_GEM_VERSION) >= Gem::Version.new("3.104.0")
+        def get(id, **params)
+          enum = object(id).enum_for(:get, **params)
+
+          begin
+            content_length = Integer(enum.peek.last["content-length"])
+          rescue StopIteration
+            content_length = 0
+          end
+
+          chunks = Enumerator.new { |y| loop { y << enum.next.first } }
+
+          [chunks, content_length]
+        end
+      else
+        def get(id, **params)
+          req = client.build_request(:get_object, bucket: bucket.name, key: object_key(id), **params)
+
+          body = req.enum_for(:send_request)
+          begin
+            body.peek # start the request
+          rescue StopIteration
+            # the S3 object is empty
+          end
+
+          content_length = Integer(req.context.http_response.headers["Content-Length"])
+          chunks         = Enumerator.new { |y| loop { y << body.next } }
+
+          [chunks, content_length]
         end
       end
 
@@ -349,42 +348,56 @@ class Shrine
         end
       end
 
-      # Upload requests will fail if filename has non-ASCII characters, because
-      # of how S3 generates signatures, so we URI-encode them. Most browsers
-      # should automatically URI-decode filenames when downloading.
-      def encode_content_disposition(content_disposition)
-        content_disposition.sub(/(?<=filename=").+(?=")/) do |filename|
-          if filename =~ /[^[:ascii:]]/
-            Shrine.deprecation("Shrine::Storage::S3 will not escape characters in the filename for Content-Disposition header in Shrine 3. Use the content_disposition gem, for example `ContentDisposition.format(disposition: 'inline', filename: '...')`.")
-            CGI.escape(filename).gsub("+", " ")
-          else
-            filename
-          end
-        end
+      # Returns object key with potential prefix.
+      def object_key(id)
+        [*prefix, id].join("/")
       end
 
-      def deprecated_stream(id)
-        Shrine.deprecation("Shrine::Storage::S3#stream is deprecated over calling #each_chunk on S3#open.")
-        object = object(id)
-        object.get { |chunk| yield chunk, object.content_length }
-      end
+      # Adds support for Aws::S3::Encryption::Client.
+      module ClientSideEncryption
+        attr_reader :encryption_client
 
-      def deprecated_download(id, **options)
-        Shrine.deprecation("Shrine::Storage::S3#download is deprecated over S3#open.")
+        # Save the encryption client and continue initialization with normal
+        # client.
+        def initialize(client: nil, **options)
+          return super unless client.class.name.start_with?("Aws::S3::Encryption")
 
-        tempfile = Tempfile.new(["shrine-s3", File.extname(id)], binmode: true)
-        data = object(id).get(response_target: tempfile, **options)
-        tempfile.content_type = data.content_type
-        tempfile.tap(&:open)
-      rescue
-        tempfile.close! if tempfile
-        raise
-      end
+          super(client: client.client, **options)
+          @encryption_client = client
+        end
+
+        private
 
-      # Tempfile with #content_type accessor which represents downloaded files.
-      class Tempfile < ::Tempfile
-        attr_accessor :content_type
+        # Encryption client doesn't support multipart uploads, so we always use
+        # #put_object.
+        def put(io, id, **options)
+          return super unless encryption_client
+
+          encryption_client.put_object(body: io, bucket: bucket.name, key: object_key(id), **options)
+        end
+
+        def get(id, **options)
+          return super unless encryption_client
+
+          # Encryption client v2 warns against streaming download, so we first
+          # download all content into a file.
+          tempfile = Tempfile.new("shrine-s3", binmode: true)
+          response = encryption_client.get_object(response_target: tempfile, bucket: bucket.name, key: object_key(id), **options)
+          tempfile.rewind
+
+          chunks = Enumerator.new do |yielder|
+            begin
+              yielder << tempfile.read(16*1024) until tempfile.eof?
+            ensure
+              tempfile.close!
+            end
+          end
+
+          [chunks, tempfile.size]
+        end
       end
+
+      prepend ClientSideEncryption
     end
   end
 end

data/lib/shrine/uploaded_file.rb

@@ -6,7 +6,6 @@ require "uri"
 
 class Shrine
   # Core class that represents a file uploaded to a storage.
-  # Base implementation is defined in InstanceMethods and ClassMethods.
   class UploadedFile
     @shrine_class = ::Shrine
 
@@ -23,31 +22,24 @@ class Shrine
     end
 
     module InstanceMethods
-      # The hash of information which defines this uploaded file.
-      attr_reader :data
+      # The location where the file was uploaded to the storage.
+      attr_reader :id
 
-      # Initializes the uploaded file with the given data hash.
-      def initialize(data)
-        raise Error, "#{data.inspect} isn't valid uploaded file data" unless data["id"] && data["storage"]
+      # The identifier of the storage the file is uploaded to.
+      attr_reader :storage_key
 
-        @data = data
-        @data["metadata"] ||= {}
-        storage # ensure storage is registered
-      end
+      # A hash of file metadata that was extracted during upload.
+      attr_reader :metadata
 
-      # The location where the file was uploaded to the storage.
-      def id
-        @data.fetch("id")
-      end
+      # Initializes the uploaded file with the given data hash.
+      def initialize(data)
+        @id          = data[:id]              || data["id"]
+        @storage_key = data[:storage]&.to_sym || data["storage"]&.to_sym
+        @metadata    = data[:metadata]        || data["metadata"]        || {}
 
-      # The string identifier of the storage the file is uploaded to.
-      def storage_key
-        @data.fetch("storage")
-      end
+        fail Error, "#{data.inspect} isn't valid uploaded file data" unless @id && @storage_key
 
-      # A hash of file metadata that was extracted during upload.
-      def metadata
-        @data.fetch("metadata")
+        storage # ensure storage is registered
       end
 
       # The filename that was extracted from the uploaded file.
@@ -58,8 +50,9 @@ class Shrine
       # The extension derived from #id if present, otherwise it's derived
       # from #original_filename.
       def extension
-        result = File.extname(id)[1..-1] || File.extname(original_filename.to_s)[1..-1]
-        result.sub!(/\?.+$/, "") if result && id =~ URI::regexp # strip query params for shrine-url
+        identifier = id =~ URI::DEFAULT_PARSER.make_regexp ? id.sub(/\?.+$/, "") : id # strip query params for shrine-url
+        result = File.extname(identifier)[1..-1]
+        result ||= File.extname(original_filename.to_s)[1..-1]
         result.downcase if result
       end
 
@@ -177,6 +170,7 @@ class Shrine
       # opened IO object.
       def close
         io.close if opened?
+        @io = nil
       end
 
       # Returns whether the file has already been opened.
@@ -196,8 +190,8 @@ class Shrine
       end
 
       # Uploads a new file to this file's location and returns it.
-      def replace(io, context = {})
-        uploader.upload(io, context.merge(location: id))
+      def replace(io, **options)
+        uploader.upload(io, **options, location: id)
       end
 
       # Calls `#delete` on the storage, which deletes the file from the
@@ -222,11 +216,16 @@ class Shrine
         data
       end
 
+      # Returns serializable hash representation of the uploaded file.
+      def data
+        { "id" => id, "storage" => storage_key.to_s, "metadata" => metadata }
+      end
+
       # Returns true if the other UploadedFile is uploaded to the same
       # storage and it has the same #id.
       def ==(other)
-        other.is_a?(self.class) &&
-        self.id == other.id &&
+        self.class       == other.class       &&
+        self.id          == other.id          &&
         self.storage_key == other.storage_key
       end
       alias eql? ==
@@ -251,6 +250,11 @@ class Shrine
         self.class.shrine_class
       end
 
+      # Returns simplified inspect output.
+      def inspect
+        "#<#{self.class.inspect} storage=#{storage_key.inspect} id=#{id.inspect} metadata=#{metadata.inspect}>"
+      end
+
       private
 
       # Returns an opened IO object for the uploaded file by calling `#open`
@@ -260,11 +264,7 @@ class Shrine
       end
 
       def _open(**options)
-        if options.any?
-          storage.open(id, **options)
-        else
-          storage.open(id) # some storage implementations might not accept additional arguments
-        end
+        storage.open(id, **options)
       end
     end
 

data/lib/shrine/version.rb

@@ -6,9 +6,9 @@ class Shrine
   end
 
   module VERSION
-    MAJOR = 2
-    MINOR = 19
-    TINY  = 3
+    MAJOR = 3
+    MINOR = 6
+    TINY  = 0
     PRE   = nil
 
     STRING = [MAJOR, MINOR, TINY, PRE].compact.join(".")