activestorage 5.2.4.4 → 6.1.1

data/lib/active_storage/service/gcs_service.rb

@@ -1,31 +1,40 @@
 # frozen_string_literal: true
 
-gem "google-cloud-storage", "~> 1.8"
-
+gem "google-cloud-storage", "~> 1.11"
 require "google/cloud/storage"
-require "net/http"
-
-require "active_support/core_ext/object/to_query"
 
 module ActiveStorage
   # Wraps the Google Cloud Storage as an Active Storage service. See ActiveStorage::Service for the generic API
   # documentation that applies to all services.
   class Service::GCSService < Service
-    def initialize(**config)
+    def initialize(public: false, **config)
       @config = config
+      @public = public
     end
 
     def upload(key, io, checksum: nil, content_type: nil, disposition: nil, filename: nil)
       instrument :upload, key: key, checksum: checksum do
-        begin
-          # GCS's signed URLs don't include params such as response-content-type response-content_disposition
-          # in the signature, which means an attacker can modify them and bypass our effort to force these to
-          # binary and attachment when the file's content type requires it. The only way to force them is to
-          # store them as object's metadata.
-          content_disposition = content_disposition_with(type: disposition, filename: filename) if disposition && filename
-          bucket.create_file(io, key, md5: checksum, content_type: content_type, content_disposition: content_disposition)
-        rescue Google::Cloud::InvalidArgumentError
-          raise ActiveStorage::IntegrityError
+        # GCS's signed URLs don't include params such as response-content-type response-content_disposition
+        # in the signature, which means an attacker can modify them and bypass our effort to force these to
+        # binary and attachment when the file's content type requires it. The only way to force them is to
+        # store them as object's metadata.
+        content_disposition = content_disposition_with(type: disposition, filename: filename) if disposition && filename
+        bucket.create_file(io, key, md5: checksum, content_type: content_type, content_disposition: content_disposition)
+      rescue Google::Cloud::InvalidArgumentError
+        raise ActiveStorage::IntegrityError
+      end
+    end
+
+    def download(key, &block)
+      if block_given?
+        instrument :streaming_download, key: key do
+          stream(key, &block)
+        end
+      else
+        instrument :download, key: key do
+          file_for(key).download.string
+        rescue Google::Cloud::NotFoundError
+          raise ActiveStorage::FileNotFoundError
         end
       end
     end
@@ -39,49 +48,28 @@ module ActiveStorage
       end
     end
 
-    # FIXME: Download in chunks when given a block.
-    def download(key)
-      instrument :download, key: key do
-        io = file_for(key).download
-        io.rewind
-
-        if block_given?
-          yield io.string
-        else
-          io.string
-        end
-      end
-    end
-
     def download_chunk(key, range)
       instrument :download_chunk, key: key, range: range do
-        file = file_for(key)
-        uri  = URI(file.signed_url(expires: 30.seconds))
-
-        Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |client|
-          client.get(uri, "Range" => "bytes=#{range.begin}-#{range.exclude_end? ? range.end - 1 : range.end}").body
-        end
+        file_for(key).download(range: range).string
+      rescue Google::Cloud::NotFoundError
+        raise ActiveStorage::FileNotFoundError
       end
     end
 
     def delete(key)
       instrument :delete, key: key do
-        begin
-          file_for(key).delete
-        rescue Google::Cloud::NotFoundError
-          # Ignore files already deleted
-        end
+        file_for(key).delete
+      rescue Google::Cloud::NotFoundError
+        # Ignore files already deleted
       end
     end
 
     def delete_prefixed(prefix)
       instrument :delete_prefixed, prefix: prefix do
         bucket.files(prefix: prefix).all do |file|
-          begin
-            file.delete
-          rescue Google::Cloud::NotFoundError
-            # Ignore concurrently-deleted files
-          end
+          file.delete
+        rescue Google::Cloud::NotFoundError
+          # Ignore concurrently-deleted files
         end
       end
     end
@@ -94,19 +82,6 @@ module ActiveStorage
       end
     end
 
-    def url(key, expires_in:, filename:, content_type:, disposition:)
-      instrument :url, key: key do |payload|
-        generated_url = file_for(key).signed_url expires: expires_in, query: {
-          "response-content-disposition" => content_disposition_with(type: disposition, filename: filename),
-          "response-content-type" => content_type
-        }
-
-        payload[:url] = generated_url
-
-        generated_url
-      end
-    end
-
     def url_for_direct_upload(key, expires_in:, checksum:, **)
       instrument :url, key: key do |payload|
         generated_url = bucket.signed_url key, method: "PUT", expires: expires_in, content_md5: checksum
@@ -117,23 +92,52 @@ module ActiveStorage
       end
     end
 
-    def headers_for_direct_upload(key, checksum:, **)
-      { "Content-MD5" => checksum }
+    def headers_for_direct_upload(key, checksum:, filename: nil, disposition: nil, **)
+      content_disposition = content_disposition_with(type: disposition, filename: filename) if filename
+
+      { "Content-MD5" => checksum, "Content-Disposition" => content_disposition }
     end
 
     private
+      def private_url(key, expires_in:, filename:, content_type:, disposition:, **)
+        file_for(key).signed_url expires: expires_in, query: {
+          "response-content-disposition" => content_disposition_with(type: disposition, filename: filename),
+          "response-content-type" => content_type
+        }
+      end
+
+      def public_url(key, **)
+        file_for(key).public_url
+      end
+
+
       attr_reader :config
 
-      def file_for(key)
-        bucket.file(key, skip_lookup: true)
+      def file_for(key, skip_lookup: true)
+        bucket.file(key, skip_lookup: skip_lookup)
+      end
+
+      # Reads the file for the given key in chunks, yielding each to the block.
+      def stream(key)
+        file = file_for(key, skip_lookup: false)
+
+        chunk_size = 5.megabytes
+        offset = 0
+
+        raise ActiveStorage::FileNotFoundError unless file.present?
+
+        while offset < file.size
+          yield file.download(range: offset..(offset + chunk_size - 1)).string
+          offset += chunk_size
+        end
       end
 
       def bucket
-        @bucket ||= client.bucket(config.fetch(:bucket))
+        @bucket ||= client.bucket(config.fetch(:bucket), skip_lookup: true)
       end
 
       def client
-        @client ||= Google::Cloud::Storage.new(config.except(:bucket))
+        @client ||= Google::Cloud::Storage.new(**config.except(:bucket))
       end
   end
 end

data/lib/active_storage/service/mirror_service.rb

@@ -4,18 +4,26 @@ require "active_support/core_ext/module/delegation"
 
 module ActiveStorage
   # Wraps a set of mirror services and provides a single ActiveStorage::Service object that will all
-  # have the files uploaded to them. A +primary+ service is designated to answer calls to +download+, +exists?+,
-  # and +url+.
+  # have the files uploaded to them. A +primary+ service is designated to answer calls to:
+  # * +download+
+  # * +exists?+
+  # * +url+
+  # * +url_for_direct_upload+
+  # * +headers_for_direct_upload+
   class Service::MirrorService < Service
     attr_reader :primary, :mirrors
 
-    delegate :download, :download_chunk, :exist?, :url, :path_for, to: :primary
+    delegate :download, :download_chunk, :exist?, :url,
+      :url_for_direct_upload, :headers_for_direct_upload, :path_for, to: :primary
 
     # Stitch together from named services.
-    def self.build(primary:, mirrors:, configurator:, **options) #:nodoc:
-      new \
+    def self.build(primary:, mirrors:, name:, configurator:, **options) #:nodoc:
+      new(
         primary: configurator.build(primary),
-        mirrors: mirrors.collect { |name| configurator.build name }
+        mirrors: mirrors.collect { |mirror_name| configurator.build mirror_name }
+      ).tap do |service_instance|
+        service_instance.name = name
+      end
     end
 
     def initialize(primary:, mirrors:)
@@ -26,7 +34,8 @@ module ActiveStorage
     # ensure a match when the upload has completed or raise an ActiveStorage::IntegrityError.
     def upload(key, io, checksum: nil, **options)
       each_service.collect do |service|
-        service.upload key, io.tap(&:rewind), checksum: checksum, **options
+        io.rewind
+        service.upload key, io, checksum: checksum, **options
       end
     end
 
@@ -40,6 +49,21 @@ module ActiveStorage
       perform_across_services :delete_prefixed, prefix
     end
 
+
+    # Copy the file at the +key+ from the primary service to each of the mirrors where it doesn't already exist.
+    def mirror(key, checksum:)
+      instrument :mirror, key: key, checksum: checksum do
+        if (mirrors_in_need_of_mirroring = mirrors.select { |service| !service.exist?(key) }).any?
+          primary.open(key, checksum: checksum) do |io|
+            mirrors_in_need_of_mirroring.each do |service|
+              io.rewind
+              service.upload key, io, checksum: checksum
+            end
+          end
+        end
+      end
+    end
+
     private
       def each_service(&block)
         [ primary, *mirrors ].each(&block)

data/lib/active_storage/service/registry.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module ActiveStorage
+  class Service::Registry #:nodoc:
+    def initialize(configurations)
+      @configurations = configurations.deep_symbolize_keys
+      @services = {}
+    end
+
+    def fetch(name)
+      services.fetch(name.to_sym) do |key|
+        if configurations.include?(key)
+          services[key] = configurator.build(key)
+        else
+          if block_given?
+            yield key
+          else
+            raise KeyError, "Missing configuration for the #{key} Active Storage service. " \
+              "Configurations available for the #{configurations.keys.to_sentence} services."
+          end
+        end
+      end
+    end
+
+    private
+      attr_reader :configurations, :services
+
+      def configurator
+        @configurator ||= ActiveStorage::Service::Configurator.new(configurations)
+      end
+  end
+end

data/lib/active_storage/service/s3_service.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+gem "aws-sdk-s3", "~> 1.48"
+
 require "aws-sdk-s3"
 require "active_support/core_ext/numeric/bytes"
 
@@ -7,21 +9,28 @@ module ActiveStorage
   # Wraps the Amazon Simple Storage Service (S3) as an Active Storage service.
   # See ActiveStorage::Service for the generic API documentation that applies to all services.
   class Service::S3Service < Service
-    attr_reader :client, :bucket, :upload_options
+    attr_reader :client, :bucket
+    attr_reader :multipart_upload_threshold, :upload_options
 
-    def initialize(bucket:, upload: {}, **options)
+    def initialize(bucket:, upload: {}, public: false, **options)
       @client = Aws::S3::Resource.new(**options)
       @bucket = @client.bucket(bucket)
 
+      @multipart_upload_threshold = upload.delete(:multipart_threshold) || 100.megabytes
+      @public = public
+
       @upload_options = upload
+      @upload_options[:acl] = "public-read" if public?
     end
 
-    def upload(key, io, checksum: nil, **)
+    def upload(key, io, checksum: nil, filename: nil, content_type: nil, disposition: nil, **)
       instrument :upload, key: key, checksum: checksum do
-        begin
-          object_for(key).put(upload_options.merge(body: io, content_md5: checksum))
-        rescue Aws::S3::Errors::BadDigest
-          raise ActiveStorage::IntegrityError
+        content_disposition = content_disposition_with(filename: filename, type: disposition) if disposition && filename
+
+        if io.size < multipart_upload_threshold
+          upload_with_single_part key, io, checksum: checksum, content_type: content_type, content_disposition: content_disposition
+        else
+          upload_with_multipart key, io, content_type: content_type, content_disposition: content_disposition
         end
       end
     end
@@ -34,13 +43,17 @@ module ActiveStorage
       else
         instrument :download, key: key do
           object_for(key).get.body.string.force_encoding(Encoding::BINARY)
+        rescue Aws::S3::Errors::NoSuchKey
+          raise ActiveStorage::FileNotFoundError
         end
       end
     end
 
     def download_chunk(key, range)
       instrument :download_chunk, key: key, range: range do
-        object_for(key).get(range: "bytes=#{range.begin}-#{range.exclude_end? ? range.end - 1 : range.end}").body.read.force_encoding(Encoding::BINARY)
+        object_for(key).get(range: "bytes=#{range.begin}-#{range.exclude_end? ? range.end - 1 : range.end}").body.string.force_encoding(Encoding::BINARY)
+      rescue Aws::S3::Errors::NoSuchKey
+        raise ActiveStorage::FileNotFoundError
       end
     end
 
@@ -64,23 +77,11 @@ module ActiveStorage
       end
     end
 
-    def url(key, expires_in:, filename:, disposition:, content_type:)
-      instrument :url, key: key do |payload|
-        generated_url = object_for(key).presigned_url :get, expires_in: expires_in.to_i,
-          response_content_disposition: content_disposition_with(type: disposition, filename: filename),
-          response_content_type: content_type
-
-        payload[:url] = generated_url
-
-        generated_url
-      end
-    end
-
     def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:)
       instrument :url, key: key do |payload|
         generated_url = object_for(key).presigned_url :put, expires_in: expires_in.to_i,
           content_type: content_type, content_length: content_length, content_md5: checksum,
-          whitelist_headers: ['content-length']
+          whitelist_headers: ["content-length"], **upload_options
 
         payload[:url] = generated_url
 
@@ -88,11 +89,42 @@ module ActiveStorage
       end
     end
 
-    def headers_for_direct_upload(key, content_type:, checksum:, **)
-      { "Content-Type" => content_type, "Content-MD5" => checksum }
+    def headers_for_direct_upload(key, content_type:, checksum:, filename: nil, disposition: nil, **)
+      content_disposition = content_disposition_with(type: disposition, filename: filename) if filename
+
+      { "Content-Type" => content_type, "Content-MD5" => checksum, "Content-Disposition" => content_disposition }
     end
 
     private
+      def private_url(key, expires_in:, filename:, disposition:, content_type:, **)
+        object_for(key).presigned_url :get, expires_in: expires_in.to_i,
+          response_content_disposition: content_disposition_with(type: disposition, filename: filename),
+          response_content_type: content_type
+      end
+
+      def public_url(key, **)
+        object_for(key).public_url
+      end
+
+
+      MAXIMUM_UPLOAD_PARTS_COUNT = 10000
+      MINIMUM_UPLOAD_PART_SIZE   = 5.megabytes
+
+      def upload_with_single_part(key, io, checksum: nil, content_type: nil, content_disposition: nil)
+        object_for(key).put(body: io, content_md5: checksum, content_type: content_type, content_disposition: content_disposition, **upload_options)
+      rescue Aws::S3::Errors::BadDigest
+        raise ActiveStorage::IntegrityError
+      end
+
+      def upload_with_multipart(key, io, content_type: nil, content_disposition: nil)
+        part_size = [ io.size.fdiv(MAXIMUM_UPLOAD_PARTS_COUNT).ceil, MINIMUM_UPLOAD_PART_SIZE ].max
+
+        object_for(key).upload_stream(content_type: content_type, content_disposition: content_disposition, part_size: part_size, **upload_options) do |out|
+          IO.copy_stream(io, out)
+        end
+      end
+
+
       def object_for(key)
         bucket.object(key)
       end
@@ -104,8 +136,10 @@ module ActiveStorage
         chunk_size = 5.megabytes
         offset = 0
 
+        raise ActiveStorage::FileNotFoundError unless object.exists?
+
         while offset < object.content_length
-          yield object.get(range: "bytes=#{offset}-#{offset + chunk_size - 1}").body.read.force_encoding(Encoding::BINARY)
+          yield object.get(range: "bytes=#{offset}-#{offset + chunk_size - 1}").body.string.force_encoding(Encoding::BINARY)
           offset += chunk_size
         end
       end

data/lib/active_storage/transformers/image_processing_transformer.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+begin
+  require "image_processing"
+rescue LoadError
+  raise LoadError, <<~ERROR.squish
+    Generating image variants require the image_processing gem.
+    Please add `gem 'image_processing', '~> 1.2'` to your Gemfile.
+  ERROR
+end
+
+module ActiveStorage
+  module Transformers
+    class ImageProcessingTransformer < Transformer
+      private
+        def process(file, format:)
+          processor.
+            source(file).
+            loader(page: 0).
+            convert(format).
+            apply(operations).
+            call
+        end
+
+        def processor
+          ImageProcessing.const_get(ActiveStorage.variant_processor.to_s.camelize)
+        end
+
+        def operations
+          transformations.each_with_object([]) do |(name, argument), list|
+            if name.to_s == "combine_options"
+              raise ArgumentError, <<~ERROR.squish
+                Active Storage's ImageProcessing transformer doesn't support :combine_options,
+                as it always generates a single ImageMagick command.
+              ERROR
+            end
+
+            if argument.present?
+              list << [ name, argument ]
+            end
+          end
+        end
+    end
+  end
+end