activestorage 6.1.7 → 7.1.0

data/lib/active_storage/service/gcs_service.rb

@@ -1,25 +1,31 @@
 # frozen_string_literal: true
 
 gem "google-cloud-storage", "~> 1.11"
+require "google/apis/iamcredentials_v1"
 require "google/cloud/storage"
 
 module ActiveStorage
+  # = Active Storage \GCS \Service
+  #
   # Wraps the Google Cloud Storage as an Active Storage service. See ActiveStorage::Service for the generic API
   # documentation that applies to all services.
   class Service::GCSService < Service
+    class MetadataServerError < ActiveStorage::Error; end
+    class MetadataServerNotFoundError < ActiveStorage::Error; end
+
     def initialize(public: false, **config)
       @config = config
       @public = public
     end
 
-    def upload(key, io, checksum: nil, content_type: nil, disposition: nil, filename: nil)
+    def upload(key, io, checksum: nil, content_type: nil, disposition: nil, filename: nil, custom_metadata: {})
       instrument :upload, key: key, checksum: checksum do
         # GCS's signed URLs don't include params such as response-content-type response-content_disposition
         # in the signature, which means an attacker can modify them and bypass our effort to force these to
         # binary and attachment when the file's content type requires it. The only way to force them is to
         # store them as object's metadata.
         content_disposition = content_disposition_with(type: disposition, filename: filename) if disposition && filename
-        bucket.create_file(io, key, md5: checksum, content_type: content_type, content_disposition: content_disposition)
+        bucket.create_file(io, key, md5: checksum, cache_control: @config[:cache_control], content_type: content_type, content_disposition: content_disposition, metadata: custom_metadata)
       rescue Google::Cloud::InvalidArgumentError
         raise ActiveStorage::IntegrityError
       end
@@ -39,11 +45,12 @@ module ActiveStorage
       end
     end
 
-    def update_metadata(key, content_type:, disposition: nil, filename: nil)
+    def update_metadata(key, content_type:, disposition: nil, filename: nil, custom_metadata: {})
       instrument :update_metadata, key: key, content_type: content_type, disposition: disposition do
         file_for(key).update do |file|
           file.content_type = content_type
           file.content_disposition = content_disposition_with(type: disposition, filename: filename) if disposition && filename
+          file.metadata = custom_metadata
         end
       end
     end
@@ -82,9 +89,35 @@ module ActiveStorage
       end
     end
 
-    def url_for_direct_upload(key, expires_in:, checksum:, **)
+    def url_for_direct_upload(key, expires_in:, checksum:, custom_metadata: {}, **)
       instrument :url, key: key do |payload|
-        generated_url = bucket.signed_url key, method: "PUT", expires: expires_in, content_md5: checksum
+        headers = {}
+        version = :v2
+
+        if @config[:cache_control].present?
+          headers["Cache-Control"] = @config[:cache_control]
+          # v2 signing doesn't support non `x-goog-` headers. Only switch to v4 signing
+          # if necessary for back-compat; v4 limits the expiration of the URL to 7 days
+          # whereas v2 has no limit
+          version = :v4
+        end
+
+        headers.merge!(custom_metadata_headers(custom_metadata))
+
+        args = {
+          content_md5: checksum,
+          expires: expires_in,
+          headers: headers,
+          method: "PUT",
+          version: version,
+        }
+
+        if @config[:iam]
+          args[:issuer] = issuer
+          args[:signer] = signer
+        end
+
+        generated_url = bucket.signed_url(key, **args)
 
         payload[:url] = generated_url
 
@@ -92,18 +125,41 @@ module ActiveStorage
       end
     end
 
-    def headers_for_direct_upload(key, checksum:, filename: nil, disposition: nil, **)
+    def headers_for_direct_upload(key, checksum:, filename: nil, disposition: nil, custom_metadata: {}, **)
       content_disposition = content_disposition_with(type: disposition, filename: filename) if filename
 
-      { "Content-MD5" => checksum, "Content-Disposition" => content_disposition }
+      headers = { "Content-MD5" => checksum, "Content-Disposition" => content_disposition, **custom_metadata_headers(custom_metadata) }
+      if @config[:cache_control].present?
+        headers["Cache-Control"] = @config[:cache_control]
+      end
+
+      headers
+    end
+
+    def compose(source_keys, destination_key, filename: nil, content_type: nil, disposition: nil, custom_metadata: {})
+      bucket.compose(source_keys, destination_key).update do |file|
+        file.content_type = content_type
+        file.content_disposition = content_disposition_with(type: disposition, filename: filename) if disposition && filename
+        file.metadata = custom_metadata
+      end
     end
 
     private
       def private_url(key, expires_in:, filename:, content_type:, disposition:, **)
-        file_for(key).signed_url expires: expires_in, query: {
-          "response-content-disposition" => content_disposition_with(type: disposition, filename: filename),
-          "response-content-type" => content_type
+        args = {
+          expires: expires_in,
+          query: {
+            "response-content-disposition" => content_disposition_with(type: disposition, filename: filename),
+            "response-content-type" => content_type
+          }
         }
+
+        if @config[:iam]
+          args[:issuer] = issuer
+          args[:signer] = signer
+        end
+
+        file_for(key).signed_url(**args)
       end
 
       def public_url(key, **)
@@ -137,7 +193,40 @@ module ActiveStorage
       end
 
       def client
-        @client ||= Google::Cloud::Storage.new(**config.except(:bucket))
+        @client ||= Google::Cloud::Storage.new(**config.except(:bucket, :cache_control, :iam, :gsa_email))
+      end
+
+      def issuer
+        @issuer ||= @config[:gsa_email].presence || email_from_metadata_server
+      end
+
+      def email_from_metadata_server
+        env = Google::Cloud.env
+        raise MetadataServerNotFoundError if !env.metadata?
+
+        email = env.lookup_metadata("instance", "service-accounts/default/email")
+        email.presence or raise MetadataServerError
+      end
+
+      def signer
+        # https://googleapis.dev/ruby/google-cloud-storage/latest/Google/Cloud/Storage/Project.html#signed_url-instance_method
+        lambda do |string_to_sign|
+          iam_client = Google::Apis::IamcredentialsV1::IAMCredentialsService.new
+
+          scopes = ["https://www.googleapis.com/auth/iam"]
+          iam_client.authorization = Google::Auth.get_application_default(scopes)
+
+          request = Google::Apis::IamcredentialsV1::SignBlobRequest.new(
+            payload: string_to_sign
+          )
+          resource = "projects/-/serviceAccounts/#{issuer}"
+          response = iam_client.sign_service_account_blob(resource, request)
+          response.signed_blob
+        end
+      end
+
+      def custom_metadata_headers(metadata)
+        metadata.transform_keys { |key| "x-goog-meta-#{key}" }
       end
   end
 end

data/lib/active_storage/service/mirror_service.rb

@@ -3,6 +3,8 @@
 require "active_support/core_ext/module/delegation"
 
 module ActiveStorage
+  # = Active Storage Mirror \Service
+  #
   # Wraps a set of mirror services and provides a single ActiveStorage::Service object that will all
   # have the files uploaded to them. A +primary+ service is designated to answer calls to:
   # * +download+
@@ -14,10 +16,10 @@ module ActiveStorage
     attr_reader :primary, :mirrors
 
     delegate :download, :download_chunk, :exist?, :url,
-      :url_for_direct_upload, :headers_for_direct_upload, :path_for, to: :primary
+      :url_for_direct_upload, :headers_for_direct_upload, :path_for, :compose, to: :primary
 
     # Stitch together from named services.
-    def self.build(primary:, mirrors:, name:, configurator:, **options) #:nodoc:
+    def self.build(primary:, mirrors:, name:, configurator:, **options) # :nodoc:
       new(
         primary: configurator.build(primary),
         mirrors: mirrors.collect { |mirror_name| configurator.build mirror_name }
@@ -30,13 +32,13 @@ module ActiveStorage
       @primary, @mirrors = primary, mirrors
     end
 
-    # Upload the +io+ to the +key+ specified to all services. If a +checksum+ is provided, all services will
+    # Upload the +io+ to the +key+ specified to all services. The upload to the primary service is done synchronously
+    # whereas the upload to the mirrors is done asynchronously. If a +checksum+ is provided, all services will
     # ensure a match when the upload has completed or raise an ActiveStorage::IntegrityError.
     def upload(key, io, checksum: nil, **options)
-      each_service.collect do |service|
-        io.rewind
-        service.upload key, io, checksum: checksum, **options
-      end
+      io.rewind
+      primary.upload key, io, checksum: checksum, **options
+      mirror_later key, checksum: checksum
     end
 
     # Delete the file at the +key+ on all services.
@@ -49,6 +51,9 @@ module ActiveStorage
       perform_across_services :delete_prefixed, prefix
     end
 
+    def mirror_later(key, checksum:) # :nodoc:
+      ActiveStorage::MirrorJob.perform_later key, checksum: checksum
+    end
 
     # Copy the file at the +key+ from the primary service to each of the mirrors where it doesn't already exist.
     def mirror(key, checksum:)

data/lib/active_storage/service/registry.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module ActiveStorage
-  class Service::Registry #:nodoc:
+  class Service::Registry # :nodoc:
     def initialize(configurations)
       @configurations = configurations.deep_symbolize_keys
       @services = {}

data/lib/active_storage/service/s3_service.rb

@@ -6,6 +6,8 @@ require "aws-sdk-s3"
 require "active_support/core_ext/numeric/bytes"
 
 module ActiveStorage
+  # = Active Storage \S3 \Service
+  #
   # Wraps the Amazon Simple Storage Service (S3) as an Active Storage service.
   # See ActiveStorage::Service for the generic API documentation that applies to all services.
   class Service::S3Service < Service
@@ -23,14 +25,14 @@ module ActiveStorage
       @upload_options[:acl] = "public-read" if public?
     end
 
-    def upload(key, io, checksum: nil, filename: nil, content_type: nil, disposition: nil, **)
+    def upload(key, io, checksum: nil, filename: nil, content_type: nil, disposition: nil, custom_metadata: {}, **)
       instrument :upload, key: key, checksum: checksum do
         content_disposition = content_disposition_with(filename: filename, type: disposition) if disposition && filename
 
         if io.size < multipart_upload_threshold
-          upload_with_single_part key, io, checksum: checksum, content_type: content_type, content_disposition: content_disposition
+          upload_with_single_part key, io, checksum: checksum, content_type: content_type, content_disposition: content_disposition, custom_metadata: custom_metadata
         else
-          upload_with_multipart key, io, content_type: content_type, content_disposition: content_disposition
+          upload_with_multipart key, io, content_type: content_type, content_disposition: content_disposition, custom_metadata: custom_metadata
         end
       end
     end
@@ -77,11 +79,11 @@ module ActiveStorage
       end
     end
 
-    def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:)
+    def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:, custom_metadata: {})
       instrument :url, key: key do |payload|
         generated_url = object_for(key).presigned_url :put, expires_in: expires_in.to_i,
           content_type: content_type, content_length: content_length, content_md5: checksum,
-          whitelist_headers: ["content-length"], **upload_options
+          metadata: custom_metadata, whitelist_headers: ["content-length"], **upload_options
 
         payload[:url] = generated_url
 
@@ -89,37 +91,55 @@ module ActiveStorage
       end
     end
 
-    def headers_for_direct_upload(key, content_type:, checksum:, filename: nil, disposition: nil, **)
+    def headers_for_direct_upload(key, content_type:, checksum:, filename: nil, disposition: nil, custom_metadata: {}, **)
       content_disposition = content_disposition_with(type: disposition, filename: filename) if filename
 
-      { "Content-Type" => content_type, "Content-MD5" => checksum, "Content-Disposition" => content_disposition }
+      { "Content-Type" => content_type, "Content-MD5" => checksum, "Content-Disposition" => content_disposition, **custom_metadata_headers(custom_metadata) }
+    end
+
+    def compose(source_keys, destination_key, filename: nil, content_type: nil, disposition: nil, custom_metadata: {})
+      content_disposition = content_disposition_with(type: disposition, filename: filename) if disposition && filename
+
+      object_for(destination_key).upload_stream(
+        content_type: content_type,
+        content_disposition: content_disposition,
+        part_size: MINIMUM_UPLOAD_PART_SIZE,
+        metadata: custom_metadata,
+        **upload_options
+      ) do |out|
+        source_keys.each do |source_key|
+          stream(source_key) do |chunk|
+            IO.copy_stream(StringIO.new(chunk), out)
+          end
+        end
+      end
     end
 
     private
-      def private_url(key, expires_in:, filename:, disposition:, content_type:, **)
+      def private_url(key, expires_in:, filename:, disposition:, content_type:, **client_opts)
         object_for(key).presigned_url :get, expires_in: expires_in.to_i,
           response_content_disposition: content_disposition_with(type: disposition, filename: filename),
-          response_content_type: content_type
+          response_content_type: content_type, **client_opts
       end
 
-      def public_url(key, **)
-        object_for(key).public_url
+      def public_url(key, **client_opts)
+        object_for(key).public_url(**client_opts)
       end
 
 
       MAXIMUM_UPLOAD_PARTS_COUNT = 10000
       MINIMUM_UPLOAD_PART_SIZE   = 5.megabytes
 
-      def upload_with_single_part(key, io, checksum: nil, content_type: nil, content_disposition: nil)
-        object_for(key).put(body: io, content_md5: checksum, content_type: content_type, content_disposition: content_disposition, **upload_options)
+      def upload_with_single_part(key, io, checksum: nil, content_type: nil, content_disposition: nil, custom_metadata: {})
+        object_for(key).put(body: io, content_md5: checksum, content_type: content_type, content_disposition: content_disposition, metadata: custom_metadata, **upload_options)
       rescue Aws::S3::Errors::BadDigest
         raise ActiveStorage::IntegrityError
       end
 
-      def upload_with_multipart(key, io, content_type: nil, content_disposition: nil)
+      def upload_with_multipart(key, io, content_type: nil, content_disposition: nil, custom_metadata: {})
         part_size = [ io.size.fdiv(MAXIMUM_UPLOAD_PARTS_COUNT).ceil, MINIMUM_UPLOAD_PART_SIZE ].max
 
-        object_for(key).upload_stream(content_type: content_type, content_disposition: content_disposition, part_size: part_size, **upload_options) do |out|
+        object_for(key).upload_stream(content_type: content_type, content_disposition: content_disposition, part_size: part_size, metadata: custom_metadata, **upload_options) do |out|
           IO.copy_stream(io, out)
         end
       end
@@ -143,5 +163,9 @@ module ActiveStorage
           offset += chunk_size
         end
       end
+
+      def custom_metadata_headers(metadata)
+        metadata.transform_keys { |key| "x-amz-meta-#{key}" }
+      end
   end
 end

data/lib/active_storage/service.rb

@@ -6,6 +6,8 @@ require "action_dispatch"
 require "action_dispatch/http/content_disposition"
 
 module ActiveStorage
+  # = Active Storage \Service
+  #
   # Abstract class serving as an interface for concrete services.
   #
   # The available services are:
@@ -16,7 +18,7 @@ module ActiveStorage
   # * +AzureStorage+, to manage attachments through Microsoft Azure Storage.
   # * +Mirror+, to be able to use several services to manage attachments.
   #
-  # Inside a Rails application, you can set-up your services through the
+  # Inside a \Rails application, you can set-up your services through the
   # generated <tt>config/storage.yml</tt> file and reference one
   # of the aforementioned constant under the +service+ key. For example:
   #
@@ -31,12 +33,12 @@ module ActiveStorage
   #
   #   config.active_storage.service = :local
   #
-  # If you are using Active Storage outside of a Ruby on Rails application, you
+  # If you are using Active Storage outside of a Ruby on \Rails application, you
   # can configure the service to use like this:
   #
   #   ActiveStorage::Blob.service = ActiveStorage::Service.configure(
-  #     :Disk,
-  #     root: Pathname("/foo/bar/storage")
+  #     :local,
+  #     { local: {service: "Disk",  root: Pathname("/tmp/foo/storage") } }
   #   )
   class Service
     extend ActiveSupport::Autoload
@@ -57,7 +59,7 @@ module ActiveStorage
       # Passes the configurator and all of the service's config as keyword args.
       #
       # See MirrorService for an example.
-      def build(configurator:, name:, service: nil, **service_config) #:nodoc:
+      def build(configurator:, name:, service: nil, **service_config) # :nodoc:
         new(**service_config).tap do |service_instance|
           service_instance.name = name
         end
@@ -90,6 +92,11 @@ module ActiveStorage
       ActiveStorage::Downloader.new(self).open(*args, **options, &block)
     end
 
+    # Concatenate multiple files into a single "composed" file.
+    def compose(source_keys, destination_key, filename: nil, content_type: nil, disposition: nil, custom_metadata: {})
+      raise NotImplementedError
+    end
+
     # Delete the file at the +key+.
     def delete(key)
       raise NotImplementedError
@@ -128,12 +135,12 @@ module ActiveStorage
     # The URL will be valid for the amount of seconds specified in +expires_in+.
     # You must also provide the +content_type+, +content_length+, and +checksum+ of the file
     # that will be uploaded. All these attributes will be validated by the service upon upload.
-    def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:)
+    def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:, custom_metadata: {})
       raise NotImplementedError
     end
 
     # Returns a Hash of headers for +url_for_direct_upload+ requests.
-    def headers_for_direct_upload(key, filename:, content_type:, content_length:, checksum:)
+    def headers_for_direct_upload(key, filename:, content_type:, content_length:, checksum:, custom_metadata: {})
       {}
     end
 
@@ -150,6 +157,9 @@ module ActiveStorage
         raise NotImplementedError
       end
 
+      def custom_metadata_headers(metadata)
+        raise NotImplementedError
+      end
 
       def instrument(operation, payload = {}, &block)
         ActiveSupport::Notifications.instrument(

data/lib/active_storage/transformers/image_processing_transformer.rb

@@ -38,7 +38,7 @@ module ActiveStorage
             if name.to_s == "combine_options"
               raise ArgumentError, <<~ERROR.squish
                 Active Storage's ImageProcessing transformer doesn't support :combine_options,
-                as it always generates a single ImageMagick command.
+                as it always generates a single command.
               ERROR
             end
 

data/lib/active_storage/transformers/transformer.rb

@@ -2,6 +2,8 @@
 
 module ActiveStorage
   module Transformers
+    # = Active Storage \Transformers \Transformer
+    #
     # A Transformer applies a set of transformations to an image.
     #
     # The following concrete subclasses are included in Active Storage:
@@ -31,7 +33,7 @@ module ActiveStorage
       private
         # Returns an open Tempfile containing a transformed image in the given +format+.
         # All subclasses implement this method.
-        def process(file, format:) #:doc:
+        def process(file, format:) # :doc:
           raise NotImplementedError
         end
     end

data/lib/active_storage/version.rb

@@ -3,7 +3,7 @@
 require_relative "gem_version"
 
 module ActiveStorage
-  # Returns the version of the currently loaded ActiveStorage as a <tt>Gem::Version</tt>
+  # Returns the currently loaded version of Active Storage as a +Gem::Version+.
   def self.version
     gem_version
   end

data/lib/active_storage.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 #--
-# Copyright (c) 2017-2022 David Heinemeier Hansson, Basecamp
+# Copyright (c) David Heinemeier Hansson, 37signals LLC
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -29,14 +29,18 @@ require "active_support/rails"
 require "active_support/core_ext/numeric/time"
 
 require "active_storage/version"
+require "active_storage/deprecator"
 require "active_storage/errors"
 
 require "marcel"
 
+# :markup: markdown
+# :include: activestorage/README.md
 module ActiveStorage
   extend ActiveSupport::Autoload
 
   autoload :Attached
+  autoload :FixtureSet
   autoload :Service
   autoload :Previewer
   autoload :Analyzer
@@ -350,16 +354,32 @@ module ActiveStorage
   mattr_accessor :unsupported_image_processing_arguments
 
   mattr_accessor :service_urls_expire_in, default: 5.minutes
+  mattr_accessor :urls_expire_in
 
   mattr_accessor :routes_prefix, default: "/rails/active_storage"
   mattr_accessor :draw_routes, default: true
   mattr_accessor :resolve_model_to_route, default: :rails_storage_redirect
 
-  mattr_accessor :replace_on_assign_to_many, default: false
   mattr_accessor :track_variants, default: false
 
   mattr_accessor :video_preview_arguments, default: "-y -vframes 1 -f image2"
 
+  def self.replace_on_assign_to_many
+    ActiveStorage.deprecator.warn("config.active_storage.replace_on_assign_to_many is deprecated and has no effect.")
+  end
+
+  def self.replace_on_assign_to_many=(value)
+    ActiveStorage.deprecator.warn("config.active_storage.replace_on_assign_to_many is deprecated and has no effect.")
+  end
+
+  def self.silence_invalid_content_types_warning
+    ActiveStorage.deprecator.warn("config.active_storage.silence_invalid_content_types_warning is deprecated and has no effect.")
+  end
+
+  def self.silence_invalid_content_types_warning=(value)
+    ActiveStorage.deprecator.warn("config.active_storage.silence_invalid_content_types_warning is deprecated and has no effect.")
+  end
+
   module Transformers
     extend ActiveSupport::Autoload