activestorage 5.2.7.1 → 6.1.4.6

data/lib/active_storage/service/s3_service.rb

@@ -9,21 +9,28 @@ module ActiveStorage
   # Wraps the Amazon Simple Storage Service (S3) as an Active Storage service.
   # See ActiveStorage::Service for the generic API documentation that applies to all services.
   class Service::S3Service < Service
-    attr_reader :client, :bucket, :upload_options
+    attr_reader :client, :bucket
+    attr_reader :multipart_upload_threshold, :upload_options
 
-    def initialize(bucket:, upload: {}, **options)
+    def initialize(bucket:, upload: {}, public: false, **options)
       @client = Aws::S3::Resource.new(**options)
       @bucket = @client.bucket(bucket)
 
+      @multipart_upload_threshold = upload.delete(:multipart_threshold) || 100.megabytes
+      @public = public
+
       @upload_options = upload
+      @upload_options[:acl] = "public-read" if public?
     end
 
-    def upload(key, io, checksum: nil, **)
+    def upload(key, io, checksum: nil, filename: nil, content_type: nil, disposition: nil, **)
       instrument :upload, key: key, checksum: checksum do
-        begin
-          object_for(key).put(upload_options.merge(body: io, content_md5: checksum))
-        rescue Aws::S3::Errors::BadDigest
-          raise ActiveStorage::IntegrityError
+        content_disposition = content_disposition_with(filename: filename, type: disposition) if disposition && filename
+
+        if io.size < multipart_upload_threshold
+          upload_with_single_part key, io, checksum: checksum, content_type: content_type, content_disposition: content_disposition
+        else
+          upload_with_multipart key, io, content_type: content_type, content_disposition: content_disposition
         end
       end
     end
@@ -36,13 +43,17 @@ module ActiveStorage
       else
         instrument :download, key: key do
           object_for(key).get.body.string.force_encoding(Encoding::BINARY)
+        rescue Aws::S3::Errors::NoSuchKey
+          raise ActiveStorage::FileNotFoundError
         end
       end
     end
 
     def download_chunk(key, range)
       instrument :download_chunk, key: key, range: range do
-        object_for(key).get(range: "bytes=#{range.begin}-#{range.exclude_end? ? range.end - 1 : range.end}").body.read.force_encoding(Encoding::BINARY)
+        object_for(key).get(range: "bytes=#{range.begin}-#{range.exclude_end? ? range.end - 1 : range.end}").body.string.force_encoding(Encoding::BINARY)
+      rescue Aws::S3::Errors::NoSuchKey
+        raise ActiveStorage::FileNotFoundError
       end
     end
 
@@ -66,23 +77,11 @@ module ActiveStorage
       end
     end
 
-    def url(key, expires_in:, filename:, disposition:, content_type:)
-      instrument :url, key: key do |payload|
-        generated_url = object_for(key).presigned_url :get, expires_in: expires_in.to_i,
-          response_content_disposition: content_disposition_with(type: disposition, filename: filename),
-          response_content_type: content_type
-
-        payload[:url] = generated_url
-
-        generated_url
-      end
-    end
-
     def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:)
       instrument :url, key: key do |payload|
         generated_url = object_for(key).presigned_url :put, expires_in: expires_in.to_i,
           content_type: content_type, content_length: content_length, content_md5: checksum,
-          whitelist_headers: ['content-length']
+          whitelist_headers: ["content-length"], **upload_options
 
         payload[:url] = generated_url
 
@@ -90,11 +89,42 @@ module ActiveStorage
       end
     end
 
-    def headers_for_direct_upload(key, content_type:, checksum:, **)
-      { "Content-Type" => content_type, "Content-MD5" => checksum }
+    def headers_for_direct_upload(key, content_type:, checksum:, filename: nil, disposition: nil, **)
+      content_disposition = content_disposition_with(type: disposition, filename: filename) if filename
+
+      { "Content-Type" => content_type, "Content-MD5" => checksum, "Content-Disposition" => content_disposition }
     end
 
     private
+      def private_url(key, expires_in:, filename:, disposition:, content_type:, **)
+        object_for(key).presigned_url :get, expires_in: expires_in.to_i,
+          response_content_disposition: content_disposition_with(type: disposition, filename: filename),
+          response_content_type: content_type
+      end
+
+      def public_url(key, **)
+        object_for(key).public_url
+      end
+
+
+      MAXIMUM_UPLOAD_PARTS_COUNT = 10000
+      MINIMUM_UPLOAD_PART_SIZE   = 5.megabytes
+
+      def upload_with_single_part(key, io, checksum: nil, content_type: nil, content_disposition: nil)
+        object_for(key).put(body: io, content_md5: checksum, content_type: content_type, content_disposition: content_disposition, **upload_options)
+      rescue Aws::S3::Errors::BadDigest
+        raise ActiveStorage::IntegrityError
+      end
+
+      def upload_with_multipart(key, io, content_type: nil, content_disposition: nil)
+        part_size = [ io.size.fdiv(MAXIMUM_UPLOAD_PARTS_COUNT).ceil, MINIMUM_UPLOAD_PART_SIZE ].max
+
+        object_for(key).upload_stream(content_type: content_type, content_disposition: content_disposition, part_size: part_size, **upload_options) do |out|
+          IO.copy_stream(io, out)
+        end
+      end
+
+
       def object_for(key)
         bucket.object(key)
       end
@@ -106,8 +136,10 @@ module ActiveStorage
         chunk_size = 5.megabytes
         offset = 0
 
+        raise ActiveStorage::FileNotFoundError unless object.exists?
+
         while offset < object.content_length
-          yield object.get(range: "bytes=#{offset}-#{offset + chunk_size - 1}").body.read.force_encoding(Encoding::BINARY)
+          yield object.get(range: "bytes=#{offset}-#{offset + chunk_size - 1}").body.string.force_encoding(Encoding::BINARY)
           offset += chunk_size
         end
       end

data/lib/active_storage/service.rb

@@ -1,10 +1,11 @@
 # frozen_string_literal: true
 
 require "active_storage/log_subscriber"
+require "active_storage/downloader"
+require "action_dispatch"
+require "action_dispatch/http/content_disposition"
 
 module ActiveStorage
-  class IntegrityError < StandardError; end
-
   # Abstract class serving as an interface for concrete services.
   #
   # The available services are:
@@ -40,8 +41,7 @@ module ActiveStorage
   class Service
     extend ActiveSupport::Autoload
     autoload :Configurator
-
-    class_attribute :url_expires_in, default: 5.minutes
+    attr_accessor :name
 
     class << self
       # Configure an Active Storage service by name from a set of configurations,
@@ -57,8 +57,10 @@ module ActiveStorage
       # Passes the configurator and all of the service's config as keyword args.
       #
       # See MirrorService for an example.
-      def build(configurator:, service: nil, **service_config) #:nodoc:
-        new(**service_config)
+      def build(configurator:, name:, service: nil, **service_config) #:nodoc:
+        new(**service_config).tap do |service_instance|
+          service_instance.name = name
+        end
       end
     end
 
@@ -84,6 +86,10 @@ module ActiveStorage
       raise NotImplementedError
     end
 
+    def open(*args, **options, &block)
+      ActiveStorage::Downloader.new(self).open(*args, **options, &block)
+    end
+
     # Delete the file at the +key+.
     def delete(key)
       raise NotImplementedError
@@ -99,11 +105,23 @@ module ActiveStorage
       raise NotImplementedError
     end
 
-    # Returns a signed, temporary URL for the file at the +key+. The URL will be valid for the amount
-    # of seconds specified in +expires_in+. You most also provide the +disposition+ (+:inline+ or +:attachment+),
-    # +filename+, and +content_type+ that you wish the file to be served with on request.
-    def url(key, expires_in:, disposition:, filename:, content_type:)
-      raise NotImplementedError
+    # Returns the URL for the file at the +key+. This returns a permanent URL for public files, and returns a
+    # short-lived URL for private files. For private files you can provide the +disposition+ (+:inline+ or +:attachment+),
+    # +filename+, and +content_type+ that you wish the file to be served with on request. Additionally, you can also provide
+    # the amount of seconds the URL will be valid for, specified in +expires_in+.
+    def url(key, **options)
+      instrument :url, key: key do |payload|
+        generated_url =
+          if public?
+            public_url(key, **options)
+          else
+            private_url(key, **options)
+          end
+
+        payload[:url] = generated_url
+
+        generated_url
+      end
     end
 
     # Returns a signed, temporary URL that a direct upload file can be PUT to on the +key+.
@@ -119,7 +137,20 @@ module ActiveStorage
       {}
     end
 
+    def public?
+      @public
+    end
+
     private
+      def private_url(key, expires_in:, filename:, disposition:, content_type:, **)
+        raise NotImplementedError
+      end
+
+      def public_url(key, **)
+        raise NotImplementedError
+      end
+
+
       def instrument(operation, payload = {}, &block)
         ActiveSupport::Notifications.instrument(
           "service_#{operation}.active_storage",
@@ -132,7 +163,8 @@ module ActiveStorage
       end
 
       def content_disposition_with(type: "inline", filename:)
-        (type.to_s.presence_in(%w( attachment inline )) || "inline") + "; #{filename.parameters}"
+        disposition = (type.to_s.presence_in(%w( attachment inline )) || "inline")
+        ActionDispatch::Http::ContentDisposition.format(disposition: disposition, filename: filename.sanitized)
       end
   end
 end

data/lib/active_storage/transformers/image_processing_transformer.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+begin
+  require "image_processing"
+rescue LoadError
+  raise LoadError, <<~ERROR.squish
+    Generating image variants require the image_processing gem.
+    Please add `gem 'image_processing', '~> 1.2'` to your Gemfile.
+  ERROR
+end
+
+module ActiveStorage
+  module Transformers
+    class ImageProcessingTransformer < Transformer
+      private
+        def process(file, format:)
+          processor.
+            source(file).
+            loader(page: 0).
+            convert(format).
+            apply(operations).
+            call
+        end
+
+        def processor
+          ImageProcessing.const_get(ActiveStorage.variant_processor.to_s.camelize)
+        end
+
+        def operations
+          transformations.each_with_object([]) do |(name, argument), list|
+            if name.to_s == "combine_options"
+              raise ArgumentError, <<~ERROR.squish
+                Active Storage's ImageProcessing transformer doesn't support :combine_options,
+                as it always generates a single ImageMagick command.
+              ERROR
+            end
+
+            if argument.present?
+              list << [ name, argument ]
+            end
+          end
+        end
+    end
+  end
+end

data/lib/active_storage/transformers/transformer.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module ActiveStorage
+  module Transformers
+    # A Transformer applies a set of transformations to an image.
+    #
+    # The following concrete subclasses are included in Active Storage:
+    #
+    # * ActiveStorage::Transformers::ImageProcessingTransformer:
+    #   backed by ImageProcessing, a common interface for MiniMagick and ruby-vips
+    class Transformer
+      attr_reader :transformations
+
+      def initialize(transformations)
+        @transformations = transformations
+      end
+
+      # Applies the transformations to the source image in +file+, producing a target image in the
+      # specified +format+. Yields an open Tempfile containing the target image. Closes and unlinks
+      # the output tempfile after yielding to the given block. Returns the result of the block.
+      def transform(file, format:)
+        output = process(file, format: format)
+
+        begin
+          yield output
+        ensure
+          output.close!
+        end
+      end
+
+      private
+        # Returns an open Tempfile containing a transformed image in the given +format+.
+        # All subclasses implement this method.
+        def process(file, format:) #:doc:
+          raise NotImplementedError
+        end
+    end
+  end
+end

data/lib/active_storage.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 #--
-# Copyright (c) 2017-2018 David Heinemeier Hansson, Basecamp
+# Copyright (c) 2017-2020 David Heinemeier Hansson, Basecamp
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -26,6 +26,7 @@
 require "active_record"
 require "active_support"
 require "active_support/rails"
+require "active_support/core_ext/numeric/time"
 
 require "active_storage/version"
 require "active_storage/errors"
@@ -42,302 +43,36 @@ module ActiveStorage
 
   mattr_accessor :logger
   mattr_accessor :verifier
-  mattr_accessor :queue
+  mattr_accessor :variant_processor, default: :mini_magick
+
+  mattr_accessor :queues, default: {}
+
   mattr_accessor :previewers, default: []
-  mattr_accessor :analyzers, default: []
+  mattr_accessor :analyzers,  default: []
+
   mattr_accessor :paths, default: {}
-  mattr_accessor :variable_content_types, default: []
+
+  mattr_accessor :variable_content_types,           default: []
+  mattr_accessor :web_image_content_types,          default: []
+  mattr_accessor :binary_content_type,              default: "application/octet-stream"
   mattr_accessor :content_types_to_serve_as_binary, default: []
-  mattr_accessor :content_types_allowed_inline, default: []
-  mattr_accessor :binary_content_type, default: "application/octet-stream"
-  mattr_accessor :supported_image_processing_methods, default: [
-    "adaptive_blur",
-    "adaptive_resize",
-    "adaptive_sharpen",
-    "adjoin",
-    "affine",
-    "alpha",
-    "annotate",
-    "antialias",
-    "append",
-    "apply",
-    "attenuate",
-    "authenticate",
-    "auto_gamma",
-    "auto_level",
-    "auto_orient",
-    "auto_threshold",
-    "backdrop",
-    "background",
-    "bench",
-    "bias",
-    "bilateral_blur",
-    "black_point_compensation",
-    "black_threshold",
-    "blend",
-    "blue_primary",
-    "blue_shift",
-    "blur",
-    "border",
-    "bordercolor",
-    "borderwidth",
-    "brightness_contrast",
-    "cache",
-    "canny",
-    "caption",
-    "channel",
-    "channel_fx",
-    "charcoal",
-    "chop",
-    "clahe",
-    "clamp",
-    "clip",
-    "clip_path",
-    "clone",
-    "clut",
-    "coalesce",
-    "colorize",
-    "colormap",
-    "color_matrix",
-    "colors",
-    "colorspace",
-    "colourspace",
-    "color_threshold",
-    "combine",
-    "combine_options",
-    "comment",
-    "compare",
-    "complex",
-    "compose",
-    "composite",
-    "compress",
-    "connected_components",
-    "contrast",
-    "contrast_stretch",
-    "convert",
-    "convolve",
-    "copy",
-    "crop",
-    "cycle",
-    "deconstruct",
-    "define",
-    "delay",
-    "delete",
-    "density",
-    "depth",
-    "descend",
-    "deskew",
-    "despeckle",
-    "direction",
-    "displace",
-    "dispose",
-    "dissimilarity_threshold",
-    "dissolve",
-    "distort",
-    "dither",
-    "draw",
-    "duplicate",
-    "edge",
-    "emboss",
-    "encoding",
-    "endian",
-    "enhance",
-    "equalize",
-    "evaluate",
-    "evaluate_sequence",
-    "extent",
-    "extract",
-    "family",
-    "features",
-    "fft",
-    "fill",
-    "filter",
-    "flatten",
-    "flip",
-    "floodfill",
-    "flop",
-    "font",
-    "foreground",
-    "format",
-    "frame",
-    "function",
-    "fuzz",
-    "fx",
-    "gamma",
-    "gaussian_blur",
-    "geometry",
-    "gravity",
-    "grayscale",
-    "green_primary",
-    "hald_clut",
-    "highlight_color",
-    "hough_lines",
-    "iconGeometry",
-    "iconic",
-    "identify",
-    "ift",
-    "illuminant",
-    "immutable",
-    "implode",
-    "insert",
-    "intensity",
-    "intent",
-    "interlace",
-    "interline_spacing",
-    "interpolate",
-    "interpolative_resize",
-    "interword_spacing",
-    "kerning",
-    "kmeans",
-    "kuwahara",
-    "label",
-    "lat",
-    "layers",
-    "level",
-    "level_colors",
-    "limit",
-    "limits",
-    "linear_stretch",
-    "linewidth",
-    "liquid_rescale",
-    "list",
-    "loader",
-    "log",
-    "loop",
-    "lowlight_color",
-    "magnify",
-    "map",
-    "mattecolor",
-    "median",
-    "mean_shift",
-    "metric",
-    "mode",
-    "modulate",
-    "moments",
-    "monitor",
-    "monochrome",
-    "morph",
-    "morphology",
-    "mosaic",
-    "motion_blur",
-    "name",
-    "negate",
-    "noise",
-    "normalize",
-    "opaque",
-    "ordered_dither",
-    "orient",
-    "page",
-    "paint",
-    "pause",
-    "perceptible",
-    "ping",
-    "pointsize",
-    "polaroid",
-    "poly",
-    "posterize",
-    "precision",
-    "preview",
-    "process",
-    "quality",
-    "quantize",
-    "quiet",
-    "radial_blur",
-    "raise",
-    "random_threshold",
-    "range_threshold",
-    "red_primary",
-    "regard_warnings",
-    "region",
-    "remote",
-    "render",
-    "repage",
-    "resample",
-    "resize",
-    "resize_to_fill",
-    "resize_to_fit",
-    "resize_to_limit",
-    "resize_and_pad",
-    "respect_parentheses",
-    "reverse",
-    "roll",
-    "rotate",
-    "sample",
-    "sampling_factor",
-    "saver",
-    "scale",
-    "scene",
-    "screen",
-    "seed",
-    "segment",
-    "selective_blur",
-    "separate",
-    "sepia_tone",
-    "shade",
-    "shadow",
-    "shared_memory",
-    "sharpen",
-    "shave",
-    "shear",
-    "sigmoidal_contrast",
-    "silent",
-    "similarity_threshold",
-    "size",
-    "sketch",
-    "smush",
-    "snaps",
-    "solarize",
-    "sort_pixels",
-    "sparse_color",
-    "splice",
-    "spread",
-    "statistic",
-    "stegano",
-    "stereo",
-    "storage_type",
-    "stretch",
-    "strip",
-    "stroke",
-    "strokewidth",
-    "style",
-    "subimage_search",
-    "swap",
-    "swirl",
-    "synchronize",
-    "taint",
-    "text_font",
-    "threshold",
-    "thumbnail",
-    "tile_offset",
-    "tint",
-    "title",
-    "transform",
-    "transparent",
-    "transparent_color",
-    "transpose",
-    "transverse",
-    "treedepth",
-    "trim",
-    "type",
-    "undercolor",
-    "unique_colors",
-    "units",
-    "unsharp",
-    "update",
-    "valid_image",
-    "view",
-    "vignette",
-    "virtual_pixel",
-    "visual",
-    "watermark",
-    "wave",
-    "wavelet_denoise",
-    "weight",
-    "white_balance",
-    "white_point",
-    "white_threshold",
-    "window",
-    "window_group",
-  ]
-  mattr_accessor :unsupported_image_processing_arguments
+  mattr_accessor :content_types_allowed_inline,     default: []
+
+  mattr_accessor :service_urls_expire_in, default: 5.minutes
+
+  mattr_accessor :routes_prefix, default: "/rails/active_storage"
+  mattr_accessor :draw_routes, default: true
+  mattr_accessor :resolve_model_to_route, default: :rails_storage_redirect
+
+  mattr_accessor :replace_on_assign_to_many, default: false
+  mattr_accessor :track_variants, default: false
+
+  mattr_accessor :video_preview_arguments, default: "-y -vframes 1 -f image2"
+
+  module Transformers
+    extend ActiveSupport::Autoload
+
+    autoload :Transformer
+    autoload :ImageProcessingTransformer
+  end
 end

data/lib/tasks/activestorage.rake

@@ -12,4 +12,15 @@ namespace :active_storage do
       Rake::Task["app:active_storage:install:migrations"].invoke
     end
   end
+
+  # desc "Copy over the migrations needed to the application upgrading"
+  task update: :environment do
+    ENV["MIGRATIONS_PATH"] = "db/update_migrate"
+
+    if Rake::Task.task_defined?("active_storage:install")
+      Rake::Task["active_storage:install"].invoke
+    else
+      Rake::Task["app:active_storage:install"].invoke
+    end
+  end
 end