activestorage 6.0.5 → 6.1.0.rc1

data/app/models/active_storage/preview.rb

@@ -3,8 +3,9 @@
 # Some non-image blobs can be previewed: that is, they can be presented as images. A video blob can be previewed by
 # extracting its first frame, and a PDF blob can be previewed by extracting its first page.
 #
-# A previewer extracts a preview image from a blob. Active Storage provides previewers for videos and PDFs:
-# ActiveStorage::Previewer::VideoPreviewer and ActiveStorage::Previewer::PDFPreviewer. Build custom previewers by
+# A previewer extracts a preview image from a blob. Active Storage provides previewers for videos and PDFs.
+# ActiveStorage::Previewer::VideoPreviewer is used for videos whereas ActiveStorage::Previewer::PopplerPDFPreviewer
+# and ActiveStorage::Previewer::MuPDFPreviewer are used for PDFs. Build custom previewers by
 # subclassing ActiveStorage::Previewer and implementing the requisite methods. Consult the ActiveStorage::Previewer
 # documentation for more details on what's required of previewers.
 #
@@ -13,11 +14,11 @@
 # by manipulating +Rails.application.config.active_storage.previewers+ in an initializer:
 #
 #   Rails.application.config.active_storage.previewers
-#   # => [ ActiveStorage::Previewer::PDFPreviewer, ActiveStorage::Previewer::VideoPreviewer ]
+#   # => [ ActiveStorage::Previewer::PopplerPDFPreviewer, ActiveStorage::Previewer::MuPDFPreviewer, ActiveStorage::Previewer::VideoPreviewer ]
 #
 #   # Add a custom previewer for Microsoft Office documents:
 #   Rails.application.config.active_storage.previewers << DOCXPreviewer
-#   # => [ ActiveStorage::Previewer::PDFPreviewer, ActiveStorage::Previewer::VideoPreviewer, DOCXPreviewer ]
+#   # => [ ActiveStorage::Previewer::PopplerPDFPreviewer, ActiveStorage::Previewer::MuPDFPreviewer, ActiveStorage::Previewer::VideoPreviewer, DOCXPreviewer ]
 #
 # Outside of a Rails application, modify +ActiveStorage.previewers+ instead.
 #
@@ -38,7 +39,7 @@ class ActiveStorage::Preview
 
   # Processes the preview if it has not been processed yet. Returns the receiving Preview instance for convenience:
   #
-  #   blob.preview(resize_to_limit: [100, 100]).processed.service_url
+  #   blob.preview(resize_to_limit: [100, 100]).processed.url
   #
   # Processing a preview generates an image from its blob and attaches the preview image to the blob. Because the preview
   # image is stored with the blob, it is only generated once.
@@ -56,10 +57,30 @@ class ActiveStorage::Preview
   # preview has not been processed yet.
   #
   # This method synchronously processes a variant of the preview image, so do not call it in views. Instead, generate
-  # a stable URL that redirects to the short-lived URL returned by this method.
-  def service_url(**options)
+  # a stable URL that redirects to the URL returned by this method.
+  def url(**options)
     if processed?
-      variant.service_url(**options)
+      variant.url(**options)
+    else
+      raise UnprocessedError
+    end
+  end
+
+  alias_method :service_url, :url
+  deprecate service_url: :url
+
+  # Returns a combination key of the blob and the variation that together identifies a specific variant.
+  def key
+    if processed?
+      variant.key
+    else
+      raise UnprocessedError
+    end
+  end
+
+  def download(&block)
+    if processed?
+      variant.download(&block)
     else
       raise UnprocessedError
     end
@@ -71,7 +92,7 @@ class ActiveStorage::Preview
     end
 
     def process
-      previewer.preview do |attachable|
+      previewer.preview(service_name: blob.service_name) do |attachable|
         ActiveRecord::Base.connected_to(role: ActiveRecord::Base.writing_role) do
           image.attach(attachable)
         end
@@ -79,7 +100,7 @@ class ActiveStorage::Preview
     end
 
     def variant
-      ActiveStorage::Variant.new(image, variation).processed
+      image.variant(variation).processed
     end
 
 

data/app/models/active_storage/record.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class ActiveStorage::Record < ActiveRecord::Base #:nodoc:
+  self.abstract_class = true
+end
+
+ActiveSupport.run_load_hooks :active_storage_record, ActiveStorage::Record

data/app/models/active_storage/variant.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "ostruct"
-
 # Image blobs can have variants that are the result of a set of transformations applied to the original.
 # These variants are used to create thumbnails, fixed-size avatars, or any other derivative image from the
 # original.
@@ -10,7 +8,7 @@ require "ostruct"
 # of the file, so you must add <tt>gem "image_processing"</tt> to your Gemfile if you wish to use variants. By
 # default, images will be processed with {ImageMagick}[http://imagemagick.org] using the
 # {MiniMagick}[https://github.com/minimagick/minimagick] gem, but you can also switch to the
-# {libvips}[http://jcupitt.github.io/libvips/] processor operated by the {ruby-vips}[https://github.com/jcupitt/ruby-vips]
+# {libvips}[http://libvips.github.io/libvips/] processor operated by the {ruby-vips}[https://github.com/libvips/ruby-vips]
 # gem).
 #
 #   Rails.application.config.active_storage.variant_processor
@@ -36,7 +34,7 @@ require "ostruct"
 # has already been processed and uploaded to the service, and, if so, just return that. Otherwise it will perform
 # the transformations, upload the variant to the service, and return itself again. Example:
 #
-#   avatar.variant(resize_to_limit: [100, 100]).processed.service_url
+#   avatar.variant(resize_to_limit: [100, 100]).processed.url
 #
 # This will create and process a variant of the avatar blob that's constrained to a height and width of 100.
 # Then it'll upload said variant to the service according to a derivative key of the blob and the transformations.
@@ -53,10 +51,9 @@ require "ostruct"
 # * {ImageProcessing::Vips}[https://github.com/janko-m/image_processing/blob/master/doc/vips.md#methods]
 # * {ruby-vips reference}[http://www.rubydoc.info/gems/ruby-vips/Vips/Image]
 class ActiveStorage::Variant
-  WEB_IMAGE_CONTENT_TYPES = %w[ image/png image/jpeg image/jpg image/gif ]
-
   attr_reader :blob, :variation
   delegate :service, to: :blob
+  delegate :content_type, to: :variation
 
   def initialize(blob, variation_or_variation_key)
     @blob, @variation = blob, ActiveStorage::Variation.wrap(variation_or_variation_key)
@@ -73,18 +70,34 @@ class ActiveStorage::Variant
     "variants/#{blob.key}/#{Digest::SHA256.hexdigest(variation.key)}"
   end
 
-  # Returns the URL of the variant on the service. This URL is intended to be short-lived for security and not used directly
-  # with users. Instead, the +service_url+ should only be exposed as a redirect from a stable, possibly authenticated URL.
-  # Hiding the +service_url+ behind a redirect also gives you the power to change services without updating all URLs. And
-  # it allows permanent URLs that redirect to the +service_url+ to be cached in the view.
+  # Returns the URL of the blob variant on the service. See {ActiveStorage::Blob#url} for details.
   #
   # Use <tt>url_for(variant)</tt> (or the implied form, like +link_to variant+ or +redirect_to variant+) to get the stable URL
   # for a variant that points to the ActiveStorage::RepresentationsController, which in turn will use this +service_call+ method
   # for its redirection.
-  def service_url(expires_in: ActiveStorage.service_urls_expire_in, disposition: :inline)
+  def url(expires_in: ActiveStorage.service_urls_expire_in, disposition: :inline)
     service.url key, expires_in: expires_in, disposition: disposition, filename: filename, content_type: content_type
   end
 
+  alias_method :service_url, :url
+  deprecate service_url: :url
+
+  # Downloads the file associated with this variant. If no block is given, the entire file is read into memory and returned.
+  # That'll use a lot of RAM for very large files. If a block is given, then the download is streamed and yielded in chunks.
+  def download(&block)
+    service.download key, &block
+  end
+
+  def filename
+    ActiveStorage::Filename.new "#{blob.filename.base}.#{variation.format}"
+  end
+
+  alias_method :content_type_for_serving, :content_type
+
+  def forced_disposition_for_serving #:nodoc:
+    nil
+  end
+
   # Returns the receiving variant. Allows ActiveStorage::Variant and ActiveStorage::Preview instances to be used interchangeably.
   def image
     self
@@ -96,36 +109,10 @@ class ActiveStorage::Variant
     end
 
     def process
-      blob.open do |image|
-        transform(image) { |output| upload(output) }
-      end
-    end
-
-    def transform(image, &block)
-      variation.transform(image, format: format, &block)
-    end
-
-    def upload(file)
-      service.upload(key, file)
-    end
-
-
-    def specification
-      @specification ||=
-        if WEB_IMAGE_CONTENT_TYPES.include?(blob.content_type)
-          Specification.new \
-            filename: blob.filename,
-            content_type: blob.content_type,
-            format: nil
-        else
-          Specification.new \
-            filename: ActiveStorage::Filename.new("#{blob.filename.base}.png"),
-            content_type: "image/png",
-            format: "png"
+      blob.open do |input|
+        variation.transform(input) do |output|
+          service.upload(key, output, content_type: content_type)
         end
+      end
     end
-
-    delegate :filename, :content_type, :format, to: :specification
-
-    class Specification < OpenStruct; end
 end

data/app/models/active_storage/variant_record.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+class ActiveStorage::VariantRecord < ActiveStorage::Record
+  self.table_name = "active_storage_variant_records"
+
+  belongs_to :blob
+  has_one_attached :image
+end

data/app/models/active_storage/variant_with_record.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+class ActiveStorage::VariantWithRecord
+  attr_reader :blob, :variation
+
+  def initialize(blob, variation)
+    @blob, @variation = blob, ActiveStorage::Variation.wrap(variation)
+  end
+
+  def processed
+    process
+    self
+  end
+
+  def process
+    transform_blob { |image| create_or_find_record(image: image) } unless processed?
+  end
+
+  def processed?
+    record.present?
+  end
+
+  def image
+    record&.image
+  end
+
+  delegate :key, :url, :download, to: :image, allow_nil: true
+
+  alias_method :service_url, :url
+  deprecate service_url: :url
+
+  private
+    def transform_blob
+      blob.open do |input|
+        variation.transform(input) do |output|
+          yield io: output, filename: "#{blob.filename.base}.#{variation.format}",
+            content_type: variation.content_type, service_name: blob.service.name
+        end
+      end
+    end
+
+    def create_or_find_record(image:)
+      @record =
+        ActiveRecord::Base.connected_to(role: ActiveRecord::Base.writing_role) do
+          blob.variant_records.create_or_find_by!(variation_digest: variation.digest) do |record|
+            record.image.attach(image)
+          end
+        end
+    end
+
+    def record
+      @record ||= blob.variant_records.find_by(variation_digest: variation.digest)
+    end
+end

data/app/models/active_storage/variation.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "mimemagic"
+
 # A set of transformations that can be applied to a blob to create a variant. This class is exposed via
 # the ActiveStorage::Blob#variant method and should rarely be used directly.
 #
@@ -43,38 +45,41 @@ class ActiveStorage::Variation
     @transformations = transformations.deep_symbolize_keys
   end
 
+  def default_to(defaults)
+    self.class.new transformations.reverse_merge(defaults)
+  end
+
   # Accepts a File object, performs the +transformations+ against it, and
-  # saves the transformed image into a temporary file. If +format+ is specified
-  # it will be the format of the result image, otherwise the result image
-  # retains the source format.
-  def transform(file, format: nil, &block)
+  # saves the transformed image into a temporary file.
+  def transform(file, &block)
     ActiveSupport::Notifications.instrument("transform.active_storage") do
       transformer.transform(file, format: format, &block)
     end
   end
 
+  def format
+    transformations.fetch(:format, :png).tap do |format|
+      if MimeMagic.by_extension(format).nil?
+        raise ArgumentError, "Invalid variant format (#{format.inspect})"
+      end
+    end
+  end
+
+  def content_type
+    MimeMagic.by_extension(format).to_s
+  end
+
   # Returns a signed key for all the +transformations+ that this variation was instantiated with.
   def key
     self.class.encode(transformations)
   end
 
+  def digest
+    Digest::SHA1.base64digest Marshal.dump(transformations)
+  end
+
   private
     def transformer
-      if ActiveStorage.variant_processor
-        begin
-          require "image_processing"
-        rescue LoadError
-          ActiveSupport::Deprecation.warn <<~WARNING.squish
-            Generating image variants will require the image_processing gem in Rails 6.1.
-            Please add `gem 'image_processing', '~> 1.2'` to your Gemfile.
-          WARNING
-
-          ActiveStorage::Transformers::MiniMagickTransformer.new(transformations)
-        else
-          ActiveStorage::Transformers::ImageProcessingTransformer.new(transformations)
-        end
-      else
-        ActiveStorage::Transformers::MiniMagickTransformer.new(transformations)
-      end
+      ActiveStorage::Transformers::ImageProcessingTransformer.new(transformations.except(:format))
     end
 end

data/config/routes.rb

@@ -2,9 +2,13 @@
 
 Rails.application.routes.draw do
   scope ActiveStorage.routes_prefix do
-    get "/blobs/:signed_id/*filename" => "active_storage/blobs#show", as: :rails_service_blob
+    get "/blobs/redirect/:signed_id/*filename" => "active_storage/blobs/redirect#show", as: :rails_service_blob
+    get "/blobs/proxy/:signed_id/*filename" => "active_storage/blobs/proxy#show", as: :rails_service_blob_proxy
+    get "/blobs/:signed_id/*filename" => "active_storage/blobs/redirect#show"
 
-    get "/representations/:signed_blob_id/:variation_key/*filename" => "active_storage/representations#show", as: :rails_blob_representation
+    get "/representations/redirect/:signed_blob_id/:variation_key/*filename" => "active_storage/representations/redirect#show", as: :rails_blob_representation
+    get "/representations/proxy/:signed_blob_id/:variation_key/*filename" => "active_storage/representations/proxy#show", as: :rails_blob_representation_proxy
+    get "/representations/:signed_blob_id/:variation_key/*filename" => "active_storage/representations/redirect#show"
 
     get  "/disk/:encoded_key/*filename" => "active_storage/disk#show", as: :rails_disk_service
     put  "/disk/:encoded_token" => "active_storage/disk#update", as: :update_rails_disk_service
@@ -19,14 +23,60 @@ Rails.application.routes.draw do
     route_for(:rails_blob_representation, signed_blob_id, variation_key, filename, options)
   end
 
-  resolve("ActiveStorage::Variant") { |variant, options| route_for(:rails_representation, variant, options) }
-  resolve("ActiveStorage::Preview") { |preview, options| route_for(:rails_representation, preview, options) }
-
+  resolve("ActiveStorage::Variant") { |variant, options| route_for(ActiveStorage.resolve_model_to_route, variant, options) }
+  resolve("ActiveStorage::VariantWithRecord") { |variant, options| route_for(ActiveStorage.resolve_model_to_route, variant, options) }
+  resolve("ActiveStorage::Preview") { |preview, options| route_for(ActiveStorage.resolve_model_to_route, preview, options) }
 
   direct :rails_blob do |blob, options|
     route_for(:rails_service_blob, blob.signed_id, blob.filename, options)
   end
 
-  resolve("ActiveStorage::Blob")       { |blob, options| route_for(:rails_blob, blob, options) }
-  resolve("ActiveStorage::Attachment") { |attachment, options| route_for(:rails_blob, attachment.blob, options) }
-end
+  resolve("ActiveStorage::Blob")       { |blob, options| route_for(ActiveStorage.resolve_model_to_route, blob, options) }
+  resolve("ActiveStorage::Attachment") { |attachment, options| route_for(ActiveStorage.resolve_model_to_route, attachment.blob, options) }
+
+  direct :rails_storage_proxy do |model, options|
+    if model.respond_to?(:signed_id)
+      route_for(
+        :rails_service_blob_proxy,
+        model.signed_id,
+        model.filename,
+        options
+      )
+    else
+      signed_blob_id = model.blob.signed_id
+      variation_key  = model.variation.key
+      filename       = model.blob.filename
+
+      route_for(
+        :rails_blob_representation_proxy,
+        signed_blob_id,
+        variation_key,
+        filename,
+        options
+      )
+    end
+  end
+
+  direct :rails_storage_redirect do |model, options|
+    if model.respond_to?(:signed_id)
+      route_for(
+        :rails_service_blob,
+        model.signed_id,
+        model.filename,
+        options
+      )
+    else
+      signed_blob_id = model.blob.signed_id
+      variation_key  = model.variation.key
+      filename       = model.blob.filename
+
+      route_for(
+        :rails_blob_representation,
+        signed_blob_id,
+        variation_key,
+        filename,
+        options
+      )
+    end
+  end
+end if ActiveStorage.draw_routes

data/db/migrate/20170806125915_create_active_storage_tables.rb

@@ -1,13 +1,14 @@
 class CreateActiveStorageTables < ActiveRecord::Migration[5.2]
   def change
     create_table :active_storage_blobs do |t|
-      t.string   :key,        null: false
-      t.string   :filename,   null: false
+      t.string   :key,          null: false
+      t.string   :filename,     null: false
       t.string   :content_type
       t.text     :metadata
-      t.bigint   :byte_size,  null: false
-      t.string   :checksum,   null: false
-      t.datetime :created_at, null: false
+      t.string   :service_name, null: false
+      t.bigint   :byte_size,    null: false
+      t.string   :checksum,     null: false
+      t.datetime :created_at,   null: false
 
       t.index [ :key ], unique: true
     end
@@ -22,5 +23,13 @@ class CreateActiveStorageTables < ActiveRecord::Migration[5.2]
       t.index [ :record_type, :record_id, :name, :blob_id ], name: "index_active_storage_attachments_uniqueness", unique: true
       t.foreign_key :active_storage_blobs, column: :blob_id
     end
+
+    create_table :active_storage_variant_records do |t|
+      t.belongs_to :blob, null: false, index: false
+      t.string :variation_digest, null: false
+
+      t.index %i[ blob_id variation_digest ], name: "index_active_storage_variant_records_uniqueness", unique: true
+      t.foreign_key :active_storage_blobs, column: :blob_id
+    end
   end
 end

data/db/update_migrate/20190112182829_add_service_name_to_active_storage_blobs.rb

@@ -0,0 +1,17 @@
+class AddServiceNameToActiveStorageBlobs < ActiveRecord::Migration[6.0]
+  def up
+    unless column_exists?(:active_storage_blobs, :service_name)
+      add_column :active_storage_blobs, :service_name, :string
+
+      if configured_service = ActiveStorage::Blob.service.name
+        ActiveStorage::Blob.unscoped.update_all(service_name: configured_service)
+      end
+
+      change_column :active_storage_blobs, :service_name, :string, null: false
+    end
+  end
+
+  def down
+    remove_column :active_storage_blobs, :service_name
+  end
+end

data/db/update_migrate/20191206030411_create_active_storage_variant_records.rb

@@ -0,0 +1,11 @@
+class CreateActiveStorageVariantRecords < ActiveRecord::Migration[6.0]
+  def change
+    create_table :active_storage_variant_records do |t|
+      t.belongs_to :blob, null: false, index: false
+      t.string :variation_digest, null: false
+
+      t.index %i[ blob_id variation_digest ], name: "index_active_storage_variant_records_uniqueness", unique: true
+      t.foreign_key :active_storage_blobs, column: :blob_id
+    end
+  end
+end

data/lib/active_storage/analyzer/image_analyzer.rb

@@ -43,6 +43,9 @@ module ActiveStorage
       rescue LoadError
         logger.info "Skipping image analysis because the mini_magick gem isn't installed"
         {}
+      rescue MiniMagick::Error => error
+        logger.error "Skipping image analysis due to an ImageMagick error: #{error.message}"
+        {}
       end
 
       def rotated_image?(image)

data/lib/active_storage/analyzer/null_analyzer.rb

@@ -6,6 +6,10 @@ module ActiveStorage
       true
     end
 
+    def self.analyze_later?
+      false
+    end
+
     def metadata
       {}
     end

data/lib/active_storage/analyzer/video_analyzer.rb

@@ -44,7 +44,8 @@ module ActiveStorage
       end
 
       def duration
-        Float(video_stream["duration"]) if video_stream["duration"]
+        duration = video_stream["duration"] || container["duration"]
+        Float(duration) if duration
       end
 
       def angle
@@ -98,12 +99,22 @@ module ActiveStorage
         probe["streams"] || []
       end
 
+      def container
+        probe["format"] || {}
+      end
+
       def probe
-        download_blob_to_tempfile { |file| probe_from(file) }
+        @probe ||= download_blob_to_tempfile { |file| probe_from(file) }
       end
 
       def probe_from(file)
-        IO.popen([ ffprobe_path, "-print_format", "json", "-show_streams", "-v", "error", file.path ]) do |output|
+        IO.popen([ ffprobe_path,
+          "-print_format", "json",
+          "-show_streams",
+          "-show_format",
+          "-v", "error",
+          file.path
+        ]) do |output|
           JSON.parse(output.read)
         end
       rescue Errno::ENOENT

data/lib/active_storage/analyzer.rb

@@ -12,6 +12,12 @@ module ActiveStorage
       false
     end
 
+    # Implement this method in concrete subclasses. It will determine if blob analysis
+    # should be done in a job or performed inline. By default, analysis is enqueued in a job.
+    def self.analyze_later?
+      true
+    end
+
     def initialize(blob)
       @blob = blob
     end

data/lib/active_storage/attached/changes/create_many.rb

@@ -6,6 +6,7 @@ module ActiveStorage
 
     def initialize(name, record, attachables)
       @name, @record, @attachables = name, record, Array(attachables)
+      blobs.each(&:identify_without_saving)
     end
 
     def attachments

data/lib/active_storage/attached/changes/create_one.rb

@@ -9,6 +9,7 @@ module ActiveStorage
 
     def initialize(name, record, attachable)
       @name, @record, @attachable = name, record, attachable
+      blob.identify_without_saving
     end
 
     def attachment
@@ -53,17 +54,29 @@ module ActiveStorage
         when ActiveStorage::Blob
           attachable
         when ActionDispatch::Http::UploadedFile, Rack::Test::UploadedFile
-          ActiveStorage::Blob.build_after_unfurling \
+          ActiveStorage::Blob.build_after_unfurling(
             io: attachable.open,
             filename: attachable.original_filename,
-            content_type: attachable.content_type
+            content_type: attachable.content_type,
+            record: record,
+            service_name: attachment_service_name
+          )
         when Hash
-          ActiveStorage::Blob.build_after_unfurling(**attachable.symbolize_keys)
+          ActiveStorage::Blob.build_after_unfurling(
+            **attachable.reverse_merge(
+              record: record,
+              service_name: attachment_service_name
+            ).symbolize_keys
+          )
         when String
-          ActiveStorage::Blob.find_signed(attachable)
+          ActiveStorage::Blob.find_signed!(attachable, record: record)
         else
           raise ArgumentError, "Could not find or build blob: expected attachable, got #{attachable.inspect}"
         end
       end
+
+      def attachment_service_name
+        record.attachment_reflections[name].options[:service_name]
+      end
   end
 end

data/lib/active_storage/attached/many.rb

@@ -29,15 +29,16 @@ module ActiveStorage
     #   document.images.attach([ first_blob, second_blob ])
     def attach(*attachables)
       if record.persisted? && !record.changed?
-        record.update(name => blobs + attachables.flatten)
+        record.public_send("#{name}=", blobs + attachables.flatten)
+        record.save
       else
         record.public_send("#{name}=", (change&.attachables || blobs) + attachables.flatten)
       end
     end
 
-    # Returns true if any attachments has been made.
+    # Returns true if any attachments have been made.
     #
-    #   class Gallery < ActiveRecord::Base
+    #   class Gallery < ApplicationRecord
     #     has_many_attached :photos
     #   end
     #