activestorage 0.1 → 5.2.0.beta1

data/lib/active_storage/service/configurator.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module ActiveStorage
+  class Service::Configurator #:nodoc:
+    attr_reader :configurations
+
+    def self.build(service_name, configurations)
+      new(configurations).build(service_name)
+    end
+
+    def initialize(configurations)
+      @configurations = configurations.deep_symbolize_keys
+    end
+
+    def build(service_name)
+      config = config_for(service_name.to_sym)
+      resolve(config.fetch(:service)).build(**config, configurator: self)
+    end
+
+    private
+      def config_for(name)
+        configurations.fetch name do
+          raise "Missing configuration for the #{name.inspect} Active Storage service. Configurations available for #{configurations.keys.inspect}"
+        end
+      end
+
+      def resolve(class_name)
+        require "active_storage/service/#{class_name.to_s.underscore}_service"
+        ActiveStorage::Service.const_get(:"#{class_name}Service")
+      end
+  end
+end

data/lib/active_storage/service/disk_service.rb

@@ -1,70 +1,129 @@
+# frozen_string_literal: true
+
 require "fileutils"
 require "pathname"
+require "digest/md5"
 require "active_support/core_ext/numeric/bytes"
 
-class ActiveStorage::Service::DiskService < ActiveStorage::Service
-  attr_reader :root
+module ActiveStorage
+  # Wraps a local disk path as an Active Storage service. See ActiveStorage::Service for the generic API
+  # documentation that applies to all services.
+  class Service::DiskService < Service
+    attr_reader :root
 
-  def initialize(root:)
-    @root = root
-  end
+    def initialize(root:)
+      @root = root
+    end
 
-  def upload(key, io, checksum: nil)
-    File.open(make_path_for(key), "wb") do |file|
-      while chunk = io.read(64.kilobytes)
-        file.write(chunk)
+    def upload(key, io, checksum: nil)
+      instrument :upload, key, checksum: checksum do
+        IO.copy_stream(io, make_path_for(key))
+        ensure_integrity_of(key, checksum) if checksum
       end
     end
 
-    ensure_integrity_of(key, checksum) if checksum
-  end
+    def download(key)
+      if block_given?
+        instrument :streaming_download, key do
+          File.open(path_for(key), "rb") do |file|
+            while data = file.read(64.kilobytes)
+              yield data
+            end
+          end
+        end
+      else
+        instrument :download, key do
+          File.binread path_for(key)
+        end
+      end
+    end
 
-  def download(key)
-    if block_given?
-      File.open(path_for(key)) do |file|
-        while data = file.read(64.kilobytes)
-          yield data
+    def delete(key)
+      instrument :delete, key do
+        begin
+          File.delete path_for(key)
+        rescue Errno::ENOENT
+          # Ignore files already deleted
         end
       end
-    else
-      File.open path_for(key), &:read
     end
-  end
 
-  def delete(key)
-    File.delete path_for(key) rescue Errno::ENOENT # Ignore files already deleted
-  end
+    def exist?(key)
+      instrument :exist, key do |payload|
+        answer = File.exist? path_for(key)
+        payload[:exist] = answer
+        answer
+      end
+    end
 
-  def exist?(key)
-    File.exist? path_for(key)
-  end
+    def url(key, expires_in:, filename:, disposition:, content_type:)
+      instrument :url, key do |payload|
+        verified_key_with_expiration = ActiveStorage.verifier.generate(key, expires_in: expires_in, purpose: :blob_key)
 
-  def url(key, expires_in:, disposition:, filename:)
-    verified_key_with_expiration = ActiveStorage::VerifiedKeyWithExpiration.encode(key, expires_in: expires_in)
+        generated_url =
+          if defined?(Rails.application)
+            Rails.application.routes.url_helpers.rails_disk_service_path \
+              verified_key_with_expiration,
+              filename: filename, disposition: content_disposition_with(type: disposition, filename: filename), content_type: content_type
+          else
+            "/rails/active_storage/disk/#{verified_key_with_expiration}/#{filename}?content_type=#{content_type}" \
+              "&disposition=#{content_disposition_with(type: disposition, filename: filename)}"
+          end
 
-    if defined?(Rails) && defined?(Rails.application)
-      Rails.application.routes.url_helpers.rails_disk_blob_path(verified_key_with_expiration, disposition: disposition)
-    else
-      "/rails/blobs/#{verified_key_with_expiration}?disposition=#{disposition}"
-    end
-  end
+        payload[:url] = generated_url
 
-  private
-    def path_for(key)
-      File.join root, folder_for(key), key
+        generated_url
+      end
     end
 
-    def folder_for(key)
-      [ key[0..1], key[2..3] ].join("/")
+    def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:)
+      instrument :url, key do |payload|
+        verified_token_with_expiration = ActiveStorage.verifier.generate(
+          {
+            key: key,
+            content_type: content_type,
+            content_length: content_length,
+            checksum: checksum
+          },
+          { expires_in: expires_in,
+          purpose: :blob_token }
+        )
+
+        generated_url =
+          if defined?(Rails.application)
+            Rails.application.routes.url_helpers.update_rails_disk_service_path verified_token_with_expiration
+          else
+            "/rails/active_storage/disk/#{verified_token_with_expiration}"
+          end
+
+        payload[:url] = generated_url
+
+        generated_url
+      end
     end
 
-    def make_path_for(key)
-      path_for(key).tap { |path| FileUtils.mkdir_p File.dirname(path) }
+    def headers_for_direct_upload(key, content_type:, **)
+      { "Content-Type" => content_type }
     end
 
-    def ensure_integrity_of(key, checksum)
-      unless Digest::MD5.file(path_for(key)).base64digest == checksum
-        raise ActiveStorage::IntegrityError
+    private
+      def path_for(key)
+        File.join root, folder_for(key), key
       end
-    end
+
+      def folder_for(key)
+        [ key[0..1], key[2..3] ].join("/")
+      end
+
+      def make_path_for(key)
+        path_for(key).tap { |path| FileUtils.mkdir_p File.dirname(path) }
+      end
+
+      def ensure_integrity_of(key, checksum)
+        unless Digest::MD5.file(path_for(key)).base64digest == checksum
+          delete key
+          raise ActiveStorage::IntegrityError
+        end
+      end
+  end
 end

data/lib/active_storage/service/gcs_service.rb

@@ -1,41 +1,99 @@
+# frozen_string_literal: true
+
 require "google/cloud/storage"
 require "active_support/core_ext/object/to_query"
 
-class ActiveStorage::Service::GCSService < ActiveStorage::Service
-  attr_reader :client, :bucket
+module ActiveStorage
+  # Wraps the Google Cloud Storage as an Active Storage service. See ActiveStorage::Service for the generic API
+  # documentation that applies to all services.
+  class Service::GCSService < Service
+    def initialize(**config)
+      @config = config
+    end
 
-  def initialize(project:, keyfile:, bucket:)
-    @client = Google::Cloud::Storage.new(project: project, keyfile: keyfile)
-    @bucket = @client.bucket(bucket)
-  end
+    def upload(key, io, checksum: nil)
+      instrument :upload, key, checksum: checksum do
+        begin
+          bucket.create_file(io, key, md5: checksum)
+        rescue Google::Cloud::InvalidArgumentError
+          raise ActiveStorage::IntegrityError
+        end
+      end
+    end
 
-  def upload(key, io, checksum: nil)
-    # FIXME: Ensure integrity by sending the checksum for service side verification
-    bucket.create_file(io, key)
-  end
+    # FIXME: Download in chunks when given a block.
+    def download(key)
+      instrument :download, key do
+        io = file_for(key).download
+        io.rewind
 
-  # FIXME: Add streaming when given a block
-  def download(key)
-    io = file_for(key).download
-    io.rewind
-    io.read
-  end
+        if block_given?
+          yield io.read
+        else
+          io.read
+        end
+      end
+    end
 
-  def delete(key)
-    file_for(key).try(:delete)
-  end
+    def delete(key)
+      instrument :delete, key do
+        begin
+          file_for(key).delete
+        rescue Google::Cloud::NotFoundError
+          # Ignore files already deleted
+        end
+      end
+    end
 
-  def exist?(key)
-    file_for(key).present?
-  end
+    def exist?(key)
+      instrument :exist, key do |payload|
+        answer = file_for(key).exists?
+        payload[:exist] = answer
+        answer
+      end
+    end
 
-  def url(key, expires_in:, disposition:, filename:)
-    file_for(key).signed_url(expires: expires_in) + "&" +
-      { "response-content-disposition" => "#{disposition}; filename=\"#{filename}\"" }.to_query
-  end
+    def url(key, expires_in:, filename:, content_type:, disposition:)
+      instrument :url, key do |payload|
+        generated_url = file_for(key).signed_url expires: expires_in, query: {
+          "response-content-disposition" => content_disposition_with(type: disposition, filename: filename),
+          "response-content-type" => content_type
+        }
 
-  private
-    def file_for(key)
-      bucket.file(key)
+        payload[:url] = generated_url
+
+        generated_url
+      end
+    end
+
+    def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:)
+      instrument :url, key do |payload|
+        generated_url = bucket.signed_url key, method: "PUT", expires: expires_in,
+          content_type: content_type, content_md5: checksum
+
+        payload[:url] = generated_url
+
+        generated_url
+      end
+    end
+
+    def headers_for_direct_upload(key, content_type:, checksum:, **)
+      { "Content-Type" => content_type, "Content-MD5" => checksum }
     end
+
+    private
+      attr_reader :config
+
+      def file_for(key)
+        bucket.file(key, skip_lookup: true)
+      end
+
+      def bucket
+        @bucket ||= client.bucket(config.fetch(:bucket))
+      end
+
+      def client
+        @client ||= Google::Cloud::Storage.new(config.except(:bucket))
+      end
+  end
 end

data/lib/active_storage/service/mirror_service.rb

@@ -1,34 +1,50 @@
+# frozen_string_literal: true
+
 require "active_support/core_ext/module/delegation"
 
-class ActiveStorage::Service::MirrorService < ActiveStorage::Service
-  attr_reader :services
+module ActiveStorage
+  # Wraps a set of mirror services and provides a single ActiveStorage::Service object that will all
+  # have the files uploaded to them. A +primary+ service is designated to answer calls to +download+, +exists?+,
+  # and +url+.
+  class Service::MirrorService < Service
+    attr_reader :primary, :mirrors
 
-  delegate :download, :exist?, :url, :byte_size, :checksum, to: :primary_service
+    delegate :download, :exist?, :url, to: :primary
 
-  def initialize(services:)
-    @services = services
-  end
+    # Stitch together from named services.
+    def self.build(primary:, mirrors:, configurator:, **options) #:nodoc:
+      new \
+        primary: configurator.build(primary),
+        mirrors: mirrors.collect { |name| configurator.build name }
+    end
 
-  def upload(key, io, checksum: nil)
-    services.collect do |service|
-      service.upload key, io, checksum: checksum
-      io.rewind
+    def initialize(primary:, mirrors:)
+      @primary, @mirrors = primary, mirrors
     end
-  end
 
-  def delete(key)
-    perform_across_services :delete, key
-  end
+    # Upload the +io+ to the +key+ specified to all services. If a +checksum+ is provided, all services will
+    # ensure a match when the upload has completed or raise an ActiveStorage::IntegrityError.
+    def upload(key, io, checksum: nil)
+      each_service.collect do |service|
+        service.upload key, io.tap(&:rewind), checksum: checksum
+      end
+    end
 
-  private
-    def primary_service
-      services.first
+    # Delete the file at the +key+ on all services.
+    def delete(key)
+      perform_across_services :delete, key
     end
 
-    def perform_across_services(method, *args)
-      # FIXME: Convert to be threaded
-      services.collect do |service|
-        service.public_send method, *args
+    private
+      def each_service(&block)
+        [ primary, *mirrors ].each(&block)
       end
-    end
+
+      def perform_across_services(method, *args)
+        # FIXME: Convert to be threaded
+        each_service.collect do |service|
+          service.public_send method, *args
+        end
+      end
+  end
 end

data/lib/active_storage/service/s3_service.rb

@@ -1,55 +1,100 @@
-require "aws-sdk"
+# frozen_string_literal: true
+
+require "aws-sdk-s3"
 require "active_support/core_ext/numeric/bytes"
 
-class ActiveStorage::Service::S3Service < ActiveStorage::Service
-  attr_reader :client, :bucket
+module ActiveStorage
+  # Wraps the Amazon Simple Storage Service (S3) as an Active Storage service.
+  # See ActiveStorage::Service for the generic API documentation that applies to all services.
+  class Service::S3Service < Service
+    attr_reader :client, :bucket, :upload_options
 
-  def initialize(access_key_id:, secret_access_key:, region:, bucket:)
-    @client = Aws::S3::Resource.new(access_key_id: access_key_id, secret_access_key: secret_access_key, region: region)
-    @bucket = @client.bucket(bucket)
-  end
+    def initialize(access_key_id:, secret_access_key:, region:, bucket:, upload: {}, **options)
+      @client = Aws::S3::Resource.new(access_key_id: access_key_id, secret_access_key: secret_access_key, region: region, **options)
+      @bucket = @client.bucket(bucket)
 
-  def upload(key, io, checksum: nil)
-    # FIXME: Ensure integrity by sending the checksum for service side verification
-    object_for(key).put(body: io)
-  end
+      @upload_options = upload
+    end
 
-  def download(key)
-    if block_given?
-      stream(key, &block)
-    else
-      object_for(key).get.body.read
+    def upload(key, io, checksum: nil)
+      instrument :upload, key, checksum: checksum do
+        begin
+          object_for(key).put(upload_options.merge(body: io, content_md5: checksum))
+        rescue Aws::S3::Errors::BadDigest
+          raise ActiveStorage::IntegrityError
+        end
+      end
     end
-  end
 
-  def delete(key)
-    object_for(key).delete
-  end
+    def download(key, &block)
+      if block_given?
+        instrument :streaming_download, key do
+          stream(key, &block)
+        end
+      else
+        instrument :download, key do
+          object_for(key).get.body.read.force_encoding(Encoding::BINARY)
+        end
+      end
+    end
 
-  def exist?(key)
-    object_for(key).exists?
-  end
+    def delete(key)
+      instrument :delete, key do
+        object_for(key).delete
+      end
+    end
 
-  def url(key, expires_in:, disposition:, filename:)
-    object_for(key).presigned_url :get, expires_in: expires_in,
-      response_content_disposition: "#{disposition}; filename=\"#{filename}\""
-  end
+    def exist?(key)
+      instrument :exist, key do |payload|
+        answer = object_for(key).exists?
+        payload[:exist] = answer
+        answer
+      end
+    end
+
+    def url(key, expires_in:, filename:, disposition:, content_type:)
+      instrument :url, key do |payload|
+        generated_url = object_for(key).presigned_url :get, expires_in: expires_in.to_i,
+          response_content_disposition: content_disposition_with(type: disposition, filename: filename),
+          response_content_type: content_type
 
-  private
-    def object_for(key)
-      bucket.object(key)
+        payload[:url] = generated_url
+
+        generated_url
+      end
     end
 
-    # Reads the object for the given key in chunks, yielding each to the block.
-    def stream(key, options = {}, &block)
-      object = object_for(key)
+    def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:)
+      instrument :url, key do |payload|
+        generated_url = object_for(key).presigned_url :put, expires_in: expires_in.to_i,
+          content_type: content_type, content_length: content_length, content_md5: checksum
 
-      chunk_size = 5.megabytes
-      offset = 0
+        payload[:url] = generated_url
 
-      while offset < object.content_length
-        yield object.read(options.merge(:range => "bytes=#{offset}-#{offset + chunk_size - 1}"))
-        offset += chunk_size
+        generated_url
       end
     end
+
+    def headers_for_direct_upload(key, content_type:, checksum:, **)
+      { "Content-Type" => content_type, "Content-MD5" => checksum }
+    end
+
+    private
+      def object_for(key)
+        bucket.object(key)
+      end
+
+      # Reads the object for the given key in chunks, yielding each to the block.
+      def stream(key)
+        object = object_for(key)
+
+        chunk_size = 5.megabytes
+        offset = 0
+
+        while offset < object.content_length
+          yield object.get(range: "bytes=#{offset}-#{offset + chunk_size - 1}").body.read.force_encoding(Encoding::BINARY)
+          offset += chunk_size
+        end
+      end
+  end
 end