active_storage_encryption 0.1.0

data/lib/active_storage_encryption/encrypted_disk_service.rb

@@ -0,0 +1,186 @@
+# frozen_string_literal: true
+
+require "block_cipher_kit"
+require "active_storage/service/disk_service"
+
+module ActiveStorageEncryption
+  # Provides a local encrypted store for ActiveStorage blobs.
+  # Configure it like so:
+  #
+  #   local_encrypted:
+  #     service: EncryptedDisk
+  #     root: <%= Rails.root.join("storage/encrypted") %>
+  #     private_url_policy: stream
+  class EncryptedDiskService < ::ActiveStorage::Service::DiskService
+    include ActiveStorageEncryption::PrivateUrlPolicy
+
+    autoload :V1Scheme, __dir__ + "/encrypted_disk_service/v1_scheme.rb"
+    autoload :V2Scheme, __dir__ + "/encrypted_disk_service/v2_scheme.rb"
+
+    FILENAME_EXTENSIONS_PER_SCHEME = {
+      ".encrypted-v1" => "V1Scheme",
+      ".encrypted-v2" => "V2Scheme"
+    }
+
+    # This lets the Blob encryption key methods know that this
+    # storage service _must_ use encryption
+    def encrypted? = true
+
+    def initialize(public: false, **options_for_disk_storage)
+      raise ArgumentError, "encrypted files cannot be served via a public URL or a CDN" if public
+      super
+    end
+
+    def upload(key, io, encryption_key:, checksum: nil, **)
+      instrument :upload, key: key, checksum: checksum do
+        scheme = create_scheme(key, encryption_key)
+        File.open(make_path_for(key), "wb") do |file|
+          scheme.streaming_encrypt(from_plaintext_io: io, into_ciphertext_io: file)
+        end
+        ensure_integrity_of(key, checksum, encryption_key) if checksum
+      end
+    end
+
+    def download(key, encryption_key:, &block)
+      if block_given?
+        instrument :streaming_download, key: key do
+          stream key, encryption_key, &block
+        end
+      else
+        instrument :download, key: key do
+          (+"").b.tap do |buf|
+            download(key, encryption_key: encryption_key) do |data|
+              buf << data
+            end
+          end
+        end
+      end
+    end
+
+    def download_chunk(key, range, encryption_key:)
+      instrument :download_chunk, key: key, range: range do
+        scheme = create_scheme(key, encryption_key)
+        File.open(path_for(key), "rb") do |file|
+          scheme.decrypt_range(from_ciphertext_io: file, range:)
+        end
+      rescue Errno::ENOENT
+        raise ActiveStorage::FileNotFoundError
+      end
+    end
+
+    def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:, encryption_key:, custom_metadata: {})
+      instrument :url, key: key do |payload|
+        upload_token = ActiveStorage.verifier.generate(
+          {
+            key: key,
+            content_type: content_type,
+            content_length: content_length,
+            encryption_key_sha256: Digest::SHA256.base64digest(encryption_key),
+            checksum: checksum,
+            service_name: name
+          },
+          expires_in: expires_in,
+          purpose: :encrypted_put
+        )
+
+        url_helpers = ActiveStorageEncryption::Engine.routes.url_helpers
+        url_helpers.encrypted_blob_put_url(upload_token, url_options).tap do |generated_url|
+          payload[:url] = generated_url
+        end
+      end
+    end
+
+    def path_for(key) # :nodoc:
+      # The extension indicates what encryption scheme the file will be using. This method
+      # gets used two ways - to get a path for a new object, and to get a path for an existing object.
+      # If an existing object is found, we need to return the path for the highest version of that
+      # object. If we want to create one - we always return the latest one.
+      glob_pattern = File.join(root, folder_for(key), key + ".encrypted-*")
+      last_existing_path = Dir.glob(glob_pattern).max
+      path_for_new_file = File.join(root, folder_for(key), key + FILENAME_EXTENSIONS_PER_SCHEME.keys.last)
+      last_existing_path || path_for_new_file
+    end
+
+    def exist?(key)
+      File.exist?(path_for(key))
+    end
+
+    def compose(source_keys, destination_key, source_encryption_keys:, encryption_key:, **)
+      if source_keys.length != source_encryption_keys.length
+        raise ArgumentError, "With #{source_keys.length} keys to compose there should be exactly as many source_encryption_keys, but got #{source_encryption_keys.length}"
+      end
+      File.open(make_path_for(destination_key), "wb") do |destination_file|
+        writing_scheme = create_scheme(destination_key, encryption_key)
+        writing_scheme.streaming_encrypt(into_ciphertext_io: destination_file) do |writable|
+          source_keys.zip(source_encryption_keys).each do |(source_key, encryption_key_for_source)|
+            File.open(path_for(source_key), "rb") do |source_file|
+              reading_scheme = create_scheme(source_key, encryption_key_for_source)
+              reading_scheme.streaming_decrypt(from_ciphertext_io: source_file, into_plaintext_io: writable)
+            end
+          end
+        end
+      end
+    end
+
+    def headers_for_direct_upload(key, content_type:, encryption_key:, checksum:, **)
+      # Both GCP and AWS require the key to be provided in the headers, together with the
+      # upload PUT request. This is not needed for the encrypted disk service, but it is
+      # useful to check it does get passed to the HTTP client and then to the upload -
+      # our controller extension will verify that this header is present, and fail if
+      # it is not in place.
+      super.merge!("x-active-storage-encryption-key" => Base64.strict_encode64(encryption_key), "content-md5" => checksum)
+    end
+
+    def headers_for_private_download(key, encryption_key:, **)
+      {"x-active-storage-encryption-key" => Base64.strict_encode64(encryption_key)}
+    end
+
+    private
+
+    def create_scheme(key, encryption_key_from_blob)
+      # Check whether this blob already exists and which version it is.
+      # path_for_key will give us the path to the existing version.
+      filename_extension = File.extname(path_for(key))
+      scheme_class_name = FILENAME_EXTENSIONS_PER_SCHEME.fetch(filename_extension)
+      scheme_class = self.class.const_get(scheme_class_name)
+      scheme_class.new(encryption_key_from_blob.b)
+    end
+
+    def private_url(key, **options)
+      private_url_for_streaming_via_controller(key, **options)
+    end
+
+    def public_url(key, filename:, encryption_key:, content_type: nil, disposition: :attachment, **)
+      raise "This should never be called"
+    end
+
+    def stream(key, encryption_key, &blk)
+      scheme = create_scheme(key, encryption_key)
+      File.open(path_for(key), "rb") do |file|
+        scheme.streaming_decrypt(from_ciphertext_io: file, &blk)
+      end
+    rescue Errno::ENOENT
+      raise ActiveStorage::FileNotFoundError
+    end
+
+    def ensure_integrity_of(key, checksum, encryption_key)
+      digest = OpenSSL::Digest.new("MD5")
+      stream(key, encryption_key) do |decrypted_data|
+        digest << decrypted_data
+      end
+      unless digest.base64digest == checksum
+        delete key
+        raise ActiveStorage::IntegrityError
+      end
+    end
+
+    def service_name
+      # Normally: ActiveStorage::Service::DiskService => Disk, so it does
+      # a split on "::" on the class name etc. Even though this is private,
+      # it does get called from the outside (or by other ActiveStorage::Service methods).
+      # Oddly it does _not_ get used in the `ActiveStorage::Configurator` to resolve
+      # the class to use.
+      "EncryptedDisk"
+    end
+  end
+end

data/lib/active_storage_encryption/encrypted_mirror_service.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require "active_storage/service/mirror_service"
+
+class ActiveStorageEncryption::EncryptedMirrorService < ActiveStorage::Service::MirrorService
+  delegate :private_url_policy, to: :primary
+
+  class MirrorJobWithEncryption < ActiveStorage::MirrorJob
+    def perform(key, checksum:, service_name:, encryption_key_token:)
+      service = lookup_service(service_name)
+      service.try(:mirror_with_encryption, key, checksum: checksum, encryption_key: encryption_key_from_token(encryption_key_token))
+    end
+
+    def encryption_key_from_token(encryption_key_token)
+      decrypted_token = ActiveStorageEncryption.token_encryptor.decrypt_and_verify(encryption_key_token, purpose: :mirror)
+      Base64.decode64(decrypted_token.fetch("encryption_key"))
+    end
+
+    def lookup_service(name)
+      # This should be the name in the config, NOT the class name
+      service = ActiveStorage::Blob.services.fetch(name) { ActiveStorage::Blob.service }
+      raise ArgumentError, "#{service.name} is not providing file encryption" unless service.try(:encrypted?)
+      service
+    end
+  end
+
+  def private_url_policy=(_)
+    raise ArgumentError, "EncryptedMirrorService uses the private_url_policy of the primary"
+  end
+
+  def encrypted?
+    true
+  end
+
+  def upload(key, io, encryption_key:, checksum: nil, **options)
+    io.rewind
+    if primary.try(:encrypted?)
+      primary.upload(key, io, checksum: checksum, encryption_key: encryption_key, **options)
+    else
+      primary.upload(key, io, checksum: checksum, **options)
+    end
+    mirror_later_with_encryption(key, checksum: checksum, encryption_key: encryption_key, **options)
+  end
+
+  def mirror_with_encryption(key, checksum:, encryption_key:)
+    instrument :mirror, key: key, checksum: checksum do
+      mirrors_in_need_of_mirroring = mirrors.select { |service| !service.exist?(key) }
+      return if mirrors_in_need_of_mirroring.empty?
+      primary.open(key, checksum: checksum, verify: checksum.present?, encryption_key: encryption_key) do |io|
+        mirrors_in_need_of_mirroring.each do |target|
+          io.rewind
+          options = target.try(:encrypted?) ? {encryption_key: encryption_key} : {}
+          target.upload(key, io, checksum: checksum, **options)
+        end
+      end
+    end
+  end
+
+  def service_name
+    # ActiveStorage::Service::DiskService => Disk
+    # Overridden because in Rails 8 this is "self.class.name.split("::").third.remove("Service")"
+    self.class.name.split("::").last.remove("Service")
+  end
+
+  private
+
+  def mirror_later_with_encryption(key, checksum:, encryption_key: nil)
+    encryption_key_token = ActiveStorageEncryption.token_encryptor.encrypt_and_sign(
+      {
+        encryption_key: Base64.strict_encode64(encryption_key)
+      },
+      purpose: :mirror
+    )
+    MirrorJobWithEncryption.perform_later(key, checksum: checksum, service_name:, encryption_key_token:)
+  end
+end

data/lib/active_storage_encryption/encrypted_s3_service.rb

@@ -0,0 +1,236 @@
+# frozen_string_literal: true
+
+require "active_storage/service/s3_service"
+
+class ActiveStorageEncryption::EncryptedS3Service < ActiveStorage::Service::S3Service
+  include ActiveStorageEncryption::PrivateUrlPolicy
+  def encrypted? = true
+
+  def initialize(public: false, **options_for_s3_service_and_private_url_policy)
+    raise ArgumentError, "encrypted files cannot be served via a public URL or a CDN" if public
+    super
+  end
+
+  def service_name
+    # ActiveStorage::Service::DiskService => Disk
+    # Overridden because in Rails 8 this is "self.class.name.split("::").third.remove("Service")"
+    self.class.name.split("::").last.remove("Service")
+  end
+
+  def headers_for_direct_upload(key, encryption_key:, **options_for_super)
+    # See https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerSideEncryptionCustomerKeys.html#specifying-s3-c-encryption
+    # This is the same as sse_options but expressed with raw header names
+    sdk_sse_options = sse_options(encryption_key)
+    super(key, **options_for_super).merge!({
+      "x-amz-server-side-encryption-customer-key" => Base64.strict_encode64(sdk_sse_options.fetch(:sse_customer_key)),
+      "x-amz-server-side-encryption-customer-key-MD5" => Digest::MD5.base64digest(sdk_sse_options.fetch(:sse_customer_key))
+    })
+  end
+
+  def exist?(key)
+    # The stock S3Service uses S3::Object#exists? here. That method does
+    # a HEAD request to the S3 bucket under the hood. But there is a problem
+    # with that approach: to get all the metadata attributes of an object on S3
+    # (which is what the HEAD request should return to you) you need the encryption key.
+    # The interface of the ActiveStorage services does not provide for extra arguments
+    # for `Service#exist?`, so all we would get using that SDK call would be an error.
+    #
+    # But we don't need the object metadata - we need to know is whether the object exists
+    # at all. And this can be done with a GET request instead. We ask S3 to give us the first byte of the
+    # object. S3 will then raise an exception - the exception will be different
+    # depending on whether the object does not exist _or_ the object does exist, but
+    # is encrypted. We can use the distinction between those exceptions to tell
+    # whether the object is there or not.
+    #
+    # There is also a case where the object is not encrypted - in that situation
+    # our single-byte GET request will actually succeed. This also means that the
+    # object exists in the bucket.
+    object_for(key).get(range: "bytes=0-0")
+    # If we get here without an exception - the object exists in the bucket,
+    # but is not encrypted. For example, it was stored using a stock S3Service.
+    true
+  rescue Aws::S3::Errors::InvalidRequest
+    # With this exception S3 tells us that the object exists but we have to furnish
+    # the encryption key (the exception will have a message with "object was stored
+    # using a form of Server Side Encryption...").
+    true
+  rescue Aws::S3::Errors::NoSuchKey
+    # And this truly means the object is not present
+    false
+  end
+
+  def headers_for_private_download(key, encryption_key:, **)
+    sdk_sse_options = sse_options(encryption_key)
+    {
+      "x-amz-server-side-encryption-customer-key" => Base64.strict_encode64(sdk_sse_options.fetch(:sse_customer_key))
+    }
+  end
+
+  def url_for_direct_upload(key, encryption_key:, **options_for_super)
+    # With direct upload we need to remove the encryption key itself from
+    # the SDK parameters. Otherwise it does get included in the URL, but that
+    # does not make S3 actually _use_ the value - _and_ it leaks the key.
+    # We _do_ need the key MD5 to be in the signed header params, so that the client can't use an encryption key
+    # it invents by itself - it must use the one we issue it.
+    sse_options_without_key = sse_options(encryption_key).without(:sse_customer_key)
+    with_upload_options_for_customer_key(sse_options_without_key) do
+      super(key, **options_for_super)
+    end
+  end
+
+  def upload(*args, encryption_key:, **kwargs)
+    with_upload_options_for_customer_key(sse_options(encryption_key)) do
+      super(*args, **kwargs)
+    end
+  end
+
+  def download(key, encryption_key:, &block)
+    if block_given?
+      instrument :streaming_download, key: key do
+        stream(key, encryption_key: encryption_key, &block)
+      end
+    else
+      instrument :download, key: key do
+        object_for(key).get(**sse_options(encryption_key)).body.string.force_encoding(Encoding::BINARY)
+      rescue Aws::S3::Errors::NoSuchKey
+        raise ActiveStorage::FileNotFoundError
+      end
+    end
+  end
+
+  def download_chunk(key, range, encryption_key:)
+    instrument :download_chunk, key: key, range: range do
+      object_for(key).get(range: "bytes=#{range.begin}-#{range.exclude_end? ? range.end - 1 : range.end}", **sse_options(encryption_key)).body.string.force_encoding(Encoding::BINARY)
+    rescue Aws::S3::Errors::NoSuchKey
+      raise ActiveStorage::FileNotFoundError
+    end
+  end
+
+  def compose(source_keys, destination_key, source_encryption_keys:, encryption_key:, filename: nil, content_type: nil, disposition: nil, custom_metadata: {})
+    if source_keys.length != source_encryption_keys.length
+      raise ArgumentError, "With #{source_keys.length} keys to compose there should be exactly as many source_encryption_keys, but got #{source_encryption_keys.length}"
+    end
+    content_disposition = content_disposition_with(type: disposition, filename: filename) if disposition && filename
+    upload_options_for_compose = upload_options.merge(sse_options(encryption_key))
+    object_for(destination_key).upload_stream(
+      content_type: content_type,
+      content_disposition: content_disposition,
+      part_size: MINIMUM_UPLOAD_PART_SIZE,
+      metadata: custom_metadata,
+      **upload_options_for_compose
+    ) do |s3_multipart_io|
+      s3_multipart_io.binmode
+      source_keys.zip(source_encryption_keys).each do |(source_key, source_encryption_key)|
+        stream(source_key, encryption_key: source_encryption_key) do |chunk|
+          s3_multipart_io.write(chunk)
+        end
+      end
+    end
+  end
+
+  private
+
+  # Reads the object for the given key in chunks, yielding each to the block.
+  def stream(key, encryption_key:)
+    object = object_for(key)
+
+    chunk_size = 5.megabytes
+    offset = 0
+
+    # Doing a HEAD (what .exists? does under the hood) also requires the encryption key headers,
+    # but the SDK does not send them along. Instead of doing a HEAD, you can also do a GET - but for the first byte.
+    # This will give you the content-length of the object, and the SDK will pass the correct encryption headers.
+    # There is an issue in the SDK here https://github.com/aws/aws-sdk-ruby/issues/1342 which is allegedly fixed
+    # by https://github.com/aws/aws-sdk-ruby/pull/1343/files but it doesn't seem like it.
+    # Also, we do not only call `S3::Object#exists?`, but also `S3::Object#content_length` - which does not have a way to pass
+    # encryption options either.
+    response = object.get(range: "bytes=0-0", **sse_options(encryption_key))
+    object_content_length = response.content_range.scan(/\d+$/).first.to_i
+
+    while offset < object_content_length
+      yield object.get(range: "bytes=#{offset}-#{offset + chunk_size - 1}", **sse_options(encryption_key)).body.string.force_encoding(Encoding::BINARY)
+      offset += chunk_size
+    end
+  rescue Aws::S3::Errors::NoSuchKey
+    raise ActiveStorage::FileNotFoundError
+  end
+
+  def sse_options(encryption_key)
+    truncated_key_bytes = encryption_key.byteslice(0, 32)
+    {
+      sse_customer_algorithm: "AES256",
+      sse_customer_key: truncated_key_bytes,
+      sse_customer_key_md5: Digest::MD5.base64digest(truncated_key_bytes)
+    }
+  end
+
+  def private_url(key, encryption_key:, **options)
+    case private_url_policy
+    when :disable
+      if private_url_policy == :disable
+        raise ActiveStorageEncryption::StreamingDisabled, <<~EOS
+          Requested a signed GET URL for #{key.inspect} on service #{name}. This service
+          has disabled presigned URLs (private_url_policy: disable), you have to use `Blob#download` instead.
+        EOS
+      end
+    when :stream
+      private_url_for_streaming_via_controller(key, encryption_key:, **options)
+    when :require_headers
+      sse_options_for_presigned_url = sse_options(encryption_key)
+
+      # Remove the key itself. If we pass it to the SDK - it will leak the key (the key will be in the URL),
+      # but the download will still fail.
+      sse_options_for_presigned_url.delete(:sse_customer_key)
+
+      options_for_super = options.merge(sse_options_for_presigned_url) # The "rest" kwargs for super are the `client_options`
+      super(key, **options_for_super)
+    end
+  end
+
+  def public_url(key, **client_opts)
+    raise "This should never be called"
+  end
+
+  def upload_options
+    super.merge(Thread.current[:aws_sse_options].to_h)
+  end
+
+  def with_upload_options_for_customer_key(overriding_upload_options)
+    # Gotta be careful here, because this call can be re-entrant.
+    # If one thread calls `upload_options` to do an upload, and does not
+    # return for some time, we want this thread to be using the upload options
+    # reserved for it - otherwise objects can get not their encryption keys, but
+    # others'. If we want to have upload_options be tailored to every specific upload,
+    # we would need to override way more of this Service class than is really needed.
+    # You can actually see that sometimes there is reentrancy here:
+    #
+    # MUX = Mutex.new
+    # opens_before = MUX.synchronize { @opens ||= 0; @opens += 1; @opens - 1 }
+    previous = Thread.current[:aws_sse_options]
+    Thread.current[:aws_sse_options] = overriding_upload_options
+    yield
+  ensure
+    # To check that there is reentrancy:
+    # opens_after = MUX.synchronize { @opens -= 1 }
+    # warn [opens_before, opens_after].inspect #exiting wo"
+    # In our tests:
+    # [2, 11]
+    # [10, 10]
+    # [0, 9]
+    # [9, 8]
+    # [5, 7]
+    # [3, 6]
+    # [6, 5]
+    # [1, 4]
+    # [8, 3]
+    # [4, 2]
+    # [7, 1]
+    # [11, 0]
+    # [0, 0]
+    # [0, 0]
+    # [0, 0]
+    # [0, 0]
+    # [0, 0]
+    Thread.current[:aws_sse_options] = previous
+  end
+end

data/lib/active_storage_encryption/engine.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module ActiveStorageEncryption
+  class Engine < ::Rails::Engine
+    isolate_namespace ActiveStorageEncryption
+  end
+end