active_storage_encryption 0.1.0

data/gemfiles/rails_8.gemfile.lock

@@ -0,0 +1,276 @@
+PATH
+  remote: ..
+  specs:
+    active_storage_encryption (0.1.0)
+      activestorage
+      block_cipher_kit (>= 0.0.4)
+      rails (>= 7.2.2.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    actioncable (8.0.1)
+      actionpack (= 8.0.1)
+      activesupport (= 8.0.1)
+      nio4r (~> 2.0)
+      websocket-driver (>= 0.6.1)
+      zeitwerk (~> 2.6)
+    actionmailbox (8.0.1)
+      actionpack (= 8.0.1)
+      activejob (= 8.0.1)
+      activerecord (= 8.0.1)
+      activestorage (= 8.0.1)
+      activesupport (= 8.0.1)
+      mail (>= 2.8.0)
+    actionmailer (8.0.1)
+      actionpack (= 8.0.1)
+      actionview (= 8.0.1)
+      activejob (= 8.0.1)
+      activesupport (= 8.0.1)
+      mail (>= 2.8.0)
+      rails-dom-testing (~> 2.2)
+    actionpack (8.0.1)
+      actionview (= 8.0.1)
+      activesupport (= 8.0.1)
+      nokogiri (>= 1.8.5)
+      rack (>= 2.2.4)
+      rack-session (>= 1.0.1)
+      rack-test (>= 0.6.3)
+      rails-dom-testing (~> 2.2)
+      rails-html-sanitizer (~> 1.6)
+      useragent (~> 0.16)
+    actiontext (8.0.1)
+      actionpack (= 8.0.1)
+      activerecord (= 8.0.1)
+      activestorage (= 8.0.1)
+      activesupport (= 8.0.1)
+      globalid (>= 0.6.0)
+      nokogiri (>= 1.8.5)
+    actionview (8.0.1)
+      activesupport (= 8.0.1)
+      builder (~> 3.1)
+      erubi (~> 1.11)
+      rails-dom-testing (~> 2.2)
+      rails-html-sanitizer (~> 1.6)
+    activejob (8.0.1)
+      activesupport (= 8.0.1)
+      globalid (>= 0.3.6)
+    activemodel (8.0.1)
+      activesupport (= 8.0.1)
+    activerecord (8.0.1)
+      activemodel (= 8.0.1)
+      activesupport (= 8.0.1)
+      timeout (>= 0.4.0)
+    activestorage (8.0.1)
+      actionpack (= 8.0.1)
+      activejob (= 8.0.1)
+      activerecord (= 8.0.1)
+      activesupport (= 8.0.1)
+      marcel (~> 1.0)
+    activesupport (8.0.1)
+      base64
+      benchmark (>= 0.3)
+      bigdecimal
+      concurrent-ruby (~> 1.0, >= 1.3.1)
+      connection_pool (>= 2.2.5)
+      drb
+      i18n (>= 1.6, < 2)
+      logger (>= 1.4.2)
+      minitest (>= 5.1)
+      securerandom (>= 0.3)
+      tzinfo (~> 2.0, >= 2.0.5)
+      uri (>= 0.13.1)
+    appraisal (2.5.0)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    ast (2.4.2)
+    aws-eventstream (1.3.1)
+    aws-partitions (1.1060.0)
+    aws-sdk-core (3.220.0)
+      aws-eventstream (~> 1, >= 1.3.0)
+      aws-partitions (~> 1, >= 1.992.0)
+      aws-sigv4 (~> 1.9)
+      base64
+      jmespath (~> 1, >= 1.6.1)
+    aws-sdk-kms (1.99.0)
+      aws-sdk-core (~> 3, >= 3.216.0)
+      aws-sigv4 (~> 1.5)
+    aws-sdk-s3 (1.182.0)
+      aws-sdk-core (~> 3, >= 3.216.0)
+      aws-sdk-kms (~> 1)
+      aws-sigv4 (~> 1.5)
+    aws-sigv4 (1.11.0)
+      aws-eventstream (~> 1, >= 1.0.2)
+    base64 (0.2.0)
+    benchmark (0.4.0)
+    bigdecimal (3.1.9)
+    block_cipher_kit (0.0.4)
+    builder (3.3.0)
+    concurrent-ruby (1.3.5)
+    connection_pool (2.5.0)
+    crass (1.0.6)
+    date (3.4.1)
+    drb (2.2.1)
+    erubi (1.13.1)
+    globalid (1.2.1)
+      activesupport (>= 6.1)
+    i18n (1.14.7)
+      concurrent-ruby (~> 1.0)
+    io-console (0.8.0)
+    irb (1.15.1)
+      pp (>= 0.6.0)
+      rdoc (>= 4.0.0)
+      reline (>= 0.4.2)
+    jmespath (1.6.2)
+    json (2.10.1)
+    language_server-protocol (3.17.0.4)
+    lint_roller (1.1.0)
+    logger (1.6.6)
+    loofah (2.24.0)
+      crass (~> 1.0.2)
+      nokogiri (>= 1.12.0)
+    magic_frozen_string_literal (1.2.0)
+    mail (2.8.1)
+      mini_mime (>= 0.1.1)
+      net-imap
+      net-pop
+      net-smtp
+    marcel (1.0.4)
+    mini_mime (1.1.5)
+    minitest (5.25.4)
+    net-http (0.6.0)
+      uri
+    net-imap (0.5.6)
+      date
+      net-protocol
+    net-pop (0.1.2)
+      net-protocol
+    net-protocol (0.2.2)
+      timeout
+    net-smtp (0.5.1)
+      net-protocol
+    nio4r (2.7.4)
+    nokogiri (1.18.3-x86_64-darwin)
+      racc (~> 1.4)
+    nokogiri (1.18.3-x86_64-linux-gnu)
+      racc (~> 1.4)
+    parallel (1.26.3)
+    parser (3.3.7.1)
+      ast (~> 2.4.1)
+      racc
+    pp (0.6.2)
+      prettyprint
+    prettyprint (0.2.0)
+    psych (5.2.3)
+      date
+      stringio
+    racc (1.8.1)
+    rack (3.1.11)
+    rack-session (2.1.0)
+      base64 (>= 0.1.0)
+      rack (>= 3.0.0)
+    rack-test (2.2.0)
+      rack (>= 1.3)
+    rackup (2.2.1)
+      rack (>= 3)
+    rails (8.0.1)
+      actioncable (= 8.0.1)
+      actionmailbox (= 8.0.1)
+      actionmailer (= 8.0.1)
+      actionpack (= 8.0.1)
+      actiontext (= 8.0.1)
+      actionview (= 8.0.1)
+      activejob (= 8.0.1)
+      activemodel (= 8.0.1)
+      activerecord (= 8.0.1)
+      activestorage (= 8.0.1)
+      activesupport (= 8.0.1)
+      bundler (>= 1.15.0)
+      railties (= 8.0.1)
+    rails-dom-testing (2.2.0)
+      activesupport (>= 5.0.0)
+      minitest
+      nokogiri (>= 1.6)
+    rails-html-sanitizer (1.6.2)
+      loofah (~> 2.21)
+      nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0)
+    railties (8.0.1)
+      actionpack (= 8.0.1)
+      activesupport (= 8.0.1)
+      irb (~> 1.13)
+      rackup (>= 1.0.0)
+      rake (>= 12.2)
+      thor (~> 1.0, >= 1.2.2)
+      zeitwerk (~> 2.6)
+    rainbow (3.1.1)
+    rake (13.2.1)
+    rdoc (6.12.0)
+      psych (>= 4.0.0)
+    regexp_parser (2.10.0)
+    reline (0.6.0)
+      io-console (~> 0.5)
+    rubocop (1.71.2)
+      json (~> 2.3)
+      language_server-protocol (>= 3.17.0)
+      parallel (~> 1.10)
+      parser (>= 3.3.0.2)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 2.9.3, < 3.0)
+      rubocop-ast (>= 1.38.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 2.4.0, < 4.0)
+    rubocop-ast (1.38.1)
+      parser (>= 3.3.1.0)
+    rubocop-performance (1.23.1)
+      rubocop (>= 1.48.1, < 2.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
+    ruby-progressbar (1.13.0)
+    securerandom (0.4.1)
+    sqlite3 (2.6.0-x86_64-darwin)
+    sqlite3 (2.6.0-x86_64-linux-gnu)
+    standard (1.45.0)
+      language_server-protocol (~> 3.17.0.2)
+      lint_roller (~> 1.0)
+      rubocop (~> 1.71.0)
+      standard-custom (~> 1.0.0)
+      standard-performance (~> 1.6)
+    standard-custom (1.0.2)
+      lint_roller (~> 1.0)
+      rubocop (~> 1.50)
+    standard-performance (1.6.0)
+      lint_roller (~> 1.1)
+      rubocop-performance (~> 1.23.0)
+    stringio (3.1.5)
+    thor (1.3.2)
+    timeout (0.4.3)
+    tzinfo (2.0.6)
+      concurrent-ruby (~> 1.0)
+    unicode-display_width (3.1.4)
+      unicode-emoji (~> 4.0, >= 4.0.4)
+    unicode-emoji (4.0.4)
+    uri (1.0.3)
+    useragent (0.16.11)
+    websocket-driver (0.7.7)
+      base64
+      websocket-extensions (>= 0.1.0)
+    websocket-extensions (0.1.5)
+    zeitwerk (2.7.2)
+
+PLATFORMS
+  x86_64-darwin
+  x86_64-linux
+
+DEPENDENCIES
+  active_storage_encryption!
+  appraisal
+  aws-sdk-s3
+  magic_frozen_string_literal
+  net-http
+  rails (>= 8.0)
+  rake
+  sqlite3
+  standard (>= 1.35.1)
+
+BUNDLED WITH
+   2.5.11

data/lib/active_storage/service/encrypted_disk_service.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+# Needed so that Rails can find our service definition. It will perform the following
+# steps. Given an "EncryptedDisk" value of the `service:` key in the YAML, it will:
+#
+# * Force-require a file at "active_storage/service/encrypted_disk", from any path on the $LOAD_PATH
+# * Instantiate a class called "ActiveStorage::Service::EncryptedDiskService"
+require_relative "../../active_storage_encryption"
+class ActiveStorage::Service::EncryptedDiskService < ActiveStorageEncryption::EncryptedDiskService
+end

data/lib/active_storage/service/encrypted_mirror_service.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+# Needed so that Rails can find our service definition. It will perform the following
+# steps. Given an "EncryptedDisk" value of the `service:` key in the YAML, it will:
+#
+# * Force-require a file at "active_storage/service/encrypted_disk", from any path on the $LOAD_PATH
+# * Instantiate a class called "ActiveStorage::Service::EncryptedDiskService"
+require_relative "../../active_storage_encryption/active_storage_encryption"
+class ActiveStorage::Service::EncryptedMirrorService < ActiveStorageEncryption::EncryptedMirrorService
+end

data/lib/active_storage/service/encrypted_s3_service.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+# Needed so that Rails can find our service definition. It will perform the following
+# steps. Given an "EncryptedDisk" value of the `service:` key in the YAML, it will:
+#
+# * Force-require a file at "active_storage/service/encrypted_disk", from any path on the $LOAD_PATH
+# * Instantiate a class called "ActiveStorage::Service::EncryptedDiskService"
+require_relative "../../active_storage_encryption"
+class ActiveStorage::Service::EncryptedS3Service < ActiveStorageEncryption::EncryptedS3Service
+end

data/lib/active_storage_encryption/encrypted_blobs_controller.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+class ActiveStorageEncryption::EncryptedBlobsController < ActionController::Base
+  include ActiveStorage::SetCurrent
+
+  # Below similar to ActiveStorage::Streaming but ActionController::Live is meh.
+  include ActionController::DataStreaming
+  include ActionController::Live
+
+  class InvalidParams < StandardError
+  end
+
+  DEFAULT_BLOB_STREAMING_DISPOSITION = "inline"
+
+  self.etag_with_template_digest = false
+  skip_forgery_protection
+
+  # Accepts PUT requests for direct uploads to the EncryptedDiskService. It can actually accept
+  # uploads to any encrypted service, but for S3 and GCP the upload can be done to the cloud storage
+  # bucket directly.
+  def update
+    params = read_params_from_token_and_headers_for_put
+    service = lookup_service(params[:service_name])
+    key = params[:key]
+
+    service.upload(key, request.body,
+      content_type: params[:content_type],
+      content_length: params[:content_length],
+      checksum: params[:checksum],
+      encryption_key: params[:encryption_key])
+  rescue InvalidParams, ActiveStorageEncryption::IncorrectEncryptionKey, ActiveSupport::MessageVerifier::InvalidSignature, ActiveStorage::IntegrityError
+    head :unprocessable_entity
+  end
+
+  # Streams the decrypted contents of an encrypted blob
+  def show
+    params = read_params_from_token_and_headers_for_get
+    service = lookup_service(params[:service_name])
+    raise InvalidParams, "#{service.name} does not allow private URLs" if service.private_url_policy == :disable
+
+    key = params[:key]
+    encryption_key = params[:encryption_key]
+
+    send_stream(filename: params[:filename], disposition: params[:disposition] || DEFAULT_BLOB_STREAMING_DISPOSITION, type: params[:content_type]) do |stream|
+      service.download(key, encryption_key: encryption_key) do |chunk|
+        stream.write chunk
+      end
+    end
+  rescue InvalidParams, ActiveStorageEncryption::StreamingTokenInvalidOrExpired, ActiveSupport::MessageEncryptor::InvalidMessage, ActiveStorageEncryption::IncorrectEncryptionKey
+    head :forbidden
+  end
+
+  # Creates a Blob record with a random encryption key and returns the details for PUTing it
+  # This is only necessary because in Rails there is some disagreement regarding the service_name parameter.
+  # See https://github.com/rails/rails/issues/38940
+  # It does not require the service to support encryption. However, we mandate that the MD5 be provided upfront,
+  # so that it gets included into the signature
+  def create_direct_upload
+    blob_params = params.require(:blob).permit(:filename, :byte_size, :checksum, :content_type, metadata: {})
+    unless blob_params[:checksum]
+      render(plain: "The `checksum' is required", status: :unprocessable_entity) and return
+    end
+
+    service = lookup_service(params.require(:service_name))
+    blob = ActiveStorage::Blob.create_before_direct_upload!(
+      **blob_params.to_h.symbolize_keys,
+      service_name: service.name
+    )
+    render json: direct_upload_json(blob)
+  end
+
+  private
+
+  def read_params_from_token_and_headers_for_put
+    token_str = params.require(:token)
+
+    # The token params for PUT / direct upload are signed but not encrypted - the encryption key
+    # is transmitted inside headers
+    token_params = ActiveStorage.verifier.verify(token_str, purpose: :encrypted_put).symbolize_keys
+
+    # Ensure we are getting sent exactly as many bytes as stated in the token
+    raise InvalidParams, "Request must specify body content-length" if request.headers["content-length"].blank?
+
+    actual_content_length = request.headers["content-length"].to_i
+    expected_content_length = token_params.fetch(:content_length)
+    if actual_content_length != expected_content_length
+      raise InvalidParams, "content-length mismatch, expecting upload of #{expected_content_length} bytes but sent #{actual_content_length}"
+    end
+
+    # Recover the encryption key from the headers (similar to how cloud storage services do it)
+    b64_encryption_key = request.headers["x-active-storage-encryption-key"]
+    raise InvalidParams, "x-active-storage-encryption-key header is missing" if b64_encryption_key.blank?
+    encryption_key = Base64.strict_decode64(b64_encryption_key)
+
+    # Verify the SHA of the encryption key
+    encryption_key_b64sha = Digest::SHA256.base64digest(encryption_key)
+    raise InvalidParams, "Incorrect checksum for the encryption key" unless Rack::Utils.secure_compare(encryption_key_b64sha, token_params.fetch(:encryption_key_sha256))
+
+    # Verify the Content-MD5
+    b64_md5_from_headers = request.headers["content-md5"]
+    raise InvalidParams, "Content-MD5 header is required" if b64_md5_from_headers.blank?
+    raise InvalidParams, "Content-MD5 differs from the known checksum" unless Rack::Utils.secure_compare(b64_md5_from_headers, token_params.fetch(:checksum))
+
+    {
+      key: token_params.fetch(:key),
+      encryption_key: encryption_key,
+      service_name: token_params.fetch(:service_name),
+      checksum: token_params[:checksum],
+      content_type: token_params.fetch(:content_type),
+      content_length: token_params.fetch(:content_length)
+    }
+  end
+
+  def read_params_from_token_and_headers_for_get
+    token_str = params.require(:token)
+
+    # The token params for GET / private_url download are encrypted, as they contain the object encryption key.
+    token_params = ActiveStorageEncryption.token_encryptor.decrypt_and_verify(token_str, purpose: :encrypted_get).symbolize_keys
+    encryption_key = Base64.decode64(token_params.fetch(:encryption_key))
+
+    service = lookup_service(token_params.fetch(:service_name))
+
+    # To be more like cloud services: verify presence of headers, if we were asked to (but this is optional)
+    if service.private_url_policy == :require_headers
+      b64_encryption_key = request.headers["x-active-storage-encryption-key"]
+      raise InvalidParams, "x-active-storage-encryption-key header is missing" if b64_encryption_key.blank?
+      raise InvalidParams, "Incorrect encryption key supplied via header" unless Rack::Utils.secure_compare(Base64.decode64(b64_encryption_key), encryption_key)
+    end
+
+    # Verify the SHA of the encryption key
+    encryption_key_b64sha = Digest::SHA256.base64digest(encryption_key)
+    raise InvalidParams, "Incorrect encryption key supplied via token" unless Rack::Utils.secure_compare(encryption_key_b64sha, token_params.fetch(:encryption_key_sha256))
+
+    {
+      key: token_params.fetch(:key),
+      encryption_key: encryption_key,
+      service_name: token_params.fetch(:service_name),
+      disposition: token_params.fetch(:disposition),
+      content_type: token_params.fetch(:content_type)
+    }
+  end
+
+  def lookup_service(name)
+    service = ActiveStorage::Blob.services.fetch(name) { ActiveStorage::Blob.service }
+    raise InvalidParams, "#{service.name} is not providing file encryption" unless service.try(:encrypted?)
+    service
+  end
+
+  def blob_args
+    params.require(:blob).permit(:filename, :byte_size, :checksum, :content_type, :service_name, metadata: {}).to_h.symbolize_keys
+  end
+
+  def service_name_from_params_or_config
+    params[:service_name] || ActiveStorage::Blob.service.name # ? Rails.application.config.active_storage.service.name
+  end
+
+  def direct_upload_json(blob)
+    blob.as_json(root: false, methods: :signed_id).merge(direct_upload: {
+      url: blob.service_url_for_direct_upload,
+      headers: blob.service_headers_for_direct_upload
+    })
+  end
+end

data/lib/active_storage_encryption/encrypted_disk_service/v1_scheme.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+class ActiveStorageEncryption::EncryptedDiskService::V1Scheme
+  def initialize(encryption_key)
+    @scheme = BlockCipherKit::AES256CFBCIVScheme.new(encryption_key)
+    @key_digest = Digest::SHA256.digest(encryption_key.byteslice(0, 16 + 32)) # In this scheme the IV is suffixed with the key
+  end
+
+  def streaming_decrypt(from_ciphertext_io:, into_plaintext_io: nil, &blk)
+    validate_key!(from_ciphertext_io)
+    @scheme.streaming_decrypt(from_ciphertext_io:, into_plaintext_io:, &blk)
+  end
+
+  def streaming_encrypt(into_ciphertext_io:, from_plaintext_io: nil, &blk)
+    into_ciphertext_io.write(@key_digest)
+    @scheme.streaming_encrypt(into_ciphertext_io:, from_plaintext_io:, &blk)
+  end
+
+  def decrypt_range(from_ciphertext_io:, range:)
+    validate_key!(from_ciphertext_io)
+    @scheme.decrypt_range(from_ciphertext_io:, range:)
+  end
+
+  def validate_key!(io)
+    key_digest_from_io = io.read(@key_digest.bytesize)
+    raise ActiveStorageEncryption::IncorrectEncryptionKey unless key_digest_from_io == @key_digest
+  end
+end

data/lib/active_storage_encryption/encrypted_disk_service/v2_scheme.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+# This scheme uses GCM encryption with CTR-based random access. The auth tag is stored
+# at the end of the message. The message is prefixed by a SHA2 digest of the encryption key.
+class ActiveStorageEncryption::EncryptedDiskService::V2Scheme
+  def initialize(encryption_key)
+    @scheme = BlockCipherKit::AES256GCMScheme.new(encryption_key)
+    @key_digest = Digest::SHA256.digest(encryption_key.byteslice(0, 32)) # In this scheme just the key is used
+  end
+
+  def streaming_decrypt(from_ciphertext_io:, into_plaintext_io: nil, &blk)
+    check_key!(from_ciphertext_io)
+    @scheme.streaming_decrypt(from_ciphertext_io:, into_plaintext_io:, &blk)
+  end
+
+  def streaming_encrypt(into_ciphertext_io:, from_plaintext_io: nil, &blk)
+    # See check_key! for rationale. We need a fast KVC (key validation code)
+    # to refuse the download if we know the key is incorrect.
+    into_ciphertext_io.write(@key_digest)
+    @scheme.streaming_encrypt(into_ciphertext_io:, from_plaintext_io:, &blk)
+  end
+
+  def decrypt_range(from_ciphertext_io:, range:)
+    check_key!(from_ciphertext_io)
+    @scheme.decrypt_range(from_ciphertext_io:, range:)
+  end
+
+  private def check_key!(io)
+    # We need a fast KCV (key check value) to refuse the download
+    # if we know the key is incorrect. We can't use the auth tag from GCM
+    # because it can only be computed if the entirety of the ciphertext has been read by the
+    # cipher - and we want random access. We could use a HMAC(encryption_key, auth_tag) at the
+    # tail of ciphertext to achieve the same, but that would require streaming_decrypt to seek inside
+    # the ciphertext IO to read the tail of the file - which we don't want to require.
+    #
+    # Besides, we want to not tie up server resources if we know
+    # that the furnished encryption key is incorrect. So: a KVC.
+    #
+    # We store the SHA2 value of the encryption key at the start of the ciphertext. We assume that the encryption
+    # key will be generated randomly and will be very high-entropy, so the only attack strategy for it is brute-force.
+    # Brute-force is keyspace / hashrate, as explained here: https://stackoverflow.com/questions/4764026/how-many-sha256-hashes-can-a-modern-computer-compute
+    # which, for our key of 32 bytes, gives us this calculation to find out the number of years to crack this SHA on
+    # a GeForce 2080Ti (based on https://hashcat.net/forum/thread-10185.html):
+    # ((256 ** 32) / (7173 * 1000 * 1000)) / 60 / 60 / 24 / 365
+    # which is
+    # 511883878862512581460395486615240253212171357229849212045742
+    # This is quite some years. So storing the digest of the key is reasonably safe.
+    key_digest_from_io = io.read(@key_digest.bytesize)
+    raise ActiveStorageEncryption::IncorrectEncryptionKey unless key_digest_from_io == @key_digest
+  end
+end