active_cipher_storage 1.0.0

data/active_cipher_storage.gemspec

@@ -0,0 +1,55 @@
+require_relative "lib/active_cipher_storage/version"
+
+Gem::Specification.new do |spec|
+  spec.name    = "active_cipher_storage"
+  spec.version = ActiveCipherStorage::VERSION
+  spec.authors = ["Jaspreet Singh"]
+  spec.email   = ["codebyjass@users.noreply.github.com"]
+
+  spec.summary     = "Transparent file encryption for Active Storage and S3 with pluggable KMS providers"
+  spec.description = <<~DESC
+    active_cipher_storage provides AES-256-GCM envelope encryption for files stored
+    via Rails Active Storage or directly via the AWS S3 SDK. Key management is
+    delegated to pluggable KMS providers: environment-variable keys, AWS KMS,
+    or any custom provider implementing the base interface.
+  DESC
+
+  spec.homepage              = "https://github.com/codebyjass/active-cipher-storage"
+  spec.license               = "MIT"
+  spec.required_ruby_version = ">= 3.2"
+
+  spec.metadata["bug_tracker_uri"]   = "#{spec.homepage}/issues"
+  spec.metadata["changelog_uri"]     = "#{spec.homepage}/blob/main/CHANGELOG.md"
+  spec.metadata["documentation_uri"] = spec.homepage
+  spec.metadata["homepage_uri"]      = spec.homepage
+  spec.metadata["rubygems_mfa_required"] = "true"
+  spec.metadata["source_code_uri"]   = spec.homepage
+
+  spec.files = Dir[
+    "lib/**/*.rb",
+    "CHANGELOG.md",
+    "CONTRIBUTING.md",
+    "LICENSE",
+    "README.md",
+    "SECURITY.md",
+    "active_cipher_storage.gemspec"
+  ]
+
+  spec.require_paths = ["lib"]
+
+  # Core — no runtime dep on Rails or AWS
+  spec.add_dependency "concurrent-ruby", "~> 1.2"
+
+  # Optional integrations — loaded only when the relevant adapter is used
+  spec.add_development_dependency "activestorage",    ">= 7.0", "< 9.0"
+  spec.add_development_dependency "aws-sdk-kms",      "~> 1.0"
+  spec.add_development_dependency "aws-sdk-s3",       "~> 1.0"
+
+  # Dev/test
+  spec.add_development_dependency "rspec",            "~> 3.12"
+  spec.add_development_dependency "rspec-mocks",      "~> 3.12"
+  spec.add_development_dependency "rubocop",          "~> 1.0"
+  spec.add_development_dependency "simplecov",        "~> 0.22"
+  spec.add_development_dependency "faker",            "~> 3.0"
+  spec.add_development_dependency "rake",             "~> 13.0"
+end

data/lib/active_cipher_storage/adapters/active_storage_service.rb

@@ -0,0 +1,129 @@
+module ActiveCipherStorage
+  module Adapters
+    # Active Storage service that transparently encrypts uploads and decrypts
+    # downloads. Configure in config/storage.yml:
+    #
+    #   encrypted_s3:
+    #     service: ActiveCipherStorage
+    #     wrapped_service: s3
+    #
+    # Backward compatibility
+    # ───────────────────────
+    # Blobs uploaded before encryption was enabled are detected via the "ACS\x01"
+    # magic header.  If the magic is absent the raw bytes are returned as-is,
+    # so the service is safe to enable on a bucket with existing plaintext objects.
+    #
+    # Range requests (download_chunk) must decrypt the full blob first because
+    # GCM authentication requires the complete ciphertext before any plaintext
+    # can be safely released.
+    class ActiveStorageService
+      BlobRef = Struct.new(:key)
+
+      attr_reader :inner
+
+      def self.build(configurator:, wrapped_service:, **kwargs)
+        new(wrapped_service: configurator.build(wrapped_service), **kwargs)
+      end
+
+      def initialize(wrapped_service:, **_kwargs)
+        @inner         = wrapped_service
+        @cipher        = Cipher.new
+        @stream_cipher = StreamCipher.new
+      end
+
+      def upload(key, io, checksum: nil, content_type: nil, filename: nil,
+                 disposition: nil, custom_metadata: {})
+        @inner.upload(key, encrypt_io(io),
+          checksum:        nil,  # checksum is over plaintext; skip for ciphertext
+          content_type:    "application/octet-stream",
+          filename:        filename,
+          disposition:     disposition,
+          custom_metadata: custom_metadata)
+
+        BlobMetadata.write(key, ActiveCipherStorage.configuration.provider)
+      end
+
+      def download(key, &block)
+        raw = collect_download(key)
+
+        # Legacy plaintext blob — no magic header present.
+        return (block ? yield(raw) : raw) unless cipher_payload?(raw)
+
+        plaintext = decrypt_raw(raw)
+        block ? yield(plaintext) : plaintext
+      end
+
+      def download_chunk(key, range)
+        download(key).b[range]
+      end
+
+      # Used by KeyRotation to fetch raw ciphertext without decrypting.
+      def download_raw(key)
+        collect_download(key)
+      end
+
+      # Used by KeyRotation to overwrite a blob's bytes without re-encrypting.
+      def upload_raw(key, io)
+        @inner.upload(key, io, content_type: "application/octet-stream")
+      end
+
+      # Re-wraps the DEK in a single blob's header under new_provider without
+      # decrypting or re-encrypting the file body.
+      def rekey(key, old_provider:, new_provider:)
+        KeyRotation.rotate_blob(
+          BlobRef.new(key),
+          old_provider: old_provider,
+          new_provider: new_provider,
+          service:      self
+        )
+      end
+
+      def delete(key)          = @inner.delete(key)
+      def delete_prefixed(pfx) = @inner.delete_prefixed(pfx)
+      def exist?(key)          = @inner.exist?(key)
+
+      def url(key, expires_in:, filename:, content_type:, disposition:, **)
+        @inner.url(key, expires_in: expires_in, filename: filename,
+                        content_type: content_type, disposition: disposition)
+      end
+
+      def url_for_direct_upload(*)
+        raise Errors::UnsupportedOperation,
+              "Direct uploads bypass encryption — use server-side upload instead"
+      end
+
+      def headers_for_direct_upload(*) = {}
+
+      private
+
+      def encrypt_io(io)
+        config = ActiveCipherStorage.configuration
+        if io.respond_to?(:size) && io.size && io.size > config.chunk_size
+          @stream_cipher.encrypt_to_io(io)
+        else
+          StringIO.new(@cipher.encrypt(io))
+        end
+      end
+
+      def collect_download(key)
+        buffer = StringIO.new("".b)
+        result = @inner.download(key) { |chunk| buffer.write(chunk) }
+        # Some services return data instead of yielding; handle both.
+        buffer.write(result.b) if buffer.pos.zero? && result.is_a?(String)
+        buffer.string
+      end
+
+      def decrypt_raw(raw)
+        io     = StringIO.new(raw.b)
+        header = Format.read_header(io)
+        io.rewind
+        header.chunked ? @stream_cipher.decrypt_to_io(io).read
+                       : @cipher.decrypt(io)
+      end
+
+      def cipher_payload?(data)
+        data.b.start_with?(Format::MAGIC)
+      end
+    end
+  end
+end

data/lib/active_cipher_storage/adapters/s3_adapter.rb

@@ -0,0 +1,252 @@
+require "stringio"
+
+module ActiveCipherStorage
+  module Adapters
+    class S3Adapter
+      include KeyUtils
+
+      DEFAULT_MULTIPART_THRESHOLD = 100 * 1024 * 1024
+
+      def initialize(bucket:, region: nil, multipart_threshold: DEFAULT_MULTIPART_THRESHOLD,
+                     s3_client: nil, config: nil)
+        @bucket              = bucket
+        @region              = region
+        @multipart_threshold = multipart_threshold
+        @client_override     = s3_client
+        @config              = config || ActiveCipherStorage.configuration
+        @config.validate!
+      end
+
+      def put_encrypted(key, io, **options)
+        large_file?(io) ? multipart_put(key, io, **options) : single_put(key, io, **options)
+      end
+
+      def get_decrypted(key)
+        resp = s3.get_object(bucket: @bucket, key: key)
+        decrypt_io(StringIO.new(resp.body.read.b))
+      end
+
+      # Streams decrypted plaintext from S3 without buffering the whole object.
+      # Yields each decrypted plaintext chunk as it becomes available.
+      # Safe for multi-gigabyte files: memory usage is bounded by chunk_size.
+      def stream_decrypted(key, &block)
+        raise ArgumentError, "stream_decrypted requires a block" unless block_given?
+
+        decryptor = StreamingDecryptor.new(@config)
+        s3.get_object(bucket: @bucket, key: key) do |s3_chunk|
+          decryptor.push(s3_chunk.b, &block)
+        end
+        decryptor.finish!
+      end
+
+      def presigned_url(key, expires_in: 3600)
+        Aws::S3::Presigner.new(client: s3)
+                          .presigned_url(:get_object, bucket: @bucket, key: key,
+                                         expires_in: expires_in)
+      end
+
+      def exist?(key)
+        s3.head_object(bucket: @bucket, key: key)
+        true
+      rescue Aws::S3::Errors::NotFound, Aws::S3::Errors::NoSuchKey
+        false
+      end
+
+      def delete(key)
+        s3.delete_object(bucket: @bucket, key: key)
+      end
+
+      private
+
+      def single_put(key, io, **options)
+        s3.put_object(bucket: @bucket, key: key,
+                      body: Cipher.new(@config).encrypt(io),
+                      **upload_options(options))
+      end
+
+      def multipart_put(key, io, **options)
+        require "aws-sdk-s3"
+        upload_id = s3.create_multipart_upload(bucket: @bucket, key: key,
+                                               **upload_options(options)).upload_id
+        parts = stream_multipart_parts(key, io, upload_id)
+        s3.complete_multipart_upload(bucket: @bucket, key: key, upload_id: upload_id,
+                                     multipart_upload: { parts: parts })
+      rescue StandardError
+        abort_multipart_upload(key, upload_id)
+        raise
+      end
+
+      def stream_multipart_parts(key, input_io, upload_id)
+        dek_bundle    = @config.provider.generate_data_key
+        plaintext_dek = dek_bundle.fetch(:plaintext_key)
+        parts         = []
+        part_number   = 0
+
+        # Buffer header + chunks together. S3 requires parts >= 5 MiB except the
+        # last, so we flush only when the buffer reaches chunk_size or at EOF.
+        buffer = StringIO.new("".b)
+        Format.write_header(buffer, Format::Header.new(
+          version:       Format::VERSION,
+          algorithm:     Format::ALGO_AES256GCM,
+          chunked:       true,
+          chunk_size:    @config.chunk_size,
+          provider_id:   @config.provider.provider_id,
+          encrypted_dek: dek_bundle.fetch(:encrypted_key)
+        ))
+
+        seq  = 0
+        done = false
+        until done
+          chunk     = input_io.read(@config.chunk_size) || "".b
+          done      = chunk.bytesize < @config.chunk_size
+          seq      += 1
+          frame_seq = done ? Format::FINAL_SEQ : seq
+          iv        = SecureRandom.random_bytes(Format::IV_SIZE)
+
+          c  = build_chunk_cipher(plaintext_dek, iv, frame_seq)
+          ct = chunk.empty? ? c.final : (c.update(chunk.b) + c.final)
+          Format.write_chunk(buffer, seq: frame_seq, iv: iv, ciphertext: ct, auth_tag: c.auth_tag)
+
+          next unless buffer.pos >= @config.chunk_size || done
+
+          buffer.rewind
+          part_number += 1
+          etag = s3.upload_part(bucket: @bucket, key: key, upload_id: upload_id,
+                                 part_number: part_number, body: buffer.read).etag
+          parts  << { part_number: part_number, etag: etag }
+          buffer = StringIO.new("".b)
+        end
+
+        parts
+      ensure
+        zero_bytes!(plaintext_dek)
+      end
+
+      def decrypt_io(io)
+        header = Format.read_header(io)
+        io.rewind
+        header.chunked ? StreamCipher.new(@config).decrypt_to_io(io)
+                       : StringIO.new(Cipher.new(@config).decrypt(io))
+      end
+
+      def large_file?(io)
+        size = io.respond_to?(:size) ? io.size : nil
+        size&.> @multipart_threshold
+      end
+
+      def upload_options(opts)
+        { content_type: "application/octet-stream" }.merge(opts.slice(:metadata, :tagging))
+      end
+
+      def build_chunk_cipher(key, iv, seq)
+        c = OpenSSL::Cipher.new(Cipher::OPENSSL_ALGO)
+        c.encrypt
+        c.key      = key
+        c.iv       = iv
+        c.auth_data = [seq].pack("N")
+        c
+      end
+
+      def s3
+        @s3 ||= begin
+          require "aws-sdk-s3"
+          @client_override || Aws::S3::Client.new(**{ region: @region }.compact)
+        end
+      rescue LoadError
+        raise Errors::ProviderError, "aws-sdk-s3 is required: add it to your Gemfile"
+      end
+
+      def abort_multipart_upload(key, upload_id)
+        return unless upload_id
+
+        s3.abort_multipart_upload(bucket: @bucket, key: key, upload_id: upload_id)
+      rescue StandardError => abort_error
+        @config.logger.warn(
+          "[ActiveCipherStorage] Could not abort multipart upload #{upload_id}: #{abort_error.message}"
+        )
+      end
+
+      # Accumulates bytes from a streaming S3 response, parses ACS frames as
+      # they arrive, and yields each decrypted plaintext chunk.
+      # Frame layout: seq(4) + iv(12) + ct_len(4) + ciphertext(ct_len) + auth_tag(16).
+      class StreamingDecryptor
+        include KeyUtils
+
+        FRAME_PREFIX_SIZE = 4 + Format::IV_SIZE + 4  # 20 bytes to determine ct_len
+
+        def initialize(config)
+          @config      = config
+          @buffer      = "".b
+          @dek         = nil
+          @header_done = false
+          @done        = false
+        end
+
+        def push(bytes, &block)
+          return if @done
+          @buffer += bytes.b
+          try_parse_header unless @header_done
+          drain_frames(&block) if @header_done
+        end
+
+        def finish!
+          raise Errors::InvalidFormat, "Stream ended before final frame" unless @done
+        ensure
+          zero_bytes!(@dek)
+        end
+
+        private
+
+        def try_parse_header
+          return if @buffer.bytesize < Format::MAGIC.bytesize
+
+          unless @buffer.start_with?(Format::MAGIC)
+            raise Errors::InvalidFormat, "Invalid magic bytes"
+          end
+
+          io     = StringIO.new(@buffer)
+          header = Format.read_header(io)
+          raise Errors::InvalidFormat, "Payload is not chunked; use #get_decrypted" unless header.chunked
+
+          @dek   = @config.provider.decrypt_data_key(header.encrypted_dek)
+          @buffer      = (@buffer.byteslice(io.pos..) || "".b).b
+          @header_done = true
+        rescue Errors::InvalidFormat => e
+          raise unless e.message.start_with?("Unexpected end of stream") && @buffer.bytesize <= 8192
+
+          # Need more bytes; keep buffering.
+        end
+
+        def drain_frames(&block)
+          until @done
+            break if @buffer.bytesize < FRAME_PREFIX_SIZE
+            ct_len     = @buffer.byteslice(16, 4).unpack1("N")
+            frame_size = FRAME_PREFIX_SIZE + ct_len + Format::AUTH_TAG_SIZE
+            break if @buffer.bytesize < frame_size
+
+            frame   = Format.read_chunk(StringIO.new(@buffer.byteslice(0, frame_size)))
+            @buffer = (@buffer.byteslice(frame_size..) || "".b).b
+
+            plaintext = decrypt_frame(frame)
+            block.call(plaintext) unless plaintext.empty?
+            @done = (frame[:seq] == Format::FINAL_SEQ)
+          end
+        end
+
+        def decrypt_frame(frame)
+          c = OpenSSL::Cipher.new(Cipher::OPENSSL_ALGO)
+          c.decrypt
+          c.key       = @dek
+          c.iv        = frame[:iv]
+          c.auth_tag  = frame[:auth_tag]
+          c.auth_data = [frame[:seq]].pack("N")
+          ct = frame[:ciphertext]
+          ct.empty? ? c.final : (c.update(ct) + c.final)
+        rescue OpenSSL::Cipher::CipherError
+          raise Errors::DecryptionError,
+                "Authentication failed on chunk seq=#{frame[:seq]} — data may be tampered"
+        end
+      end
+    end
+  end
+end

data/lib/active_cipher_storage/blob_metadata.rb

@@ -0,0 +1,84 @@
+module ActiveCipherStorage
+  # Reads and writes encryption metadata on ActiveStorage::Blob records.
+  #
+  # Written fields (all stored under the blob's existing `metadata` JSON column):
+  #   encrypted      => true
+  #   cipher_version => Integer  (Format::VERSION)
+  #   provider_id    => String   (e.g. "aws_kms", "env")
+  #   kms_key_id     => String   (CMK ARN, env-var name, or nil)
+  #
+  # These are for operational visibility — rotation queries, auditing,
+  # backward-compat detection. The encrypted file header is always the
+  # authoritative source for decryption.
+  module BlobMetadata
+    def self.write(storage_key, provider)
+      return unless active_storage_available?
+
+      blob = ActiveStorage::Blob.find_by(key: storage_key)
+      return unless blob
+
+      blob.update_columns(
+        metadata: blob.metadata.merge(
+          "encrypted"      => true,
+          "cipher_version" => Format::VERSION,
+          "provider_id"    => provider.provider_id,
+          "kms_key_id"     => provider.key_id
+        ).compact
+      )
+    rescue => e
+      ActiveCipherStorage.configuration.logger.warn(
+        "[ActiveCipherStorage] Could not write blob metadata for #{storage_key}: #{e.message}"
+      )
+    end
+
+    def self.update_after_rotation(storage_key, new_provider)
+      return unless active_storage_available?
+
+      blob = ActiveStorage::Blob.find_by(key: storage_key)
+      return unless blob
+
+      blob.update_columns(
+        metadata: blob.metadata.merge(
+          "provider_id" => new_provider.provider_id,
+          "kms_key_id"  => new_provider.key_id
+        ).compact
+      )
+    rescue => e
+      ActiveCipherStorage.configuration.logger.warn(
+        "[ActiveCipherStorage] Could not update rotation metadata for #{storage_key}: #{e.message}"
+      )
+    end
+
+    # Returns the metadata hash for a blob, or nil if AR is unavailable.
+    def self.for(storage_key)
+      return nil unless active_storage_available?
+      ActiveStorage::Blob.find_by(key: storage_key)&.metadata
+    end
+
+    # Finds all blobs whose metadata matches the given provider.
+    # Iterates in batches to avoid loading all blobs into memory.
+    # Yields each matching blob.
+    #
+    # For large tables, add a DB-level index on `metadata->>'kms_key_id'`
+    # and narrow the scope before passing to this method.
+    def self.blobs_for(provider)
+      return enum_for(:blobs_for, provider) unless block_given?
+      return unless active_storage_available?
+
+      ActiveStorage::Blob.find_each do |blob|
+        meta = blob.metadata
+        next unless meta["encrypted"] == true
+        next unless meta["provider_id"] == provider.provider_id
+        next if provider.key_id && meta["kms_key_id"] != provider.key_id
+
+        yield blob
+      end
+    end
+
+    private_class_method def self.active_storage_available?
+      defined?(ActiveStorage::Blob) && ActiveStorage::Blob.table_exists?
+    rescue
+      false
+    end
+  end
+end

data/lib/active_cipher_storage/cipher.rb

@@ -0,0 +1,97 @@
+require "openssl"
+require "securerandom"
+require "stringio"
+
+module ActiveCipherStorage
+  class Cipher
+    include KeyUtils
+
+    OPENSSL_ALGO = "aes-256-gcm"
+    KEY_SIZE     = 32
+
+    def initialize(config = ActiveCipherStorage.configuration)
+      config.validate!
+      @config   = config
+      @provider = config.provider
+    end
+
+    def encrypt(io)
+      plaintext  = io.read.b
+      dek_bundle = @provider.generate_data_key
+      key        = dek_bundle.fetch(:plaintext_key)
+      iv         = SecureRandom.random_bytes(Format::IV_SIZE)
+
+      c          = build_cipher(:encrypt, key, iv)
+      ciphertext = c.update(plaintext) + c.final
+      auth_tag   = c.auth_tag
+
+      out = StringIO.new("".b)
+      Format.write_header(out, header(dek_bundle, chunked: false))
+      out.write(iv)
+      out.write(ciphertext)
+      out.write(auth_tag)
+      out.string
+    ensure
+      zero_bytes!(key)
+      zero_bytes!(plaintext)
+    end
+
+    def decrypt(encrypted_data)
+      io     = to_binary_io(encrypted_data)
+      header = Format.read_header(io)
+      raise Errors::InvalidFormat, "Payload is chunked; use StreamCipher#decrypt" if header.chunked
+
+      key       = @provider.decrypt_data_key(header.encrypted_dek)
+      iv        = io.read(Format::IV_SIZE)
+      tail      = drain(io)
+      ciphertext = tail.byteslice(0, tail.bytesize - Format::AUTH_TAG_SIZE)
+      auth_tag   = tail.byteslice(-Format::AUTH_TAG_SIZE, Format::AUTH_TAG_SIZE)
+
+      c = build_cipher(:decrypt, key, iv, auth_tag)
+      c.update(ciphertext) + c.final
+    rescue OpenSSL::Cipher::CipherError
+      raise Errors::DecryptionError, "Authentication failed — ciphertext may be tampered or the key is wrong"
+    ensure
+      zero_bytes!(key)
+    end
+
+    private
+
+    def build_cipher(mode, key, iv, auth_tag = nil)
+      c = OpenSSL::Cipher.new(OPENSSL_ALGO)
+      mode == :encrypt ? c.encrypt : c.decrypt
+      c.key      = key
+      c.iv       = iv
+      c.auth_tag = auth_tag if auth_tag
+      c.auth_data = ""  # required by GCM even when empty
+      c
+    end
+
+    def header(dek_bundle, chunked:)
+      Format::Header.new(
+        version:       Format::VERSION,
+        algorithm:     Format::ALGO_AES256GCM,
+        chunked:       chunked,
+        chunk_size:    0,
+        provider_id:   @provider.provider_id,
+        encrypted_dek: dek_bundle.fetch(:encrypted_key)
+      )
+    end
+
+    def to_binary_io(data)
+      case data
+      when StringIO then data.tap(&:rewind)
+      when IO       then data
+      else StringIO.new(data.to_s.b)
+      end
+    end
+
+    def drain(io)
+      buf = "".b
+      while (chunk = io.read(65_536))
+        buf << chunk
+      end
+      buf
+    end
+  end
+end

data/lib/active_cipher_storage/configuration.rb

@@ -0,0 +1,57 @@
+require "logger"
+
+module ActiveCipherStorage
+  class Configuration
+    # Supported algorithm identifiers.
+    ALGORITHMS = %w[aes-256-gcm].freeze
+
+    # Bytes per plaintext chunk in streaming mode (default 5 MiB — matches the
+    # minimum S3 multipart part size, so each chunk maps to exactly one part).
+    DEFAULT_CHUNK_SIZE = 5 * 1024 * 1024
+
+    attr_reader   :provider
+    attr_accessor :algorithm, :chunk_size, :logger
+
+    def initialize
+      @algorithm  = "aes-256-gcm"
+      @chunk_size = DEFAULT_CHUNK_SIZE
+      @provider   = nil
+      @logger     = Logger.new($stdout, level: Logger::WARN)
+    end
+
+    # Accept a provider instance or a symbol shorthand (:env, :aws_kms).
+    def provider=(value)
+      @provider = case value
+                  when Symbol then resolve_provider(value)
+                  when Providers::Base then value
+                  else
+                    raise ArgumentError,
+                          "provider must be a Providers::Base instance or " \
+                          "one of :env, :aws_kms — got #{value.inspect}"
+                  end
+    end
+
+    def validate!
+      raise ProviderError, "No KMS provider configured. " \
+            "Set ActiveCipherStorage.configuration.provider." unless @provider
+
+      unless ALGORITHMS.include?(@algorithm)
+        raise ArgumentError, "Unsupported algorithm: #{@algorithm.inspect}. " \
+              "Supported: #{ALGORITHMS.join(', ')}"
+      end
+
+      raise ArgumentError, "chunk_size must be positive" unless @chunk_size.positive?
+    end
+
+    private
+
+    def resolve_provider(sym)
+      case sym
+      when :env     then Providers::EnvProvider.new
+      when :aws_kms then Providers::AwsKmsProvider.new
+      else
+        raise ArgumentError, "Unknown provider shorthand: #{sym.inspect}"
+      end
+    end
+  end
+end