lockbox 0.4.1 → 0.4.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fe55b94af6abc4b2cf562badea225c8aff715ae3be8cc40e81f645ae6c0a6a9b
-  data.tar.gz: dd6c5dbd4d3280548212ca145f56b99909b5d4af07784bc095fc1df6afd156ff
+  metadata.gz: 2fb82c6baf0c56ae7f051c8363f7cb52554ebab8180b891d4aa25caf7505cec3
+  data.tar.gz: 64b6bda4259cc3fddd7e5635333b0b7c4a6680c4ff2da31bba735603ea50b010
 SHA512:
-  metadata.gz: 3a50542c82cfbb3cab24e6e18f2aaec1862b0a76dfef762b416710a67b17ec9171be6620b48bd59c50ef8d02f29bd195e860579e96fcbb8f2667cd151d277a56
-  data.tar.gz: 42ce91cccba7bbed944a6454e076f7f8db08d107aa96da5e9a69e862faa0c465e27e4441007b9964e0ce953ce5294bbd3120d55e6b91aea516a2597ab9375e9d
+  metadata.gz: e1953d9159c2cb1f1ec55436d925738777ea0b78afedcb32864280f7772d1d9e12a48f65c69385fd6bcdab166668049b3889f1e17e3b316c696705bcb98650dd
+  data.tar.gz: a3559c399a385949526137eed03b73c38baae424bed3b597c89f287222e6e18fd2c5665b2cbba6b20dad71bf680f0840ee2753a68befd3af40fdb522e45e4088

data/CHANGELOG.md

@@ -1,3 +1,27 @@
+## 0.4.6 (2020-07-02)
+
+- Added support for `update_column` and `update_columns`
+
+## 0.4.5 (2020-06-26)
+
+- Improved error message for non-string values
+- Fixed error with migrating Action Text
+- Fixed error with migrating serialized attributes
+
+## 0.4.4 (2020-06-23)
+
+- Added support for `pluck`
+
+## 0.4.3 (2020-05-26)
+
+- Improved error message for bad key length
+- Fixed missing attribute error
+
+## 0.4.2 (2020-05-11)
+
+- Added experimental support for migrating Active Storage files
+- Fixed `metadata` support for Active Storage
+
 ## 0.4.1 (2020-05-08)
 
 - Added support for Action Text

data/README.md

@@ -151,7 +151,9 @@ Be sure to include the `inspect` at the end or it won’t be encoded properly in
 
 #### Migrating Existing Data
 
-Lockbox makes it easy to encrypt an existing column. Add a new column for the ciphertext, then add to your model:
+Lockbox makes it easy to encrypt an existing column without downtime.
+
+Add a new column for the ciphertext, then add to your model:
 
 ```ruby
 class User < ApplicationRecord
@@ -274,6 +276,34 @@ def license
 end
 ```
 
+#### Migrating Existing Files [experimental]
+
+**Note:** This feature is experimental. Please try it in a non-production environment and [share](https://github.com/ankane/lockbox/issues/44) how it goes.
+
+Lockbox makes it easy to encrypt existing files without downtime.
+
+Add to your model:
+
+```ruby
+class User < ApplicationRecord
+  encrypts_attached :license, migrating: true
+end
+```
+
+Migrate existing files:
+
+```ruby
+Lockbox.migrate(User)
+```
+
+Then update the model to the desired state:
+
+```ruby
+class User < ApplicationRecord
+  encrypts_attached :license
+end
+```
+
 ## CarrierWave
 
 Add to your uploader:
@@ -313,6 +343,51 @@ def license
 end
 ```
 
+#### Migrating Existing Files
+
+Encrypt existing files without downtime. Create a new encrypted uploader:
+
+```ruby
+class LicenseV2Uploader < CarrierWave::Uploader::Base
+  encrypt key: Lockbox.attribute_key(table: "users", attribute: "license")
+end
+```
+
+Add a new column for the uploader, then add to your model:
+
+```ruby
+class User < ApplicationRecord
+  mount_uploader :license_v2, LicenseV2Uploader
+
+  before_save :migrate_license, if: :license_changed?
+
+  def migrate_license
+    self.license_v2 = license
+  end
+end
+```
+
+Migrate existing files:
+
+```ruby
+User.find_each do |user|
+  if user.license? && !user.license_v2?
+    user.migrate_license
+    user.save!
+  end
+end
+```
+
+Then update the model to the desired state:
+
+```ruby
+class User < ApplicationRecord
+  mount_uploader :license, LicenseV2Uploader, mount_on: :license_v2
+end
+```
+
+Finally, delete the unencrypted files and drop the column for the original uploader. You can also remove the `key` option from the uploader.
+
 ## Shrine
 
 Generate a key
@@ -448,7 +523,7 @@ Use `master_key` instead of `key` if passing the master key.
 To rotate existing files, use:
 
 ```ruby
-User.find_each do |user|
+User.with_attached_license.find_each do |user|
   user.license.rotate_encryption!
 end
 ```
@@ -572,7 +647,7 @@ Heroku [comes with libsodium](https://devcenter.heroku.com/articles/stack-packag
 
 ##### Ubuntu
 
-For Ubuntu 18.04, use:
+For Ubuntu 20.04 and 18.04, use:
 
 ```sh
 sudo apt-get install libsodium23
@@ -890,3 +965,5 @@ cd lockbox
 bundle install
 bundle exec rake test
 ```
+
+For security issues, send an email to the address on [this page](https://github.com/ankane).

data/SECURITY.md

@@ -0,0 +1,3 @@
+# Security Policy
+
+For security issues, send an email to the address on [this page](https://github.com/ankane).

data/lib/lockbox.rb

@@ -5,6 +5,7 @@ require "securerandom"
 
 # modules
 require "lockbox/box"
+require "lockbox/calculations"
 require "lockbox/encryptor"
 require "lockbox/key_generator"
 require "lockbox/io"
@@ -25,6 +26,7 @@ if defined?(ActiveSupport)
   ActiveSupport.on_load(:active_record) do
     extend Lockbox::Model
     extend Lockbox::Model::Attached
+    ActiveRecord::Calculations.prepend Lockbox::Calculations
   end
 
   ActiveSupport.on_load(:mongoid) do
@@ -95,7 +97,6 @@ module Lockbox
   end
 
   def self.encrypts_action_text_body(**options)
-    # runs every reload
     ActiveSupport.on_load(:action_text_rich_text) do
       ActionText::RichText.encrypts :body, **options
     end

data/lib/lockbox/active_storage_extensions.rb

@@ -16,14 +16,6 @@ module Lockbox
       def encrypt_attachable(attachable)
         Utils.encrypt_attachable(record, name, attachable)
       end
-
-      def rebuild_attachable(attachment)
-        {
-          io: StringIO.new(attachment.download),
-          filename: attachment.filename,
-          content_type: attachment.content_type
-        }
-      end
     end
 
     module AttachedOne
@@ -37,7 +29,7 @@ module Lockbox
       def rotate_encryption!
         raise "Not encrypted" unless encrypted?
 
-        attach(rebuild_attachable(self)) if attached?
+        attach(Utils.rebuild_attachable(self)) if attached?
 
         true
       end
@@ -65,7 +57,7 @@ module Lockbox
 
         attachables =
           previous_attachments.map do |attachment|
-            rebuild_attachable(attachment)
+            Utils.rebuild_attachable(attachment)
           end
 
         ActiveStorage::Attachment.transaction do
@@ -88,13 +80,15 @@ module Lockbox
     end
 
     module Attachment
-      extend ActiveSupport::Concern
-
       def download
         result = super
 
         options = Utils.encrypted_options(record, name)
-        if options
+        # only trust the metadata when migrating
+        # as earlier versions of Lockbox won't have it
+        # and it's not a good practice to trust modifiable data
+        encrypted = options && (!options[:migrating] || blob.metadata["encrypted"])
+        if encrypted
           result = Utils.decrypt_result(record, name, options, result)
         end
 
@@ -105,7 +99,11 @@ module Lockbox
         def open(**options)
           blob.open(**options) do |file|
             options = Utils.encrypted_options(record, name)
-            if options
+            # only trust the metadata when migrating
+            # as earlier versions of Lockbox won't have it
+            # and it's not a good practice to trust modifiable data
+            encrypted = options && (!options[:migrating] || blob.metadata["encrypted"])
+            if encrypted
               result = Utils.decrypt_result(record, name, options, file.read)
               file.rewind
               # truncate may not be available on all platforms
@@ -120,16 +118,6 @@ module Lockbox
           end
         end
       end
-
-      def mark_analyzed
-        if Utils.encrypted_options(record, name)
-          blob.update!(metadata: blob.metadata.merge(analyzed: true))
-        end
-      end
-
-      included do
-        after_save :mark_analyzed
-      end
     end
 
     module Blob

data/lib/lockbox/calculations.rb

@@ -0,0 +1,36 @@
+module Lockbox
+  module Calculations
+    def pluck(*column_names)
+      return super unless model.respond_to?(:lockbox_attributes)
+
+      lockbox_columns = column_names.map.with_index { |c, i| [model.lockbox_attributes[c.to_sym], i] }.select(&:first)
+      return super unless lockbox_columns.any?
+
+      # replace column with ciphertext column
+      lockbox_columns.each do |la, i|
+        column_names[i] = la[:encrypted_attribute]
+      end
+
+      # pluck
+      result = super(*column_names)
+
+      # decrypt result
+      # handle pluck to single columns and multiple
+      #
+      # we can't pass context to decrypt method
+      # so this won't work if any options are a symbol or proc
+      if column_names.size == 1
+        la = lockbox_columns.first.first
+        result.map! { |v| model.send("decrypt_#{la[:encrypted_attribute]}", v) }
+      else
+        lockbox_columns.each do |la, i|
+          result.each do |v|
+            v[i] = model.send("decrypt_#{la[:encrypted_attribute]}", v[i])
+          end
+        end
+      end
+
+      result
+    end
+  end
+end

data/lib/lockbox/carrier_wave_extensions.rb

@@ -5,13 +5,13 @@ module Lockbox
         before :cache, :encrypt
 
         def encrypt(file)
-          @file = CarrierWave::SanitizedFile.new(with_notification("encrypt_file") { lockbox.encrypt_io(file) })
+          @file = CarrierWave::SanitizedFile.new(lockbox_notify("encrypt_file") { lockbox.encrypt_io(file) })
         end
 
         # TODO safe to memoize?
         def read
           r = super
-          with_notification("decrypt_file") { lockbox.decrypt(r) } if r
+          lockbox_notify("decrypt_file") { lockbox.decrypt(r) } if r
         end
 
         def size
@@ -58,7 +58,7 @@ module Lockbox
           end
         end
 
-        def with_notification(type)
+        def lockbox_notify(type)
           if defined?(ActiveSupport::Notifications)
             name = lockbox_name
 

data/lib/lockbox/encryptor.rb

@@ -13,7 +13,7 @@ module Lockbox
     end
 
     def encrypt(message, **options)
-      message = check_string(message, "message")
+      message = check_string(message)
       ciphertext = @boxes.first.encrypt(message, **options)
       ciphertext = Base64.strict_encode64(ciphertext) if @encode
       ciphertext
@@ -21,7 +21,7 @@ module Lockbox
 
     def decrypt(ciphertext, **options)
       ciphertext = Base64.decode64(ciphertext) if @encode
-      ciphertext = check_string(ciphertext, "ciphertext")
+      ciphertext = check_string(ciphertext)
 
       # ensure binary
       if ciphertext.encoding != Encoding::BINARY
@@ -66,9 +66,10 @@ module Lockbox
 
     private
 
-    def check_string(str, name)
+    def check_string(str)
       str = str.read if str.respond_to?(:read)
-      raise TypeError, "can't convert #{name} to string" unless str.respond_to?(:to_str)
+      # Ruby uses "no implicit conversion of Object into String"
+      raise TypeError, "can't convert #{str.class.name} to String" unless str.respond_to?(:to_str)
       str.to_str
     end
 

data/lib/lockbox/key_generator.rb

@@ -11,7 +11,7 @@ module Lockbox
       raise ArgumentError, "Missing attribute for key generation" if attribute.to_s.empty?
 
       c = "\xB4"*32
-      hkdf(Lockbox::Utils.decode_key(@master_key), salt: table.to_s, info: "#{c}#{attribute}", length: 32, hash: "sha384")
+      hkdf(Lockbox::Utils.decode_key(@master_key, name: "Master key"), salt: table.to_s, info: "#{c}#{attribute}", length: 32, hash: "sha384")
     end
 
     private

data/lib/lockbox/migrator.rb

@@ -24,26 +24,49 @@ module Lockbox
 
     # TODO add attributes option
     def migrate(restart:)
-      fields = model.lockbox_attributes.select { |k, v| v[:migrating] }
+      fields = model.respond_to?(:lockbox_attributes) ? model.lockbox_attributes.select { |k, v| v[:migrating] } : {}
 
       # need blind indexes for building relation
       blind_indexes = model.respond_to?(:blind_indexes) ? model.blind_indexes.select { |k, v| v[:migrating] } : {}
 
-      perform(fields: fields, blind_indexes: blind_indexes, restart: restart)
+      attachments = model.respond_to?(:lockbox_attachments) ? model.lockbox_attachments.select { |k, v| v[:migrating] } : {}
+
+      perform(fields: fields, blind_indexes: blind_indexes, restart: restart) if fields.any? || blind_indexes.any?
+      perform_attachments(attachments: attachments, restart: restart) if attachments.any?
     end
 
     private
 
-    def perform(fields:, blind_indexes: [], restart: true, rotate: false)
-      relation = @relation
+    def perform_attachments(attachments:, restart:)
+      relation = base_relation
 
-      # unscope if passed a model
-      unless ar_relation?(relation) || mongoid_relation?(relation)
-        relation = relation.unscoped
+      # eager load attachments
+      attachments.each_key do |k|
+        relation = relation.send("with_attached_#{k}")
       end
 
-      # convert from possible class to ActiveRecord::Relation or Mongoid::Criteria
-      relation = relation.all
+      each_batch(relation) do |records|
+        records.each do |record|
+          attachments.each_key do |k|
+            attachment = record.send(k)
+            if attachment.attached?
+              if attachment.is_a?(ActiveStorage::Attached::One)
+                unless attachment.metadata["encrypted"]
+                  attachment.rotate_encryption!
+                end
+              else
+                unless attachment.all? { |a| a.metadata["encrypted"] }
+                  attachment.rotate_encryption!
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+
+    def perform(fields:, blind_indexes: [], restart: true, rotate: false)
+      relation = base_relation
 
       unless restart
         attributes = fields.map { |_, v| v[:encrypted_attribute] }
@@ -138,6 +161,18 @@ module Lockbox
       end
     end
 
+    def base_relation
+      relation = @relation
+
+      # unscope if passed a model
+      unless ar_relation?(relation) || mongoid_relation?(relation)
+        relation = relation.unscoped
+      end
+
+      # convert from possible class to ActiveRecord::Relation or Mongoid::Criteria
+      relation.all
+    end
+
     def ar_relation?(relation)
       defined?(ActiveRecord::Relation) && relation.is_a?(ActiveRecord::Relation)
     end

data/lib/lockbox/model.rb

@@ -87,7 +87,10 @@ module Lockbox
                 # essentially a no-op if already loaded
                 # an exception is thrown if decryption fails
                 self.class.lockbox_attributes.each do |_, lockbox_attribute|
-                  send(lockbox_attribute[:attribute])
+                  # it is possible that the encrypted attribute is not loaded, eg.
+                  # if the record was fetched partially (`User.select(:id).first`).
+                  # accessing a not loaded attribute raises an `ActiveModel::MissingAttributeError`.
+                  send(lockbox_attribute[:attribute]) if has_attribute?(lockbox_attribute[:encrypted_attribute])
                 end
                 super
               end
@@ -104,6 +107,49 @@ module Lockbox
                   end
                 end
               end
+
+              def update_columns(attributes)
+                return super unless attributes.is_a?(Hash)
+
+                # transform keys like Active Record
+                attributes = attributes.transform_keys do |key|
+                  n = key.to_s
+                  self.class.attribute_aliases[n] || n
+                end
+
+                lockbox_attributes = self.class.lockbox_attributes.slice(*attributes.keys.map(&:to_sym))
+                return super unless lockbox_attributes.any?
+
+                attributes_to_set = {}
+
+                lockbox_attributes.each do |key, lockbox_attribute|
+                  attribute = key.to_s
+                  # check read only
+                  verify_readonly_attribute(attribute)
+
+                  message = attributes[attribute]
+                  attributes.delete(attribute) unless lockbox_attribute[:migrating]
+                  encrypted_attribute = lockbox_attribute[:encrypted_attribute]
+                  ciphertext = self.class.send("generate_#{encrypted_attribute}", message, context: self)
+                  attributes[encrypted_attribute] = ciphertext
+                  attributes_to_set[attribute] = message
+                  attributes_to_set[lockbox_attribute[:attribute]] = message if lockbox_attribute[:migrating]
+                end
+
+                result = super(attributes)
+
+                # same logic as Active Record
+                # (although this happens before saving)
+                attributes_to_set.each do |k, v|
+                  if respond_to?(:write_attribute_without_type_cast, true)
+                    write_attribute_without_type_cast(k, v)
+                  else
+                    raw_write_attribute(k, v)
+                  end
+                end
+
+                result
+              end
             else
               def reload
                 self.class.lockbox_attributes.each do |_, v|
@@ -143,6 +189,10 @@ module Lockbox
               # however, we can try to use the original type if its already defined
               if attributes_to_define_after_schema_loads.key?(original_name.to_s)
                 attribute name, attributes_to_define_after_schema_loads[original_name.to_s].first
+              elsif options[:migrating]
+                # we use the original attribute for serialization in the encrypt and decrypt methods
+                # so we can use a generic value here
+                attribute name, ActiveRecord::Type::Value.new
               else
                 attribute name, :string
               end
@@ -267,7 +317,7 @@ module Lockbox
                 # cache
                 # decrypt method does type casting
                 if respond_to?(:write_attribute_without_type_cast, true)
-                  write_attribute_without_type_cast(name, message) if !@attributes.frozen?
+                  write_attribute_without_type_cast(name.to_s, message) if !@attributes.frozen?
                 else
                   raw_write_attribute(name, message) if !@attributes.frozen?
                 end
@@ -316,7 +366,8 @@ module Lockbox
                 # do nothing
                 # encrypt will convert to binary
               else
-                type = (try(:attribute_types) || {})[name.to_s]
+                # use original name for serialized attributes
+                type = (try(:attribute_types) || {})[original_name.to_s]
                 message = type.serialize(message) if type
               end
             end
@@ -358,7 +409,8 @@ module Lockbox
                 # do nothing
                 # decrypt returns binary string
               else
-                type = (try(:attribute_types) || {})[name.to_s]
+                # use original name for serialized attributes
+                type = (try(:attribute_types) || {})[original_name.to_s]
                 message = type.deserialize(message) if type
                 message.force_encoding(Encoding::UTF_8) if !type || type.is_a?(ActiveModel::Type::String)
               end

data/lib/lockbox/railtie.rb

@@ -5,18 +5,27 @@ module Lockbox
 
       if defined?(ActiveStorage)
         require "lockbox/active_storage_extensions"
+
         ActiveStorage::Attached.prepend(Lockbox::ActiveStorageExtensions::Attached)
         if ActiveStorage::VERSION::MAJOR >= 6
           ActiveStorage::Attached::Changes::CreateOne.prepend(Lockbox::ActiveStorageExtensions::CreateOne)
         end
         ActiveStorage::Attached::One.prepend(Lockbox::ActiveStorageExtensions::AttachedOne)
         ActiveStorage::Attached::Many.prepend(Lockbox::ActiveStorageExtensions::AttachedMany)
-      end
 
-      app.config.to_prepare do
-        if defined?(ActiveStorage)
-          ActiveStorage::Attachment.include(Lockbox::ActiveStorageExtensions::Attachment)
-          ActiveStorage::Blob.prepend(Lockbox::ActiveStorageExtensions::Blob)
+        # use load hooks when possible
+        if ActiveStorage::VERSION::MAJOR >= 6
+          ActiveSupport.on_load(:active_storage_attachment) do
+            include Lockbox::ActiveStorageExtensions::Attachment
+          end
+          ActiveSupport.on_load(:active_storage_blob) do
+            prepend Lockbox::ActiveStorageExtensions::Blob
+          end
+        else
+          app.config.to_prepare do
+            ActiveStorage::Attachment.include(Lockbox::ActiveStorageExtensions::Attachment)
+            ActiveStorage::Blob.prepend(Lockbox::ActiveStorageExtensions::Blob)
+          end
         end
       end
     end

data/lib/lockbox/utils.rb

@@ -4,9 +4,13 @@ module Lockbox
       options = options.except(:attribute, :encrypted_attribute, :migrating, :attached, :type)
       options[:encode] = false unless options.key?(:encode)
       options.each do |k, v|
-        if v.is_a?(Proc)
-          options[k] = context.instance_exec(&v) if v.respond_to?(:call)
+        if v.respond_to?(:call)
+          # context not present for pluck
+          # still possible to use if not dependent on context
+          options[k] = context ? context.instance_exec(&v) : v.call
         elsif v.is_a?(Symbol)
+          # context not present for pluck
+          raise Error, "Not available since :#{k} depends on record" unless context
           options[k] = context.send(v)
         end
       end
@@ -31,13 +35,13 @@ module Lockbox
       record.class.respond_to?(:lockbox_attachments) ? record.class.lockbox_attachments[name.to_sym] : nil
     end
 
-    def self.decode_key(key, size: 32)
+    def self.decode_key(key, size: 32, name: "Key")
       if key.encoding != Encoding::BINARY && key =~ /\A[0-9a-f]{#{size * 2}}\z/i
         key = [key].pack("H*")
       end
 
-      raise Lockbox::Error, "Key must use binary encoding" if key.encoding != Encoding::BINARY
-      raise Lockbox::Error, "Key must be 32 bytes" if key.bytesize != size
+      raise Lockbox::Error, "#{name} must be 32 bytes (64 hex digits)" if key.bytesize != size
+      raise Lockbox::Error, "#{name} must use binary encoding" if key.encoding != Encoding::BINARY
 
       key
     end
@@ -63,14 +67,17 @@ module Lockbox
           }
         when Hash
           io = attachable[:io]
-          attachable = {
-            io: box.encrypt_io(io),
-            filename: attachable[:filename],
-            content_type: attachable[:content_type]
-          }
+          attachable = attachable.dup
+          attachable[:io] = box.encrypt_io(io)
         else
-          raise NotImplementedError, "Not supported"
+          # TODO raise ArgumentError
+          raise NotImplementedError, "Could not find or build blob: expected attachable, got #{attachable.inspect}"
         end
+
+        # don't analyze encrypted data
+        metadata = {"analyzed" => true}
+        metadata["encrypted"] = true if options[:migrating]
+        attachable[:metadata] = (attachable[:metadata] || {}).merge(metadata)
       end
 
       # set content type based on unencrypted data
@@ -85,5 +92,13 @@ module Lockbox
         Utils.build_box(record, options, record.class.table_name, name).decrypt(result)
       end
     end
+
+    def self.rebuild_attachable(attachment)
+      {
+        io: StringIO.new(attachment.download),
+        filename: attachment.filename,
+        content_type: attachment.content_type
+      }
+    end
   end
 end

data/lib/lockbox/version.rb

@@ -1,3 +1,3 @@
 module Lockbox
-  VERSION = "0.4.1"
+  VERSION = "0.4.6"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: lockbox
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.4.6
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-08 00:00:00.000000000 Z
+date: 2020-07-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -173,6 +173,7 @@ files:
 - CHANGELOG.md
 - LICENSE.txt
 - README.md
+- SECURITY.md
 - lib/generators/lockbox/audits_generator.rb
 - lib/generators/lockbox/templates/migration.rb.tt
 - lib/generators/lockbox/templates/model.rb.tt
@@ -180,6 +181,7 @@ files:
 - lib/lockbox/active_storage_extensions.rb
 - lib/lockbox/aes_gcm.rb
 - lib/lockbox/box.rb
+- lib/lockbox/calculations.rb
 - lib/lockbox/carrier_wave_extensions.rb
 - lib/lockbox/encryptor.rb
 - lib/lockbox/io.rb