mass_encryption 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: cc8df4e76e1e8014ef21dbaa9602c2912d7ea52b548e0a16e9dda04e32b27ae8
+  data.tar.gz: e81be1a7a60e9d9866aa8a2d3dda9395b45ebde59f2edac3b5f0f0a336377412
+SHA512:
+  metadata.gz: 92133d3921f5e5ad0021f6eb9088a90765be980cdb078dff480a09a36af6c01d267d5890fa392c0fab92abd4a904b6199f74d9f5558c005322e3c9a521cb4403
+  data.tar.gz: 6964dcdd7e9df1fdc5eb991407dc92947fa17ac7a3208166d7e5695c1d98b3adf4f4f9a5fedbd7d37288b60c5e7f8df67b60e97467a6751b528051f964094692

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2021 Jorge Manrubia
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,81 @@
+![example workflow](https://github.com/basecamp/mass_encryption/actions/workflows/build.yml/badge.svg)
+
+# MassEncryption
+
+MassEncryption lets you encrypt large sets of data using [Active Record encryption](https://edgeguides.rubyonrails.org/active_record_encryption.html). 
+
+Its main use case is adding encryption to existing applications where you have a large amount of existing data to encrypt.
+
+It relies on [Active Job](https://guides.rubyonrails.org/active_job_basics.html) to create encryption jobs that take care of encrypting data in batches.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'mass_encryption'
+```
+
+## Usage
+
+MassEncryption offers two modes of operation:
+
+- Encrypt data in tracks (recommended)
+- Encrypt data in parallel jobs
+
+### Encrypt data in tracks (recommended)
+
+When encrypting in tracks, you create a limited number of jobs that will encrypt a batch of records. Each job represents a track. When the job encrypts its batch, it enqueues the next batch in the track. 
+
+![](docs/images/encryption-in-tracks.png)
+
+This mode of encryption lets you keep the number of jobs you enqueue under control. This has two advantages:
+
+- It avoids having to enqueue all the jobs ahead of time. For example, you don't normally want to enqueue millions of jobs up front to encrypt billions of rows.
+- It lets you limit concurrency to avoid capacity issues.
+
+You can launch the encryption in this mode with:
+
+```shell
+rake mass_encryption:encrypt_all_in_tracks
+```
+
+By default it will encrypt all the models with encrypted attributes using a batch size of 1000 records per job and one track, so only one job will encrypt data at any given moment.
+
+For example:
+
+```shell
+# Encrypt al the posts starting with id 10 using 6 encryption jobs
+rake mass_encryption:encrypt_all_in_tracks EXCEPT="Post" FROM_ID=10 TRACKS=6
+```
+
+### Encrypt data in parallel jobs
+
+In this mode, it will simply loop through all the batches of records and enqueue a job for each.
+
+By default it will encrypt all the models with encrypted attributes using a batch size of 1000 records per job.
+
+```shell
+# Encrypt al the posts starting with id 10 using as many jobs as needed to encrypt them in batches of 500 records 
+rake mass_encryption:encrypt_all_in_parallel_jobs EXCEPT="Post" FROM_ID=10 BATCH_SIZE=500
+```
+
+### Options
+
+You can customize it by passing the following environment variables when invoking the rake task:
+
+* `ONLY`. Comma-separated list of class names to encrypt.
+* `EXCLUDE`. Comma-separated list of class name to exclude.
+* `FROM_ID`. Id to use as an anchor to start encryption. This is handy to resume encryption operations that got interrupted. Ids lower than it won't be encrypted. By default it will be the id of the first model record.
+* `BATCH_SIZE`. The amount of records each job will encrypt. By default it's 1000.
+* `TRACKS`: The number of tracks to use (only available when encrypting in tracks). By default it's 1.
+
+## How it works
+
+* MassEncryption internally uses [`upsert_all`](https://edgeapi.rubyonrails.org/classes/ActiveRecord/Persistence/ClassMethods.html#method-i-upsert_all) to perform fast updates in bulk.
+
+* If there was some error when trying to update the records, MassEncryption jobs will try to encrypt the records in the batch one by one. They will collect all the individual errors and raise a single `MassEncryption::MassEncryption::BatchEncryptionError` error aggregating them all. This way, one record failing to encrypt won't prevent other records in teh batch from being encrypted. 
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,18 @@
+require "bundler/setup"
+
+APP_RAKEFILE = File.expand_path("test/dummy/Rakefile", __dir__)
+load "rails/tasks/engine.rake"
+
+load "rails/tasks/statistics.rake"
+
+require "bundler/gem_tasks"
+
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = false
+end
+
+task default: :test

data/app/jobs/mass_encryption/application_job.rb

@@ -0,0 +1,5 @@
+module MassEncryption
+  class ApplicationJob < ActiveJob::Base
+    queue_as :encryption
+  end
+end

data/app/jobs/mass_encryption/batch_encryption_job.rb

@@ -0,0 +1,10 @@
+class MassEncryption::BatchEncryptionJob < MassEncryption::ApplicationJob
+  def perform(batch, auto_enqueue_next: true)
+    if batch.present?
+      batch.encrypt_now
+      self.class.perform_later batch.next if auto_enqueue_next
+    end
+  end
+
+  ActiveSupport.run_load_hooks(:mass_encryption_batch_job, self)
+end

data/config/routes.rb

@@ -0,0 +1,2 @@
+MassEncryption::Engine.routes.draw do
+end

data/lib/mass_encryption/batch.rb

@@ -0,0 +1,100 @@
+class MassEncryption::Batch
+  attr_reader :from_id, :size, :track, :tracks_count
+
+  DEFAULT_BATCH_SIZE = 1000
+
+  delegate :logger, to: MassEncryption
+
+  def initialize(klass:, from_id:, size: DEFAULT_BATCH_SIZE, track: 0, tracks_count: 1)
+    @class_name = klass.name # not storing class as instance variable as it causes stack overflow error with json serialization
+    @from_id = from_id
+    @size = size
+    @track = track
+    @tracks_count = tracks_count
+  end
+
+  def klass
+    @class_name.constantize
+  end
+
+  def encrypt_now
+    if klass.encrypted_attributes.present? && present?
+      validate_encrypting_is_allowed
+      encrypt_records
+    end
+  end
+
+  def validate_encrypting_is_allowed
+    raise ActiveRecord::Encryption::Errors::Configuration, "can't mass encrypt while in protected mode" if ActiveRecord::Encryption.context.frozen_encryption?
+  end
+
+  def encrypt_later(auto_enqueue_next: false)
+    MassEncryption::BatchEncryptionJob.perform_later(self, auto_enqueue_next: auto_enqueue_next)
+  end
+
+  def present?
+    # we deliberately load the association to avoid 2 queries when checking if there are records
+    # before encrypting in +MassEncryption::BatchEncryptionJob+
+    records.present?
+  end
+
+  def next
+    self.class.new(klass: klass, from_id: next_track_records.last.id + 1, size: size, track: track, tracks_count: tracks_count)
+  end
+
+  def records
+    @records ||= klass.where("id >= ?", determine_from_id).order(id: :asc).limit(size)
+  end
+
+  def to_s
+    "<#{klass}> from: #{from_id} size: #{size} (track=#{track}, tracks_count=#{tracks_count}) | #{records.first.id} - #{records.last.id}"
+  end
+
+  private
+    def encrypt_records
+      encrypt_using_upsert
+    rescue StandardError => error
+      logger.error "Upsert failed with #{error.inspect}. Trying to encrypt record by record..."
+      encrypt_record_by_record
+    end
+
+    def encrypt_using_upsert
+      klass.upsert_all records.collect(&:attributes), update_only: klass.encrypted_attributes, record_timestamps: false
+    end
+
+    def encrypt_record_by_record
+      errors_by_record = {}
+
+      records.each do |record|
+        record.encrypt
+      rescue StandardError => error
+        errors_by_record[record] = error
+      end
+
+      raise MassEncryption::BatchEncryptionError.new(errors_by_record) if errors_by_record.present?
+    end
+
+    def determine_from_id
+      if track == 0
+        from_id # save a query to determine the id for the first track
+      else
+        last_track_id.present? ? (last_track_id + 1) : from_id
+      end
+    end
+
+    def last_track_id
+      @last_track_id ||= ids_in_the_same_track.last
+    end
+
+    def offset
+      track * size
+    end
+
+    def ids_in_the_same_track
+      klass.where("id >= ?", from_id).order(id: :asc).limit(offset).ids
+    end
+
+    def next_track_records
+      klass.where("id >= ?", from_id).order(id: :asc).limit(size * tracks_count)
+    end
+end

data/lib/mass_encryption/batch_encryption_error.rb

@@ -0,0 +1,9 @@
+class MassEncryption::BatchEncryptionError < StandardError
+  attr_reader :errors_by_record
+
+  def initialize(errors_by_record)
+    @errors_by_record = errors_by_record
+    message = errors_by_record.collect { |record, error| "[#{record.class}:#{record.id}] #{error.inspect}" }.join(", ")
+    super(message)
+  end
+end

data/lib/mass_encryption/batch_serializer.rb

@@ -0,0 +1,19 @@
+class MassEncryption::BatchSerializer < ActiveJob::Serializers::ObjectSerializer
+  def serialize?(argument)
+    argument.kind_of?(MassEncryption::Batch)
+  end
+
+  def serialize(batch)
+    super(
+      "klass" => batch.klass.name,
+      "from_id" => batch.from_id,
+      "size" => batch.size,
+      "track" => batch.track || 0,
+      "tracks_count" => batch.tracks_count || 1
+    )
+  end
+
+  def deserialize(hash)
+    MassEncryption::Batch.new(klass: hash["klass"].constantize, from_id: hash["from_id"], size: hash["size"], track: hash["track"], tracks_count: hash["tracks_count"])
+  end
+end

data/lib/mass_encryption/encryptor.rb

@@ -0,0 +1,89 @@
+class MassEncryption::Encryptor
+  DEFAULT_BATCH_SIZE = 1000
+
+  delegate :logger, to: MassEncryption
+
+  def initialize(from_id: nil, only: nil, except: nil, batch_size: DEFAULT_BATCH_SIZE, tracks_count: nil, silent: true)
+    only = Array(only || all_encryptable_classes)
+    except = Array(except)
+
+    @from_id = from_id
+    @encryptable_classes = only - except
+    @batch_size = batch_size
+    @silent = silent
+    @tracks_count = tracks_count
+
+    logger.info info_message unless silent
+  end
+
+  def encrypt_all_later
+    encryptable_classes.each { |klass| enqueue_encryption_jobs_for(klass) }
+  end
+
+  private
+    attr_reader :from_id, :encryptable_classes, :batch_size, :silent, :tracks_count
+
+    def info_message
+      message = "Encrypting #{encryptable_classes.count} models"
+      message << if execute_in_sequential_tracks?
+        " with #{tracks_count} head jobs"
+      else
+        " with parallel jobs"
+      end
+      message << "\n\t#{encryptable_classes.collect(&:name).join(", ")}\n\n"
+      message << [ "Batch size: #{batch_size}", ("From id: #{from_id}" if from_id), "Tracks count: #{tracks_count}" ].compact.join(" | ")
+
+      message
+    end
+
+    def enqueue_encryption_jobs_for(klass)
+      if execute_in_sequential_tracks?
+        enqueue_track_encryption_jobs_for(klass)
+      else
+        enqueue_all_encryption_jobs_for(klass)
+      end
+    end
+
+    def execute_in_sequential_tracks?
+      tracks_count.present?
+    end
+
+    def enqueue_all_encryption_jobs_for(klass)
+      all_records_for(klass).select(:id).in_batches(of: batch_size, load: true) do |records|
+        MassEncryption::Batch.new(klass: klass, from_id: records.first.id, size: batch_size).encrypt_later(auto_enqueue_next: false)
+      end
+    end
+
+    def all_records_for(klass)
+      base = klass
+      base = base.where("id >= ?", from_id) if from_id.present?
+      base
+    end
+
+    def enqueue_track_encryption_jobs_for(klass)
+      tracks_count.times.each do |track|
+        if first_record = all_records_for(klass).first
+          MassEncryption::Batch.new(klass: klass, from_id: first_record.id, size: batch_size, track: track, tracks_count: tracks_count)&.encrypt_later(auto_enqueue_next: true)
+        end
+      end
+    end
+
+    def all_encryptable_classes
+      @all_encryptable_classes ||= begin
+        Rails.application.eager_load! unless Rails.application.config.eager_load
+        ActiveRecord::Base.descendants.find_all { |klass| encryptable_class?(klass) }
+      end
+    end
+
+    def encryptable_class?(klass)
+      has_encrypted_attributes?(klass) || has_encrypted_rich_text_attribute?(klass)
+    end
+
+    def has_encrypted_attributes?(klass)
+      klass.encrypted_attributes.present?
+    end
+
+    def has_encrypted_rich_text_attribute?(klass)
+      klass.reflect_on_all_associations(:has_one).find { |relation| relation.klass == ActionText::EncryptedRichText }
+    end
+end

data/lib/mass_encryption/engine.rb

@@ -0,0 +1,9 @@
+module MassEncryption
+  class Engine < ::Rails::Engine
+    isolate_namespace MassEncryption
+
+    initializer "mass_encryption.active_job" do
+      config.active_job.custom_serializers << MassEncryption::BatchSerializer
+    end
+  end
+end

data/lib/mass_encryption/version.rb

@@ -0,0 +1,3 @@
+module MassEncryption
+  VERSION = '0.1.1'
+end

data/lib/mass_encryption.rb

@@ -0,0 +1,10 @@
+require "mass_encryption/version"
+require "mass_encryption/engine"
+
+require "zeitwerk"
+loader = Zeitwerk::Loader.for_gem
+loader.setup
+
+module MassEncryption
+  mattr_accessor :logger, default: ActiveSupport::Logger.new(STDOUT)
+end

data/lib/tasks/mass_encryption_tasks.rake

@@ -0,0 +1,31 @@
+namespace :mass_encryption do
+  task encrypt_all_in_tracks: :environment do
+    from_id = ENV["FROM_ID"]
+    only = MassEncryption::Tasks.classes_from(ENV["ONLY"])
+    except = MassEncryption::Tasks.classes_from(ENV["EXCEPT"])
+    tracks = (ENV["TRACKS"] || 1).to_i
+    batch_size = (ENV["BATCH_SIZE"] || 1000).to_i
+
+    MassEncryption::Encryptor.new(from_id: from_id, only: only, except: except, tracks_count: tracks, silent: false, batch_size: batch_size).encrypt_all_later
+  end
+
+  task encrypt_all_in_parallel_jobs: :environment do
+    from_id = ENV["FROM_ID"]
+    only = MassEncryption::Tasks.classes_from(ENV["ONLY"])
+    except = MassEncryption::Tasks.classes_from(ENV["EXCEPT"])
+    batch_size = (ENV["BATCH_SIZE"] || 1000).to_i
+
+    MassEncryption::Encryptor.new(from_id: from_id, only: only, except: except, silent: false, batch_size: batch_size).encrypt_all_later
+  end
+end
+
+module MassEncryption::Tasks
+  extend self
+
+  def classes_from(string)
+    if string.present?
+      class_strings = string.split(/[\s,]/).filter(&:present?)
+      class_strings.collect(&:constantize)
+    end
+  end
+end

metadata

@@ -0,0 +1,157 @@
+--- !ruby/object:Gem::Specification
+name: mass_encryption
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Jorge Manrubia
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2021-12-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 7.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 7.0.0
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop-performance
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description:
+email:
+- jorge@hey.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- MIT-LICENSE
+- README.md
+- Rakefile
+- app/jobs/mass_encryption/application_job.rb
+- app/jobs/mass_encryption/batch_encryption_job.rb
+- config/routes.rb
+- lib/mass_encryption.rb
+- lib/mass_encryption/batch.rb
+- lib/mass_encryption/batch_encryption_error.rb
+- lib/mass_encryption/batch_serializer.rb
+- lib/mass_encryption/encryptor.rb
+- lib/mass_encryption/engine.rb
+- lib/mass_encryption/version.rb
+- lib/tasks/mass_encryption_tasks.rake
+homepage: http://github.com/basecamp/mass_encryption
+licenses:
+- MIT
+metadata:
+  homepage_uri: http://github.com/basecamp/mass_encryption
+  allowed_push_host: https://rubygems.org
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.32
+signing_key:
+specification_version: 4
+summary: Encrypt data in mass with Active Record Encryption
+test_files: []