online_migrations 0.4.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eb5f39f15a73531072b0fe0298a119bd4e9be9ad9c1070fae8fbf21e532f6e20
-  data.tar.gz: a9a3ff44991e42db36f4680cd2bdfc9acd0cf8c8221ff5be999ed53b2266d7ee
+  metadata.gz: c2515b6dd51e983fbcbdb448d33273e77d5b7f30ef53bc42dacb5127ac390fc8
+  data.tar.gz: b099a129bfea91da13d1dbde5c08d18fb09ae195dd83b56495c5513a881f609f
 SHA512:
-  metadata.gz: 0fbb6f4d7455db8c41915520a25b8ae9c94cacb1041cd7111b7d4af3373000c06f647fab0e5bcf43c9fd04af7c13c82b591f7bf7f4fd82048edc07c47860100c
-  data.tar.gz: 2a207fe4cd1ca20b3cf02866445f814e0687249928530aac33062aaf823496b9eb503c9d3bf746236a48e346ea1599518d62004ce2420199407cb381edb87d1d
+  metadata.gz: '0220675939d3a08c0ecb16ba3e4a212bb40106d34478b7008c2458d2eebbead2c897873634589a6ae55b0665fdf0a02fb691432c639198c7c3a59e378c10d023'
+  data.tar.gz: b3d9a76edb0f4e220790a7de2a2ba5f7930c9106599dd488f585f4003ae61166341feccd66cbc63965b5a181217fe5639eb17e836be48542b0070591c74b8ab6

data/BACKGROUND_MIGRATIONS.md

@@ -116,9 +116,12 @@ enqueue_background_migration("MyMigrationWithArgs", arg1, arg2, ...)
 
 ## Predefined background migrations
 
-* `BackfillColumn` - backfills column(s) with scalar values
-* `CopyColumn` - copies data from one column(s) to other(s)
-* `ResetCounters` - resets one or more counter caches to their correct value
+* `BackfillColumn` - backfills column(s) with scalar values (enqueue using `backfill_column_in_background`)
+* `CopyColumn` - copies data from one column(s) to other(s) (enqueue using `copy_column_in_background`)
+* `DeleteAssociatedRecords` - deletes records associated with a parent object (enqueue using `delete_associated_records_in_background`)
+* `DeleteOrphanedRecords` - deletes records with one or more missing relations (enqueue using `delete_orphaned_records_in_background`)
+* `PerformActionOnRelation` - performs specific action on a relation or indvidual records (enqueue using `perform_action_on_relation_in_background`)
+* `ResetCounters` - resets one or more counter caches to their correct value (enqueue using `reset_counters_in_background`)
 
 ## Testing
 
@@ -262,8 +265,6 @@ The error handler should be a lambda that accepts 2 arguments:
 
 * `error`: The exception that was raised.
 * `errored_job`: An `OnlineMigrations::BackgroundMigrations::MigrationJob` object that represents a failed batch.
-* `errored_element`: The `OnlineMigrations::BackgroundMigrations::MigrationJob` object representing a batch,
-  that was being processed when the Background Migration raised an exception.
 
 ### Customizing the background migrations module
 

data/CHANGELOG.md

@@ -1,5 +1,74 @@
 ## master (unreleased)
 
+## 0.5.0 (2022-06-23)
+
+- Added check for index corruption with PostgreSQL 14.0 to 14.3
+
+- No need to separately remove indexes when removing a column from the small table
+
+- Add ability to perform specific action on a relation or individual records using background migrations
+
+    Example, assuming you have lots and lots of fraud likes:
+
+    ```ruby
+    class DeleteFraudLikes < ActiveRecord::Migration[7.0]
+      def up
+        perform_action_on_relation_in_background("Like", { fraud: true }, :delete_all)
+      end
+    end
+    ```
+
+    Example, assuming you added a new column to the users and want to populate it:
+
+    ```ruby
+    class User < ApplicationRecord
+      def generate_invite_token
+        self.invite_token = # some complex logic
+      end
+    end
+
+    perform_action_on_relation_in_background("User", { invite_token: nil }, :generate_invite_token)
+    ```
+
+    You can use `delete_all`/`destroy_all`/`update_all` for the whole relation or run specific methods on individual records.
+
+- Add ability to delete records associated with a parent object using background migrations
+
+    ```ruby
+      class Link < ActiveRecord::Base
+        has_many :clicks
+      end
+
+      class Click < ActiveRecord::Base
+        belongs_to :link
+      end
+
+      class DeleteSomeLinkClicks < ActiveRecord::Migration[7.0]
+        def up
+          some_link = ...
+          delete_associated_records_in_background("Link", some_link.id, :clicks)
+        end
+      end
+    ```
+
+- Add ability to delete orphaned records using background migrations
+
+    ```ruby
+      class User < ApplicationRecord
+        has_many :posts
+      end
+
+      class Post < ApplicationRecord
+        belongs_to :author, class_name: 'User'
+      end
+
+      class DeleteOrphanedPosts < ActiveRecord::Migration[7.0]
+        def up
+          delete_orphaned_records_in_background("Post", :author)
+        end
+      end
+    ```
+
 ## 0.4.1 (2022-03-21)
 
 - Fix missing options in suggested command for columns removal

data/README.md

@@ -351,7 +351,7 @@ A safer approach can be accomplished in several steps:
   ```ruby
   class CleanupChangeFilesSizeType < ActiveRecord::Migration[7.0]
     def up
-      cleanup_change_column_type_concurrently :files, :size
+      cleanup_column_type_change :files, :size
     end
 
     def down

data/lib/online_migrations/background_migrations/delete_associated_records.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module OnlineMigrations
+  module BackgroundMigrations
+    # @private
+    class DeleteAssociatedRecords < BackgroundMigration
+      attr_reader :record, :association
+
+      def initialize(model_name, record_id, association, _options = {})
+        model = Object.const_get(model_name, false)
+        @record = model.find(record_id)
+        @association = association
+      end
+
+      def relation
+        unless @record.respond_to?(association)
+          raise ArgumentError, "'#{@record.class.name}' has no association called '#{association}'"
+        end
+
+        record.public_send(association)
+      end
+
+      def process_batch(relation)
+        relation.delete_all(:delete_all)
+      end
+    end
+  end
+end

data/lib/online_migrations/background_migrations/delete_orphaned_records.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module OnlineMigrations
+  module BackgroundMigrations
+    # @private
+    class DeleteOrphanedRecords < BackgroundMigration
+      attr_reader :model, :associations
+
+      def initialize(model_name, associations, _options = {})
+        @model = Object.const_get(model_name, false)
+        @associations = associations.map(&:to_sym)
+      end
+
+      def relation
+        # For ActiveRecord 6.1+ we can use `where.missing`
+        # https://github.com/rails/rails/pull/34727
+        associations.inject(model.unscoped) do |relation, association|
+          reflection = model.reflect_on_association(association)
+          unless reflection
+            raise ArgumentError, "'#{model.name}' has no association called '#{association}'"
+          end
+
+          # left_joins was added in ActiveRecord 5.0 - https://github.com/rails/rails/pull/12071
+          relation
+            .left_joins(association)
+            .where(reflection.table_name => { reflection.association_primary_key => nil })
+        end
+      end
+
+      def process_batch(relation)
+        if Utils.ar_version > 5.0
+          relation.delete_all
+        else
+          # Older ActiveRecord generates incorrect query when running delete_all
+          primary_key = model.primary_key
+          model.unscoped.where(primary_key => relation.select(primary_key)).delete_all
+        end
+      end
+
+      def count
+        Utils.estimated_count(model.connection, model.table_name)
+      end
+    end
+  end
+end

data/lib/online_migrations/background_migrations/migration_helpers.rb

@@ -21,7 +21,7 @@ module OnlineMigrations
       # @example Additional background migration options
       #   backfill_column_in_background(:users, :admin, false, batch_size: 10_000)
       #
-      # @note This method is better suited for extra large tables (100s of millions of records).
+      # @note This method is better suited for large tables (10/100s of millions of records).
       #     For smaller tables it is probably better and easier to use more flexible `update_column_in_batches`.
       #
       # @note Consider `backfill_columns_in_background` when backfilling multiple columns
@@ -76,7 +76,7 @@ module OnlineMigrations
       # @example Additional background migration options
       #   backfill_column_for_type_change_in_background(:files, :size, batch_size: 10_000)
       #
-      # @note This method is better suited for extra large tables (100s of millions of records).
+      # @note This method is better suited for large tables (10/100s of millions of records).
       #     For smaller tables it is probably better and easier to use more flexible `backfill_column_for_type_change`.
       #
       def backfill_column_for_type_change_in_background(table_name, column_name, model_name: nil,
@@ -131,7 +131,7 @@ module OnlineMigrations
       # @example
       #   copy_column_in_background(:users, :id, :id_for_type_change)
       #
-      # @note This method is better suited for extra large tables (100s of millions of records).
+      # @note This method is better suited for large tables (10/100s of millions of records).
       #     For smaller tables it is probably better and easier to use more flexible `update_column_in_batches`.
       #
       def copy_column_in_background(table_name, copy_from, copy_to, model_name: nil, type_cast_function: nil, **options)
@@ -190,7 +190,7 @@ module OnlineMigrations
       #
       # @see https://api.rubyonrails.org/classes/ActiveRecord/CounterCache/ClassMethods.html#method-i-reset_counters
       #
-      # @note This method is better suited for extra large tables (100s of millions of records).
+      # @note This method is better suited for large tables (10/100s of millions of records).
       #     For smaller tables it is probably better and easier to use `reset_counters` from the ActiveRecord.
       #
       def reset_counters_in_background(model_name, *counters, touch: nil, **options)
@@ -205,6 +205,115 @@ module OnlineMigrations
         )
       end
 
+      # Deletes records with one or more missing relations using background migrations.
+      # This is useful when some referential integrity in the database is broken and
+      # you want to delete orphaned records.
+      #
+      # @param model_name [String]
+      # @param associations [Array]
+      # @param options [Hash] used to control the behavior of background migration.
+      #     See `#enqueue_background_migration`
+      #
+      # @return [OnlineMigrations::BackgroundMigrations::Migration]
+      #
+      # @example
+      #     delete_orphaned_records_in_background("Post", :author)
+      #
+      # @note This method is better suited for large tables (10/100s of millions of records).
+      #     For smaller tables it is probably better and easier to directly find and delete orpahed records.
+      #
+      def delete_orphaned_records_in_background(model_name, *associations, **options)
+        if Utils.ar_version <= 4.2
+          raise "#{__method__} does not support ActiveRecord <= 4.2 yet"
+        end
+
+        model_name = model_name.name if model_name.is_a?(Class)
+
+        enqueue_background_migration(
+          "DeleteOrphanedRecords",
+          model_name,
+          associations,
+          **options
+        )
+      end
+
+      # Deletes associated records for a specific parent record using background migrations.
+      # This is useful when you are planning to remove a parent object (user, account etc)
+      # and needs to remove lots of its associated objects.
+      #
+      # @param model_name [String]
+      # @param record_id [Integer, String] parent record primary key's value
+      # @param association [String, Symbol] association name for which records will be removed
+      # @param options [Hash] used to control the behavior of background migration.
+      #     See `#enqueue_background_migration`
+      #
+      # @return [OnlineMigrations::BackgroundMigrations::Migration]
+      #
+      # @example
+      #     delete_associated_records_in_background("Link", 1, :clicks)
+      #
+      # @note This method is better suited for large tables (10/100s of millions of records).
+      #     For smaller tables it is probably better and easier to directly delete associated records.
+      #
+      def delete_associated_records_in_background(model_name, record_id, association, **options)
+        model_name = model_name.name if model_name.is_a?(Class)
+
+        enqueue_background_migration(
+          "DeleteAssociatedRecords",
+          model_name,
+          record_id,
+          association,
+          **options
+        )
+      end
+
+      # Performs specific action on a relation or individual records.
+      # This is useful when you want to delete/destroy/update/etc records based on some conditions.
+      #
+      # @param model_name [String]
+      # @param conditions [Array, Hash, String] conditions to filter the relation
+      # @param action [String, Symbol] action to perform on the relation or individual records.
+      #     Relation-wide available actions: `:delete_all`, `:destroy_all`, and `:update_all`.
+      # @param updates [Hash] updates to perform when `action` is set to `:update_all`
+      # @param options [Hash] used to control the behavior of background migration.
+      #     See `#enqueue_background_migration`
+      #
+      # @return [OnlineMigrations::BackgroundMigrations::Migration]
+      #
+      # @example Delete records
+      #     perform_action_on_relation_in_background("User", { banned: true }, :delete_all)
+      #
+      # @example Destroy records
+      #     perform_action_on_relation_in_background("User", { banned: true }, :destroy_all)
+      #
+      # @example Update records
+      #     perform_action_on_relation_in_background("User", { banned: nil }, :update_all, updates: { banned: false })
+      #
+      # @example Perform custom method on individual records
+      #     class User < ApplicationRecord
+      #       def generate_invite_token
+      #         self.invite_token = # some complex logic
+      #       end
+      #     end
+      #
+      #     perform_action_on_relation_in_background("User", { invite_token: nil }, :generate_invite_token)
+      #
+      # @note This method is better suited for large tables (10/100s of millions of records).
+      #     For smaller tables it is probably better and easier to directly delete associated records.
+      #
+      def perform_action_on_relation_in_background(model_name, conditions, action, updates: nil, **options)
+        model_name = model_name.name if model_name.is_a?(Class)
+
+        enqueue_background_migration(
+          "PerformActionOnRelation",
+          model_name,
+          conditions,
+          action,
+          { updates: updates },
+          **options
+        )
+      end
+
       # Creates a background migration for the given job class name.
       #
       # A background migration runs one job at a time, computing the bounds of the next batch

data/lib/online_migrations/background_migrations/perform_action_on_relation.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module OnlineMigrations
+  module BackgroundMigrations
+    # @private
+    class PerformActionOnRelation < BackgroundMigration
+      attr_reader :model, :conditions, :action, :options
+
+      def initialize(model_name, conditions, action, options = {})
+        @model = Object.const_get(model_name, false)
+        @conditions = conditions
+        @action = action.to_sym
+        @options = options.symbolize_keys
+      end
+
+      def relation
+        model.unscoped.where(conditions)
+      end
+
+      def process_batch(relation)
+        case action
+        when :update_all
+          updates = options.fetch(:updates)
+          relation.public_send(action, updates)
+        when :delete_all, :destroy_all
+          relation.public_send(action)
+        else
+          relation.each(&action)
+        end
+      end
+    end
+  end
+end

data/lib/online_migrations/change_column_type_helpers.rb

@@ -173,7 +173,7 @@ module OnlineMigrations
     #   backfill_column_for_type_change(:files, :size, batch_size: 10_000)
     #
     # @note This method should not be run within a transaction
-    # @note For extra large tables (100s of millions of records)
+    # @note For large tables (10/100s of millions of records)
     #   it is recommended to use `backfill_column_for_type_change_in_background`.
     #
     def backfill_column_for_type_change(table_name, column_name, type_cast_function: nil, **options)

data/lib/online_migrations/command_checker.rb

@@ -316,7 +316,7 @@ module OnlineMigrations
               initialize_change_code: command_str(:initialize_column_type_change, table_name, column_name, type, **options),
               backfill_code: command_str(:backfill_column_for_type_change, table_name, column_name, **options),
               finalize_code: command_str(:finalize_column_type_change, table_name, column_name),
-              cleanup_code: command_str(:cleanup_change_column_type_concurrently, table_name, column_name),
+              cleanup_code: command_str(:cleanup_column_type_change, table_name, column_name),
               cleanup_down_code: command_str(:initialize_column_type_change, table_name, column_name, existing_type)
           end
         end
@@ -383,7 +383,8 @@ module OnlineMigrations
             columns: columns.inspect,
             command: command_str(command, *args, options),
             table_name: table_name.inspect,
-            indexes: indexes.map { |i| i.name.to_sym.inspect }
+            indexes: indexes.map { |i| i.name.to_sym.inspect },
+            small_table: small_table?(table_name)
         end
       end
 
@@ -450,6 +451,10 @@ module OnlineMigrations
               command: command_str(:add_index, table_name, column_name, **options.merge(algorithm: :concurrently))
           end
 
+          if options[:algorithm] == :concurrently && index_corruption?
+            raise_error :add_index_corruption
+          end
+
           if @removed_indexes.any?
             index = IndexDefinition.new(table: table_name, columns: column_name, **options)
             existing_indexes = connection.indexes(table_name)
@@ -576,16 +581,17 @@ module OnlineMigrations
       end
 
       def new_or_small_table?(table_name)
-        small_tables = OnlineMigrations.config.small_tables
-
-        new_table?(table_name) ||
-          small_tables.include?(table_name.to_s)
+        new_table?(table_name) || small_table?(table_name)
       end
 
       def new_table?(table_name)
         @new_tables.include?(table_name.to_s)
       end
 
+      def small_table?(table_name)
+        OnlineMigrations.config.small_tables.include?(table_name.to_s)
+      end
+
       def postgresql_version
         version =
           if Utils.developer_env? && (target_version = OnlineMigrations.config.target_version)
@@ -737,6 +743,12 @@ module OnlineMigrations
         [table1.to_s, table2.to_s].sort.join("\0").gsub(/^(.*_)(.+)\0\1(.+)/, '\1\2_\3').tr("\0", "_")
       end
 
+      def index_corruption?
+        postgresql_version >= Gem::Version.new("14.0") &&
+          postgresql_version < Gem::Version.new("14.4") &&
+          !Utils.developer_env?
+      end
+
       def run_custom_checks(method, args)
         OnlineMigrations.config.checks.each do |options, check|
           if !options[:start_after] || version > options[:start_after]

data/lib/online_migrations/command_recorder.rb

@@ -16,7 +16,9 @@ module OnlineMigrations
       :swap_column_names,
       :add_column_with_default,
       :add_not_null_constraint,
+      :remove_not_null_constraint,
       :add_text_limit_constraint,
+      :remove_text_limit_constraint,
       :add_reference_concurrently,
       :change_column_type_in_background,
       :enqueue_background_migration,
@@ -127,11 +129,14 @@ module OnlineMigrations
       end
 
       def invert_remove_text_limit_constraint(args)
-        unless args[2]
+        options = args.extract_options!
+        table_name, column, limit = args
+
+        unless limit
           raise ActiveRecord::IrreversibleMigration, "remove_text_limit_constraint is only reversible if given a limit."
         end
 
-        super
+        [:add_text_limit_constraint, [table_name, column, limit, **options]]
       end
   end
 end

data/lib/online_migrations/error_messages.rb

@@ -233,7 +233,7 @@ class <%= migration_name %> < <%= migration_parent %>
 end",
 
       remove_column:
-"<% if indexes.any? %>
+"<% if !small_table && indexes.any? %>
 Removing a column will automatically remove all of the indexes that involved the removed column.
 But the indexes would be removed non-concurrently, so you need to safely remove the indexes first:
 
@@ -333,6 +333,11 @@ class <%= migration_name %> < <%= migration_parent %>
   end
 end",
 
+      add_index_corruption:
+"Adding an index concurrently can cause silent data corruption in PostgreSQL 14.0 to 14.3.
+Upgrade PostgreSQL before adding new indexes, or wrap this step in a safety_assured { ... }
+block to accept the risk.",
+
       remove_index:
 "Removing an index non-concurrently blocks writes. Instead, use:
 

data/lib/online_migrations/schema_statements.rb

@@ -48,7 +48,7 @@ module OnlineMigrations
     # @note This method should not be run within a transaction
     # @note Consider `update_columns_in_batches` when updating multiple columns
     #   to avoid rewriting the table multiple times.
-    # @note For extra large tables (100s of millions of records)
+    # @note For large tables (10/100s of millions of records)
     #   you may consider using `backfill_column_in_background` or `copy_column_in_background`.
     #
     def update_column_in_batches(table_name, column_name, value, **options, &block)
@@ -361,7 +361,7 @@ module OnlineMigrations
     # These steps ensure a column can be added to a large and commonly used table
     # without locking the entire table for the duration of the table modification.
     #
-    # For extra large tables (100s of millions of records) you may consider implementing
+    # For large tables (10/100s of millions of records) you may consider implementing
     #   the steps from this helper method yourself as a separate migrations, replacing step #3
     #   with the help of background migrations (see `backfill_column_in_background`).
     #

data/lib/online_migrations/verbose_sql_logs.rb

@@ -26,7 +26,7 @@ module OnlineMigrations
 
       private
         def verbose_query_logs
-          if Utils.ar_version > 7.0
+          if Utils.ar_version >= 7.0
             ActiveRecord.verbose_query_logs
           elsif Utils.ar_version >= 5.2
             ActiveRecord::Base.verbose_query_logs
@@ -34,7 +34,7 @@ module OnlineMigrations
         end
 
         def set_verbose_query_logs(value) # rubocop:disable Naming/AccessorMethodName
-          if Utils.ar_version > 7.0
+          if Utils.ar_version >= 7.0
             ActiveRecord.verbose_query_logs = value
           elsif Utils.ar_version >= 5.2
             ActiveRecord::Base.verbose_query_logs = value

data/lib/online_migrations/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module OnlineMigrations
-  VERSION = "0.4.1"
+  VERSION = "0.5.0"
 end

data/lib/online_migrations.rb

@@ -46,6 +46,9 @@ module OnlineMigrations
     autoload :BackgroundMigrationClassValidator
     autoload :BackfillColumn
     autoload :CopyColumn
+    autoload :DeleteAssociatedRecords
+    autoload :DeleteOrphanedRecords
+    autoload :PerformActionOnRelation
     autoload :ResetCounters
     autoload :MigrationJob
     autoload :Migration

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: online_migrations
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.5.0
 platform: ruby
 authors:
 - fatkodima
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-03-21 00:00:00.000000000 Z
+date: 2022-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -47,6 +47,8 @@ files:
 - lib/online_migrations/background_migrations/background_migration_class_validator.rb
 - lib/online_migrations/background_migrations/config.rb
 - lib/online_migrations/background_migrations/copy_column.rb
+- lib/online_migrations/background_migrations/delete_associated_records.rb
+- lib/online_migrations/background_migrations/delete_orphaned_records.rb
 - lib/online_migrations/background_migrations/migration.rb
 - lib/online_migrations/background_migrations/migration_helpers.rb
 - lib/online_migrations/background_migrations/migration_job.rb
@@ -54,6 +56,7 @@ files:
 - lib/online_migrations/background_migrations/migration_job_status_validator.rb
 - lib/online_migrations/background_migrations/migration_runner.rb
 - lib/online_migrations/background_migrations/migration_status_validator.rb
+- lib/online_migrations/background_migrations/perform_action_on_relation.rb
 - lib/online_migrations/background_migrations/reset_counters.rb
 - lib/online_migrations/background_migrations/scheduler.rb
 - lib/online_migrations/batch_iterator.rb
@@ -98,7 +101,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
 summary: Catch unsafe PostgreSQL migrations in development and run them easier in