online_migrations 0.26.0 → 0.27.0

data/lib/generators/online_migrations/templates/{background_data_migration.rb.tt → data_migration.rb.tt}

@@ -2,17 +2,16 @@
 
 module <%= migrations_module %>
 <% module_namespacing do -%>
-  class <%= class_name %> < OnlineMigrations::BackgroundMigration
-    def relation
-      # return ActiveRecord::Relation to be iterated over
+  class <%= class_name %> < OnlineMigrations::DataMigration
+    def collection
+      # Collection to iterate over.
+      # Must be ActiveRecord::Relation, ActiveRecord::Batches::BatchEnumerator, or Array.
     end
 
-    def process_batch(relation)
-      # 'relation' is an ActiveRecord::Relation instance containing a batch to process.
-      #
-      # The work to be done in a single iteration of the background migration.
-      # This should be idempotent, as the same batch may be processed more
-      # than once if the background migration is interrupted and resumed.
+    def process(element)
+      # The work to be done in a single iteration of the migration.
+      # This should be idempotent, as the same element may be processed more
+      # than once if the migration is interrupted and resumed.
     end
 
     # Optional.

data/lib/generators/online_migrations/templates/initializer.rb.tt

@@ -96,40 +96,34 @@ OnlineMigrations.configure do |config|
   config.backtrace_cleaner = Rails.backtrace_cleaner
 
   # ==> Background data migrations configuration
-  # The path where generated background migrations will be placed.
-  config.background_migrations.migrations_path = "lib"
-
-  # The module in which background migrations will be placed.
-  config.background_migrations.migrations_module = "OnlineMigrations::BackgroundMigrations"
-
-  # The number of rows to process in a single background migration run.
-  config.background_migrations.batch_size = 1_000
-
-  # The smaller batches size that the batches will be divided into.
-  config.background_migrations.sub_batch_size = 100
-
-  # The pause interval between each background migration job's execution (in seconds).
-  config.background_migrations.batch_pause = 0.seconds
+  #
+  # The path where generated data migrations will be placed.
+  config.background_data_migrations.migrations_path = "lib"
 
-  # The number of milliseconds to sleep between each sub_batch execution.
-  config.background_migrations.sub_batch_pause_ms = 100
+  # The module in which data migrations will be placed.
+  config.background_data_migrations.migrations_module = "OnlineMigrations::DataMigrations"
 
-  # Maximum number of batch run attempts.
-  # When attempts are exhausted, the individual batch is marked as failed.
-  config.background_migrations.batch_max_attempts = 5
+  # Maximum number of run attempts.
+  # When attempts are exhausted, the data migration is marked as failed.
+  config.background_data_migrations.max_attempts = 5
 
-  # The number of seconds that must pass before the running job is considered stuck.
-  config.background_migrations.stuck_jobs_timeout = 1.hour
+  # The number of seconds that must pass before the cancelling or pausing data migration is considered stuck.
+  config.background_data_migrations.stuck_timeout = 5.minutes
 
-  # The callback to perform when an error occurs in the migration job.
-  # config.background_migrations.error_handler = ->(error, errored_job) do
+  # The callback to perform when an error occurs during the data migration.
+  # config.background_data_migrations.error_handler = ->(error, errored_migration) do
   #   Bugsnag.notify(error) do |notification|
-  #     notification.add_metadata(:background_migration, { name: errored_job.migration_name })
+  #     notification.add_metadata(:background_migration, { name: errored_migration.migration_name })
   #   end
   # end
 
+  # The name of the sidekiq job to be used to perform data migrations.
+  config.background_data_migrations.job = "OnlineMigrations::BackgroundDataMigrations::MigrationJob"
+
   # ==> Background schema migrations configuration
-  # When attempts are exhausted, the failing migration stops to be retried.
+  #
+  # Maximum number of run attempts.
+  # When attempts are exhausted, the background schema migration is marked as failed.
   config.background_schema_migrations.max_attempts = 5
 
   # Statement timeout value used when running background schema migration.

data/lib/generators/online_migrations/templates/install_migration.rb.tt

@@ -1,63 +1,34 @@
 class InstallOnlineMigrations < <%= migration_parent %>
   def change
-    create_table :background_migrations do |t|
-      t.bigint :parent_id
+    create_table :background_data_migrations do |t|
       t.string :migration_name, null: false
       t.jsonb :arguments, default: [], null: false
-      t.string :batch_column_name, null: false
-      t.bigint :min_value, null: false
-      t.bigint :max_value, null: false
-      t.bigint :rows_count
-      t.integer :batch_size, null: false
-      t.integer :sub_batch_size, null: false
-      t.integer :batch_pause, null: false
-      t.integer :sub_batch_pause_ms, null: false
-      t.integer :batch_max_attempts, null: false
       t.string :status, default: "enqueued", null: false
       t.string :shard
-      t.boolean :composite, default: false, null: false
+      t.string :cursor
+      t.string :jid
       t.datetime :started_at
       t.datetime :finished_at
-      t.timestamps
-
-      t.foreign_key :background_migrations, column: :parent_id, on_delete: :cascade
-
-      t.index [:migration_name, :arguments, :shard],
-        unique: true, name: :index_background_migrations_on_unique_configuration
-    end
-
-    create_table :background_migration_jobs do |t|
-      t.bigint :migration_id, null: false
-      t.bigint :min_value, null: false
-      t.bigint :max_value, null: false
-      t.integer :batch_size, null: false
-      t.integer :sub_batch_size, null: false
-      t.integer :pause_ms, null: false
-      t.datetime :started_at
-      t.datetime :finished_at
-      t.string :status, default: "enqueued", null: false
+      t.bigint :tick_total
+      t.bigint :tick_count, default: 0, null: false
+      t.float :time_running, default: 0.0, null: false
       t.integer :max_attempts, null: false
-      t.integer :attempts, default: 0, null: false
       t.string :error_class
       t.string :error_message
       t.string :backtrace, array: true
+      t.string :connection_class_name
       t.timestamps
 
-      t.foreign_key :background_migrations, column: :migration_id, on_delete: :cascade
-
-      t.index [:migration_id, :max_value], name: :index_background_migration_jobs_on_max_value
-      t.index [:migration_id, :status, :updated_at], name: :index_background_migration_jobs_on_updated_at
-      t.index [:migration_id, :finished_at], name: :index_background_migration_jobs_on_finished_at
+      t.index [:migration_name, :arguments, :shard],
+        unique: true, name: :index_background_data_migrations_on_unique_configuration
     end
 
     create_table :background_schema_migrations do |t|
-      t.bigint :parent_id
       t.string :migration_name, null: false
       t.string :table_name, null: false
       t.string :definition, null: false
       t.string :status, default: "enqueued", null: false
       t.string :shard
-      t.boolean :composite, default: false, null: false
       t.integer :statement_timeout
       t.datetime :started_at
       t.datetime :finished_at
@@ -69,8 +40,6 @@ class InstallOnlineMigrations < <%= migration_parent %>
       t.string :connection_class_name
       t.timestamps
 
-      t.foreign_key :background_schema_migrations, column: :parent_id, on_delete: :cascade
-
       t.index [:migration_name, :shard, :connection_class_name], unique: true,
         name: :index_background_schema_migrations_on_unique_configuration
     end

data/lib/generators/online_migrations/upgrade_generator.rb

@@ -11,33 +11,41 @@ module OnlineMigrations
     source_root File.expand_path("templates", __dir__)
 
     def copy_templates
-      migrations_to_be_applied.each do |migration|
+      migrations_to_apply.each do |migration|
         migration_template("#{migration}.rb", File.join(db_migrate_path, "#{migration}.rb"))
       end
     end
 
     private
-      def migrations_to_be_applied
-        connection = BackgroundMigrations::Migration.connection
-        columns = connection.columns(BackgroundMigrations::Migration.table_name).map(&:name)
+      def migrations_to_apply
+        connection = BackgroundDataMigrations::Migration.connection
+        data_table = "background_migrations"
+        schema_table = "background_schema_migrations"
+        columns = connection.columns(data_table).map(&:name)
 
         migrations = []
-        migrations << "add_sharding_to_online_migrations" if !columns.include?("shard")
+        if connection.table_exists?(data_table) && !columns.include?("shard")
+          migrations << "add_sharding_to_online_migrations"
+        end
 
-        if !connection.table_exists?(BackgroundSchemaMigrations::Migration.table_name)
+        if !connection.table_exists?(schema_table)
           migrations << "create_background_schema_migrations"
         end
 
-        indexes = connection.indexes(BackgroundSchemaMigrations::Migration.table_name)
+        indexes = connection.indexes(schema_table)
         unique_index = indexes.find { |i| i.unique && i.columns.sort == ["connection_class_name", "migration_name", "shard"] }
         if !unique_index
           migrations << "background_schema_migrations_change_unique_index"
         end
 
-        if !connection.column_exists?(BackgroundMigrations::Migration.table_name, :started_at)
+        if connection.table_exists?(data_table) && !connection.column_exists?(data_table, :started_at)
           migrations << "add_timestamps_to_background_migrations"
         end
 
+        if connection.table_exists?(data_table)
+          migrations << "change_background_data_migrations"
+        end
+
         migrations
       end
 

data/lib/online_migrations/active_record_batch_enumerator.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module OnlineMigrations
+  # @private
+  module ActiveRecordBatchEnumerator
+    attr_reader :use_ranges
+  end
+end

data/lib/online_migrations/background_data_migrations/backfill_column.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module OnlineMigrations
+  module BackgroundDataMigrations
+    # @private
+    class BackfillColumn < DataMigration
+      attr_reader :table_name, :updates, :model
+
+      def initialize(table_name, updates, model_name = nil)
+        @table_name = table_name
+        @updates = updates
+
+        @model =
+          if model_name
+            Object.const_get(model_name, false)
+          else
+            Utils.define_model(table_name)
+          end
+      end
+
+      def collection
+        column, value = updates.first
+
+        relation =
+          if updates.size == 1 && !value.nil?
+            # If value is nil, the generated SQL is correct (`WHERE column IS NOT NULL`).
+            # Otherwise, the SQL is `WHERE column != value`. This condition ignores column
+            # with NULLs in it, so we need to also manually check for NULLs.
+            arel_column = model.arel_table[column]
+            model.unscoped.where(arel_column.not_eq(value).or(arel_column.eq(nil)))
+          else
+            model.unscoped.where.not(updates)
+          end
+
+        relation.in_batches(of: 100, use_ranges: true)
+      end
+
+      def process(relation)
+        relation.update_all(updates)
+      end
+
+      def count
+        # Exact counts are expensive on large tables, since PostgreSQL
+        # needs to do a full scan. An estimated count should give a pretty decent
+        # approximation of rows count in this case.
+        Utils.estimated_count(model.connection, table_name)
+      end
+    end
+  end
+end

data/lib/online_migrations/background_data_migrations/config.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module OnlineMigrations
+  module BackgroundDataMigrations
+    # Class representing configuration options for data migrations.
+    class Config
+      # The path where generated data migrations will be placed.
+      # @return [String] defaults to "lib"
+      attr_accessor :migrations_path
+
+      # The module in which data migrations will be placed.
+      # @return [String] defaults to "OnlineMigrations::DataMigrations"
+      attr_accessor :migrations_module
+
+      # Maximum number of run attempts.
+      #
+      # When attempts are exhausted, the data migration is marked as failed.
+      # @return [Integer] defaults to 5
+      attr_accessor :max_attempts
+
+      # The number of seconds that must pass before the cancelling or pausing data migration is considered stuck.
+      #
+      # @return [Integer] defaults to 5 minutes
+      #
+      attr_accessor :stuck_timeout
+
+      # The pause interval between each data migration's `process` method execution (in seconds).
+      # @return [Integer] defaults to 0
+      #
+      attr_accessor :iteration_pause
+
+      # The callback to perform when an error occurs during the data migration.
+      #
+      # @example
+      #   OnlineMigrations.config.background_migrations.error_handler = ->(error, errored_migration) do
+      #     Bugsnag.notify(error) do |notification|
+      #       notification.add_metadata(:background_migration, { name: errored_migration.migration_name })
+      #     end
+      #   end
+      #
+      # @return [Proc]
+      #
+      attr_accessor :error_handler
+
+      # The name of the sidekiq job to be used to perform data migrations.
+      #
+      # @return [String] defaults to "OnlineMigrations::BackgroundDataMigrations::MigrationJob"
+      #
+      attr_accessor :job
+
+      def initialize
+        @migrations_path = "lib"
+        @migrations_module = "OnlineMigrations::DataMigrations"
+        @max_attempts = 5
+        @stuck_timeout = 5.minutes
+        @iteration_pause = 0.seconds
+        @error_handler = ->(error, errored_migration) {}
+        @job = "OnlineMigrations::BackgroundDataMigrations::MigrationJob"
+      end
+    end
+  end
+end

data/lib/online_migrations/{background_migrations → background_data_migrations}/copy_column.rb

@@ -1,9 +1,9 @@
 # frozen_string_literal: true
 
 module OnlineMigrations
-  module BackgroundMigrations
+  module BackgroundDataMigrations
     # @private
-    class CopyColumn < BackgroundMigration
+    class CopyColumn < DataMigration
       attr_reader :table_name, :copy_from, :copy_to, :model_name, :type_cast_functions
 
       def initialize(table_name, copy_from, copy_to, model_name = nil, type_cast_functions = {})
@@ -21,15 +21,21 @@ module OnlineMigrations
         end
 
         @model_name = model_name
+        @model =
+          if model_name.present?
+            Object.const_get(model_name, false)
+          else
+            Utils.define_model(table_name)
+          end
+
         @type_cast_functions = type_cast_functions
       end
 
-      def relation
-        model.unscoped
+      def collection
+        @model.unscoped.in_batches(of: 100, use_ranges: true)
       end
 
-      def process_batch(relation)
-        arel = relation.arel
+      def process(relation)
         arel_table = relation.arel_table
 
         old_values = copy_from.map do |from_column|
@@ -46,35 +52,16 @@ module OnlineMigrations
           old_value
         end
 
-        stmt = Arel::UpdateManager.new
-        stmt.table(arel_table)
-        stmt.wheres = arel.constraints
-
-        updates = copy_to.zip(old_values).map { |to_column, old_value| [arel_table[to_column], old_value] }
-        stmt.set(updates)
-
-        connection.update(stmt)
+        updates = copy_to.zip(old_values).to_h { |to_column, old_value| [to_column, old_value] }
+        relation.update_all(updates)
       end
 
       def count
         # Exact counts are expensive on large tables, since PostgreSQL
         # needs to do a full scan. An estimated count should give a pretty decent
         # approximation of rows count in this case.
-        Utils.estimated_count(connection, table_name)
+        Utils.estimated_count(@model.connection, table_name)
       end
-
-      private
-        def model
-          @model ||= if model_name.present?
-                       Object.const_get(model_name, false)
-                     else
-                       Utils.define_model(table_name)
-                     end
-        end
-
-        def connection
-          model.connection
-        end
     end
   end
 end

data/lib/online_migrations/{background_migrations → background_data_migrations}/delete_associated_records.rb

@@ -1,9 +1,9 @@
 # frozen_string_literal: true
 
 module OnlineMigrations
-  module BackgroundMigrations
+  module BackgroundDataMigrations
     # @private
-    class DeleteAssociatedRecords < BackgroundMigration
+    class DeleteAssociatedRecords < DataMigration
       attr_reader :record, :association
 
       def initialize(model_name, record_id, association, _options = {})
@@ -12,17 +12,21 @@ module OnlineMigrations
         @association = association
       end
 
-      def relation
+      def collection
         if !@record.respond_to?(association)
           raise ArgumentError, "'#{@record.class.name}' has no association called '#{association}'"
         end
 
-        record.public_send(association)
+        record.public_send(association).in_batches(of: 100)
       end
 
-      def process_batch(relation)
+      def process(relation)
         relation.delete_all
       end
+
+      def count
+        record.public_send(association).count
+      end
     end
   end
 end

data/lib/online_migrations/{background_migrations → background_data_migrations}/delete_orphaned_records.rb

@@ -1,9 +1,9 @@
 # frozen_string_literal: true
 
 module OnlineMigrations
-  module BackgroundMigrations
+  module BackgroundDataMigrations
     # @private
-    class DeleteOrphanedRecords < BackgroundMigration
+    class DeleteOrphanedRecords < DataMigration
       attr_reader :model, :associations
 
       def initialize(model_name, associations, _options = {})
@@ -11,16 +11,12 @@ module OnlineMigrations
         @associations = associations.map(&:to_sym)
       end
 
-      def relation
+      def collection
         model.unscoped.where.missing(*associations)
       end
 
-      def process_batch(relation)
-        relation.delete_all
-      end
-
-      def count
-        Utils.estimated_count(model.connection, model.table_name)
+      def process(record)
+        record.destroy
       end
     end
   end