online_migrations 0.24.0 → 0.25.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 99170e244f0f008e8d097222c88a96555692aa5afc136087d03341866ed8412c
-  data.tar.gz: 6068a722274c73d8a0ab3c1bed4d1f2673cd99d15ca77c24f7fff5af75e77129
+  metadata.gz: f6b7685527d71319ba50e3a5c29744d9ef28103c1cacfdc650cba12c276782d8
+  data.tar.gz: e10e0d1dfb1c807b05927189f682633a7ab17c53f37974c5df5a2b452d78ab92
 SHA512:
-  metadata.gz: 9d2a127aee978d36676d8bdd85352487f66a14289cca28173eb1126aee65ceeea5dda067171ffb320b7c0cb2be903712ab158af12312452180b8e56da1e7b641
-  data.tar.gz: 305f89162f87a3746d91bfdfacae00b7d0f2a374f27755c846cd4386d65694442b70fc6963a7dec8845c192eca2cc625bb020c7c4aec664320a395b124bb3e71
+  metadata.gz: d7c62d15765a2837cf0dcba5e9a0c63c89ecc73f33fbd5500284184a78027f80a4ad12d980b5ac5de8235550384bd123708a075adab0243a06202154348268a3
+  data.tar.gz: 24449c58e1bec9b3a31e925a27df17c56c7cd923f2cdd33a5d6781f1f253204776901e92fd4024d3340e4eeb533b043a71a1afeaacb046bcc6ea62d5e5444668

data/CHANGELOG.md

@@ -1,5 +1,22 @@
 ## master (unreleased)
 
+## 0.25.0 (2025-02-03)
+
+- Track start/finish time of background data migrations
+
+    Note: Make sure to run `bin/rails generate online_migrations:upgrade` if using background migrations.
+
+- Add new state for errored background migrations
+
+    * **errored** - migration raised an error during last run
+    * **failed** - migration raises an error when running and retry attempts exceeded
+
+- Fix thread safety issue for lock retrier
+
+    Note: Lock retrier changed its API (`LockRetrier#connection` accessor was removed and
+    `LockRetrier#with_lock_retries` now accepts a connection argument). This change might be of interest
+    if you implemented a custom lock retrier class.
+
 ## 0.24.0 (2025-01-20)
 
 - Add ability to run a separate background migrations scheduler per shard

data/docs/background_schema_migrations.md

@@ -134,7 +134,8 @@ Background Schema Migrations can be in various states during its execution:
 
 * **enqueued**: A migration has been enqueued by the user.
 * **running**: A migration is being performed by a migration executor.
-* **failed**: A migration raises an exception when running.
+* **errored**: A migration raised an error during last run.
+* **failed**: A migration raises an error when running and retry attempts exceeded.
 * **succeeded**: A migration finished without error.
 * **cancelled**: A migration was cancelled by the user.
 

data/lib/generators/online_migrations/templates/add_timestamps_to_background_migrations.rb.tt

@@ -0,0 +1,31 @@
+class AddTimestampsToBackgroundMigrations < <%= migration_parent %>
+  def change
+    safety_assured do
+      add_column :background_migrations, :started_at, :datetime
+      add_column :background_migrations, :finished_at, :datetime
+
+      up_only do
+        # Set started_at.
+        execute(<<~SQL)
+          UPDATE background_migrations
+          SET started_at = (
+            SELECT min(started_at)
+            FROM background_migration_jobs
+            WHERE background_migration_jobs.migration_id = background_migrations.id
+          )
+        SQL
+
+        # Set finished_at.
+        execute(<<~SQL)
+          UPDATE background_migrations
+          SET finished_at = (
+            SELECT max(finished_at)
+            FROM background_migration_jobs
+            WHERE background_migration_jobs.migration_id = background_migrations.id
+          )
+          WHERE status IN ('failed', 'succeeded')
+        SQL
+      end
+    end
+  end
+end

data/lib/generators/online_migrations/templates/install_migration.rb.tt

@@ -16,6 +16,8 @@ class InstallOnlineMigrations < <%= migration_parent %>
       t.string :status, default: "enqueued", null: false
       t.string :shard
       t.boolean :composite, default: false, null: false
+      t.datetime :started_at
+      t.datetime :finished_at
       t.timestamps
 
       t.foreign_key :background_migrations, column: :parent_id, on_delete: :cascade

data/lib/generators/online_migrations/templates/migration.rb.tt

@@ -1,10 +1,10 @@
 class Enqueue<%= class_name %> < <%= migration_parent %>
   def up
-    enqueue_background_data_migration("<%= class_name %>", ...args)
+    enqueue_background_data_migration("<%= class_name %>")
   end
 
   def down
     # Make sure to pass the same arguments as in the "up" method, if any.
-    remove_background_data_migration("<%= class_name %>", ...args)
+    remove_background_data_migration("<%= class_name %>")
   end
 end

data/lib/generators/online_migrations/upgrade_generator.rb

@@ -34,6 +34,10 @@ module OnlineMigrations
           migrations << "background_schema_migrations_change_unique_index"
         end
 
+        if !connection.column_exists?(BackgroundMigrations::Migration.table_name, :started_at)
+          migrations << "add_timestamps_to_background_migrations"
+        end
+
         migrations
       end
 

data/lib/online_migrations/background_migrations/migration.rb

@@ -189,15 +189,23 @@ module OnlineMigrations
       #
       def retry
         if composite? && failed?
-          children.failed.each(&:retry)
-          enqueued!
+          transaction do
+            update!(status: :enqueued, finished_at: nil)
+            children.failed.each(&:retry)
+          end
+
           true
         elsif failed?
-          iterator = BatchIterator.new(migration_jobs.failed)
-          iterator.each_batch(of: 100) do |batch|
-            batch.each(&:retry)
+          transaction do
+            parent.update!(status: :enqueued, finished_at: nil) if parent
+            update!(status: :enqueued, started_at: nil, finished_at: nil)
+
+            iterator = BatchIterator.new(migration_jobs.failed)
+            iterator.each_batch(of: 100) do |batch|
+              batch.each(&:retry)
+            end
           end
-          enqueued!
+
           true
         else
           false
@@ -205,20 +213,6 @@ module OnlineMigrations
       end
       alias retry_failed_jobs retry
 
-      # Returns the time this migration started running.
-      def started_at
-        # To be precise, we should get the minimum of `started_at` amongst the children jobs
-        # (for simple migrations) and amongst the children migrations (for composite migrations).
-        # But we do not have an appropriate index on the jobs table and using this will lead to
-        # N+1 queries if used inside some dashboard, for example.
-        created_at
-      end
-
-      # Returns the time this migration finished running.
-      def finished_at
-        updated_at if completed?
-      end
-
       # @private
       def on_shard(&block)
         abstract_class = Utils.find_connection_class(migration_model)
@@ -229,9 +223,9 @@ module OnlineMigrations
 
       # @private
       def reset_failed_jobs_attempts
-        iterator = BatchIterator.new(migration_jobs.failed.attempts_exceeded)
+        iterator = BatchIterator.new(migration_jobs.failed)
         iterator.each_batch(of: 100) do |relation|
-          relation.update_all(attempts: 0)
+          relation.update_all(status: :enqueued, attempts: 0)
         end
       end
 

data/lib/online_migrations/background_migrations/migration_job.rb

@@ -6,6 +6,7 @@ module OnlineMigrations
       STATUSES = [
         :enqueued,
         :running,
+        :errored,
         :failed,
         :succeeded,
         :cancelled,
@@ -13,7 +14,7 @@ module OnlineMigrations
 
       self.table_name = :background_migration_jobs
 
-      scope :active, -> { where(status: [:enqueued, :running]) }
+      scope :active, -> { where(status: [:enqueued, :running, :errored]) }
       scope :completed, -> { where(status: [:failed, :succeeded]) }
       scope :stuck, -> do
         timeout = OnlineMigrations.config.background_migrations.stuck_jobs_timeout
@@ -21,14 +22,11 @@ module OnlineMigrations
       end
 
       scope :retriable, -> do
-        failed_retriable = failed.where("attempts < max_attempts")
-
-        stuck_sql             = connection.unprepared_statement { stuck.to_sql }
-        failed_retriable_sql  = connection.unprepared_statement { failed_retriable.to_sql }
+        stuck_sql = connection.unprepared_statement { stuck.to_sql }
 
         from(Arel.sql(<<~SQL))
           (
-            (#{failed_retriable_sql})
+            (SELECT * FROM background_migration_jobs WHERE status = 'errored')
             UNION
             (#{stuck_sql})
           ) AS #{table_name}
@@ -36,7 +34,6 @@ module OnlineMigrations
       end
 
       scope :except_succeeded, -> { where.not(status: :succeeded) }
-      scope :attempts_exceeded, -> { where("attempts >= max_attempts") }
 
       enum :status, STATUSES.index_with(&:to_s)
 
@@ -60,6 +57,10 @@ module OnlineMigrations
         running? && updated_at <= timeout.seconds.ago
       end
 
+      def attempts_exceeded?
+        attempts >= max_attempts
+      end
+
       # Mark this job as ready to be processed again.
       #
       # This is used when retrying failed jobs.

data/lib/online_migrations/background_migrations/migration_job_runner.rb

@@ -37,8 +37,10 @@ module OnlineMigrations
       rescue Exception => e # rubocop:disable Lint/RescueException
         backtrace_cleaner = ::OnlineMigrations.config.backtrace_cleaner
 
+        status = migration_job.attempts_exceeded? ? :failed : :errored
+
         migration_job.update!(
-          status: :failed,
+          status: status,
           finished_at: Time.current,
           error_class: e.class.name,
           error_message: e.message,

data/lib/online_migrations/background_migrations/migration_job_status_validator.rb

@@ -6,7 +6,8 @@ module OnlineMigrations
     class MigrationJobStatusValidator < ActiveModel::Validator
       VALID_STATUS_TRANSITIONS = {
         "enqueued" => ["running", "cancelled"],
-        "running" => ["succeeded", "failed", "cancelled"],
+        "running" => ["succeeded", "errored", "failed", "cancelled"],
+        "errored" => ["running", "failed", "cancelled"],
         "failed" => ["enqueued", "running", "cancelled"],
       }
 

data/lib/online_migrations/background_migrations/migration_runner.rb

@@ -34,9 +34,9 @@ module OnlineMigrations
           job_runner.run
         elsif !migration.migration_jobs.active.exists?
           if migration.migration_jobs.failed.exists?
-            migration.failed!
+            migration.update!(status: :failed, finished_at: Time.current)
           else
-            migration.succeeded!
+            migration.update!(status: :succeeded, finished_at: Time.current)
           end
 
           ActiveSupport::Notifications.instrument("completed.background_migrations", migration_payload)
@@ -100,8 +100,15 @@ module OnlineMigrations
       private
         def mark_as_running
           Migration.transaction do
-            migration.running!
-            migration.parent.running! if migration.parent && migration.parent.enqueued?
+            migration.update!(status: :running, started_at: Time.current, finished_at: nil)
+
+            if (parent = migration.parent)
+              if parent.started_at
+                parent.update!(status: :running, finished_at: nil)
+              else
+                parent.update!(status: :running, started_at: Time.current, finished_at: nil)
+              end
+            end
           end
         end
 
@@ -133,10 +140,10 @@ module OnlineMigrations
           parent.with_lock do
             children = parent.children.select(:status)
             if children.all?(&:succeeded?)
-              parent.succeeded!
+              parent.update!(status: :succeeded, finished_at: Time.current)
               completed = true
             elsif children.any?(&:failed?)
-              parent.failed!
+              parent.update!(status: :failed, finished_at: Time.current)
               completed = true
             end
           end

data/lib/online_migrations/background_schema_migrations/migration.rb

@@ -11,7 +11,8 @@ module OnlineMigrations
       STATUSES = [
         :enqueued,    # The migration has been enqueued by the user.
         :running,     # The migration is being performed by a migration executor.
-        :failed,      # The migration raises an exception when running.
+        :errored,     # The migration raised an error during last run.
+        :failed,      # The migration raises an error when running and retry attempts exceeded.
         :succeeded,   # The migration finished without error.
         :cancelled,   # The migration was cancelled by the user.
       ]
@@ -23,7 +24,7 @@ module OnlineMigrations
       scope :queue_order, -> { order(created_at: :asc) }
       scope :parents, -> { where(parent_id: nil) }
       scope :runnable, -> { where(composite: false) }
-      scope :active, -> { where(status: [statuses[:enqueued], statuses[:running]]) }
+      scope :active, -> { where(status: [:enqueued, :running, :errored]) }
       scope :except_succeeded, -> { where.not(status: :succeeded) }
 
       scope :stuck, -> do
@@ -33,14 +34,11 @@ module OnlineMigrations
       end
 
       scope :retriable, -> do
-        failed_retriable = runnable.failed.where("attempts < max_attempts")
-
-        stuck_sql             = connection.unprepared_statement { stuck.to_sql }
-        failed_retriable_sql  = connection.unprepared_statement { failed_retriable.to_sql }
+        stuck_sql = connection.unprepared_statement { stuck.to_sql }
 
         from(Arel.sql(<<~SQL))
           (
-            (#{failed_retriable_sql})
+            (SELECT * FROM background_schema_migrations WHERE NOT composite AND status = 'errored')
             UNION
             (#{stuck_sql})
           ) AS #{table_name}
@@ -139,22 +137,27 @@ module OnlineMigrations
       #
       def retry
         if composite? && failed?
-          children.failed.each(&:retry)
-          update!(
-            status: self.class.statuses[:enqueued],
-            finished_at: nil
-          )
+          transaction do
+            update!(status: :enqueued, finished_at: nil)
+            children.failed.each(&:retry)
+          end
+
           true
         elsif failed?
-          update!(
-            status: self.class.statuses[:enqueued],
-            attempts: 0,
-            started_at: nil,
-            finished_at: nil,
-            error_class: nil,
-            error_message: nil,
-            backtrace: nil
-          )
+          transaction do
+            parent.update!(status: :enqueued, finished_at: nil) if parent
+
+            update!(
+              status: :enqueued,
+              attempts: 0,
+              started_at: nil,
+              finished_at: nil,
+              error_class: nil,
+              error_message: nil,
+              backtrace: nil
+            )
+          end
+
           true
         else
           false

data/lib/online_migrations/background_schema_migrations/migration_runner.rb

@@ -13,7 +13,7 @@ module OnlineMigrations
       def run
         return if migration.cancelled? || migration.succeeded?
 
-        mark_as_running if migration.enqueued? || migration.failed?
+        mark_as_running if migration.enqueued? || migration.errored?
 
         if migration.composite?
           migration.children.each do |child_migration|
@@ -83,14 +83,18 @@ module OnlineMigrations
         rescue Exception => e # rubocop:disable Lint/RescueException
           backtrace_cleaner = ::OnlineMigrations.config.backtrace_cleaner
 
+          status = migration.attempts_exceeded? ? :failed : :errored
+
           migration.update!(
-            status: :failed,
+            status: status,
             finished_at: Time.current,
             error_class: e.class.name,
             error_message: e.message,
             backtrace: backtrace_cleaner ? backtrace_cleaner.clean(e.backtrace) : e.backtrace
           )
 
+          complete_parent_if_needed(migration) if migration.parent.present?
+
           ::OnlineMigrations.config.background_schema_migrations.error_handler.call(e, migration)
           raise if Utils.run_background_migrations_inline?
         end

data/lib/online_migrations/background_schema_migrations/migration_status_validator.rb

@@ -8,8 +8,12 @@ module OnlineMigrations
         # enqueued -> running occurs when the migration starts performing.
         "enqueued" => ["running", "cancelled"],
         # running -> succeeded occurs when the migration completes successfully.
-        # running -> failed occurs when the migration raises an exception when running and retry attempts exceeded.
-        "running" => ["succeeded", "failed", "cancelled"],
+        # running -> errored occurs when the migration raised an error during the last run.
+        # running -> failed occurs when the migration raises an error when running and retry attempts exceeded.
+        "running" => ["succeeded", "errored", "failed", "cancelled"],
+        # errored -> running occurs when previously errored migration starts running
+        # errored -> failed occurs when the migration raises an error when running and retry attempts exceeded.
+        "errored" => ["running", "failed", "cancelled"],
         # failed -> enqueued occurs when the failed migration is enqueued to be retried.
         # failed -> running occurs when the failed migration is retried.
         "failed" => ["enqueued", "running", "cancelled"],

data/lib/online_migrations/lock_retrier.rb

@@ -46,10 +46,6 @@ module OnlineMigrations
   #     end
   #
   class LockRetrier
-    # Database connection on which retries are run
-    #
-    attr_accessor :connection
-
     # Returns the number of retrying attempts
     #
     def attempts
@@ -73,14 +69,15 @@ module OnlineMigrations
     # Executes the block with a retry mechanism that alters the `lock_timeout`
     # and sleep time between attempts.
     #
+    # @param connection The connection on which to retry lock timeouts
     # @return [void]
     #
     # @example
-    #   retrier.with_lock_retries do
+    #   retrier.with_lock_retries(connection) do
     #     add_column(:users, :name, :string)
     #   end
     #
-    def with_lock_retries(&block)
+    def with_lock_retries(connection, &block)
       return yield if lock_retries_disabled?
 
       current_attempt = 0
@@ -90,7 +87,7 @@ module OnlineMigrations
 
         current_lock_timeout = lock_timeout(current_attempt)
         if current_lock_timeout
-          with_lock_timeout(current_lock_timeout.in_milliseconds, &block)
+          with_lock_timeout(connection, current_lock_timeout.in_milliseconds, &block)
         else
           yield
         end
@@ -110,7 +107,7 @@ module OnlineMigrations
         Utils.to_bool(ENV["DISABLE_LOCK_RETRIES"])
       end
 
-      def with_lock_timeout(value)
+      def with_lock_timeout(connection, value)
         value = value.ceil.to_i
         prev_value = connection.select_value("SHOW lock_timeout")
         connection.execute("SET lock_timeout TO #{connection.quote("#{value}ms")}")
@@ -234,7 +231,7 @@ module OnlineMigrations
     def delay(*)
     end
 
-    def with_lock_retries
+    def with_lock_retries(_connection)
       yield
     end
   end

data/lib/online_migrations/schema_statements.rb

@@ -892,14 +892,10 @@ module OnlineMigrations
       views = self.views
 
       table_renames = OnlineMigrations.config.table_renames
-      renamed_tables = table_renames.select do |old_name, _|
-        views.include?(old_name)
-      end
+      renamed_tables = table_renames.slice(*views)
 
       column_renames = OnlineMigrations.config.column_renames
-      renamed_columns = column_renames.select do |table_name, _|
-        views.include?(table_name)
-      end
+      renamed_columns = column_renames.slice(*views)
 
       if renamed_tables.key?(table)
         super(renamed_tables[table])
@@ -919,8 +915,7 @@ module OnlineMigrations
       __ensure_not_in_transaction!
 
       retrier = OnlineMigrations.config.lock_retrier
-      retrier.connection = self
-      retrier.with_lock_retries(&block)
+      retrier.with_lock_retries(self, &block)
     end
 
     private

data/lib/online_migrations/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module OnlineMigrations
-  VERSION = "0.24.0"
+  VERSION = "0.25.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: online_migrations
 version: !ruby/object:Gem::Version
-  version: 0.24.0
+  version: 0.25.0
 platform: ruby
 authors:
 - fatkodima
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-01-20 00:00:00.000000000 Z
+date: 2025-02-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -40,6 +40,7 @@ files:
 - lib/generators/online_migrations/background_migration_generator.rb
 - lib/generators/online_migrations/install_generator.rb
 - lib/generators/online_migrations/templates/add_sharding_to_online_migrations.rb.tt
+- lib/generators/online_migrations/templates/add_timestamps_to_background_migrations.rb.tt
 - lib/generators/online_migrations/templates/background_data_migration.rb.tt
 - lib/generators/online_migrations/templates/background_schema_migrations_change_unique_index.rb.tt
 - lib/generators/online_migrations/templates/create_background_schema_migrations.rb.tt