online_migrations 0.34.0 → 0.36.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b7f1dd28a4540742f77c2a33d7336f4f6f2b12a4f267a1cc556975f7b7122ba4
-  data.tar.gz: db448dca841dc39418cd92e5d86108ee0b59b69b72eeda391b0a81a30aa7ce71
+  metadata.gz: 6740adaa6d20273e8d148e8e0788a6e27c6c6701187fe60e006ff3ba8e2129d2
+  data.tar.gz: 445a3ef72888dd358cc5d917ca31de7499df3368797c9c73324704e0e2d8307a
 SHA512:
-  metadata.gz: b423580e44eefc2f1ff71b2ebdcd15e4ff449f869231aaebe38c4598dc364d817eb3b484a82b7a0cc493727f94d956610ced75ac409092227c2e5cc727935eee
-  data.tar.gz: 3287e23ec07a09a1026790ab4747427eafbc70f675fd2cbf7382f302a86f748c3fde45f5a7546b3e66848cd1e12eb86f073ff3e5461471bb35ba79f5b0a160ec
+  metadata.gz: 338cf3a26b64efb827b5691addc67987048e0522f467d89c5ff4a18494f6b0e3cbeb93513e419a01b04be643d9f7e7a36c1aaf1b0a9d7f5c8158ecff1eb315e7
+  data.tar.gz: 4a6b9282a87124e037b3f101a5856d4a8756e86f9fb5b8bf3a4e42e7e2790e0edbd78b7263b8e9c653bedc25700d8a3fb8ade671e2f99e1e74dbfd49780c20eb

data/CHANGELOG.md

@@ -1,5 +1,20 @@
 ## master (unreleased)
 
+## 0.36.0 (2026-07-20)
+
+- Serialize to JSON `cursor` value for background data migrations
+
+    Note: Run `bin/rails generate online_migrations:upgrade` if using background migrations.
+
+- Do not consider running background migrations as stuck
+- Do not automatically retry stuck background migrations
+
+## 0.35.0 (2026-07-01)
+
+- Drop support for Ruby < 3.3
+- Defer `tick_total` calculation to the background data migration starting time instead of enqueueing time
+- Make "replacing an index" check independent of the live database state
+
 ## 0.34.0 (2026-05-29)
 
 - Drop support for Rails < 7.2

data/README.md

@@ -16,7 +16,7 @@ See [comparison to `strong_migrations`](#comparison-to-strong_migrations)
 
 ## Requirements
 
-- Ruby 3.1+
+- Ruby 3.3+
 - Rails 7.2+
 - PostgreSQL 12+
 
@@ -342,6 +342,8 @@ during writes works automatically). For most column type changes, this does not
 
    ```ruby
    class CleanupChangeFilesSizeType < ActiveRecord::Migration[8.0]
+     disable_ddl_transaction!
+
      def up
        cleanup_column_type_change :files, :size
      end

data/lib/generators/online_migrations/templates/background_data_migrations_convert_cursor.rb.tt

@@ -0,0 +1,9 @@
+class BackgroundDataMigrationsConvertCursor < <%= migration_parent %>
+  def change
+    up_only do
+      OnlineMigrations::DataMigrations::Migration.find_each do |migration|
+        OnlineMigrations::DataMigrations::Migration.where(id: migration.id).update_all(cursor: migration.cursor)
+      end
+    end
+  end
+end

data/lib/generators/online_migrations/upgrade_generator.rb

@@ -48,7 +48,7 @@ module OnlineMigrations
         end
 
         iteration_pause_column = connection.columns(:background_data_migrations).find { |c| c.name == "iteration_pause" }
-        if iteration_pause_column && iteration_pause_column.default
+        if iteration_pause_column&.default
           migrations << "background_data_migrations_remove_iteration_pause_default"
         end
 
@@ -57,6 +57,8 @@ module OnlineMigrations
           migrations << "background_migrations_change_status_default"
         end
 
+        migrations << "background_data_migrations_convert_cursor"
+
         migrations
       end
 

data/lib/online_migrations/background_data_migrations/migration.rb

@@ -52,6 +52,8 @@ module OnlineMigrations
 
       enum :status, STATUSES.index_with(&:to_s)
 
+      serialize :cursor, coder: JSON
+
       validates :migration_name, presence: true
       validates :arguments, uniqueness: { scope: [:migration_name, :shard] }
 
@@ -66,6 +68,13 @@ module OnlineMigrations
         migration_name.sub(/^(::)?#{namespace}::/, "")
       end
 
+      # TODO: delete in some future version
+      def cursor
+        self[:cursor]
+      rescue JSON::ParserError
+        cursor_before_type_cast
+      end
+
       def migration_name=(class_name)
         class_name = class_name.name if class_name.is_a?(Class)
         write_attribute(:migration_name, self.class.normalize_migration_name(class_name))
@@ -111,19 +120,25 @@ module OnlineMigrations
       end
 
       # Returns whether a migration is stuck, which is defined as having a status of
-      # running, cancelling or pausing, and not having been updated in the last 5 minutes.
+      # cancelling or pausing, and not having been updated in the last 5 minutes.
       #
       # @return [Boolean] whether the migration is stuck.
       #
       def stuck?
         stuck_timeout = OnlineMigrations.config.background_data_migrations.stuck_timeout
-        (running? || cancelling? || pausing?) && updated_at <= stuck_timeout.ago
+        (cancelling? || pausing?) && updated_at <= stuck_timeout.ago
       end
 
       # @private
       def start
         if enqueued?
-          update!(status: :running, started_at: Time.current)
+          # Defer `tick_total` calculation to the migration starting time
+          # instead of the enqueueing time to avoid failed deploys.
+          self.tick_total ||= safely_calculate_tick_total
+          self.status = :running
+          self.started_at = Time.current
+          save!
+
           data_migration.after_start
           true
         else
@@ -309,11 +324,20 @@ module OnlineMigrations
         def set_defaults
           config = ::OnlineMigrations.config.background_data_migrations
           self.max_attempts ||= config.max_attempts
-          self.tick_total ||= on_shard_if_present do
+          self.iteration_pause ||= config.iteration_pause
+        end
+
+        def safely_calculate_tick_total
+          on_shard_if_present do
             data_migration.count
           end
+        rescue ActiveRecord::QueryCanceled => e
+          OnlineMigrations.config.background_data_migrations.error_handler.call(e, self)
 
-          self.iteration_pause ||= config.iteration_pause
+          # `tick_total` is not required and is used only for progress tracking.
+          # Probably the `count` method was implemented in a non-efficient way.
+          # Better to not track progress than have a failing migration.
+          nil
         end
 
         def instrument_status_change

data/lib/online_migrations/background_data_migrations/migration_job.rb

@@ -72,6 +72,7 @@ module OnlineMigrations
         # Job was manually cancelled.
         @migration.cancel if cancelled?
 
+        @migration.update!(jid: nil)
         @migration.complete
       end
 

data/lib/online_migrations/background_data_migrations/migration_status_validator.rb

@@ -13,14 +13,12 @@ module OnlineMigrations
         # enqueued -> failed occurs when the migration job fails to be enqueued, or
         #   if the migration is deleted before is starts running.
         "enqueued" => ["running", "paused", "cancelled", "failed"],
-        # running -> enqueued occurs when the migration is stuck and rescheduled by the scheduler.
         # running -> succeeded occurs when the migration completes successfully.
         # running -> pausing occurs when a user pauses the migration as it's performing.
         # running -> cancelling occurs when a user cancels the migration as it's performing.
         # running -> errored occurs when the migration raised an error during the last run.
         # running -> failed occurs when the migration raises an error when running and retry attempts exceeded.
         "running" => [
-          "enqueued",
           "succeeded",
           "pausing",
           "cancelling",

data/lib/online_migrations/background_data_migrations/scheduler.rb

@@ -36,11 +36,10 @@ module OnlineMigrations
         migrations_to_enqueue = []
 
         with_lock do
-          stuck_migrations, active_migrations = relation.running.partition(&:stuck?)
-          runnable_migrations = migrations_with_existing_classes(relation.pending) + stuck_migrations
+          runnable_migrations = migrations_with_existing_classes(relation.pending)
 
           # Ensure no more than 'concurrency' migrations are running at the same time.
-          remaining_to_enqueue = concurrency - active_migrations.count
+          remaining_to_enqueue = concurrency - relation.running.count
           if remaining_to_enqueue > 0
             runnable_migrations.take(remaining_to_enqueue).each do |migration|
               migration.update!(status: :enqueued)

data/lib/online_migrations/background_schema_migrations/migration.rb

@@ -23,6 +23,7 @@ module OnlineMigrations
       MAX_IDENTIFIER_LENGTH = 63
 
       self.table_name = :background_schema_migrations
+      self.ignored_columns += ["parent_id", "composite"]
 
       scope :queue_order, -> { order(created_at: :asc) }
       scope :active, -> { where(status: [:pending, :running, :errored]) }
@@ -80,15 +81,10 @@ module OnlineMigrations
       def progress
       end
 
-      # Whether the migration is considered stuck (is running for some configured time).
+      # Whether the migration is considered stuck.
       #
       def stuck?
-        if index_addition?
-          running? && !index_build_in_progress?
-        else
-          stuck_timeout = (statement_timeout || 1.day) + 10.minutes
-          running? && updated_at <= stuck_timeout.seconds.ago
-        end
+        index_addition? && running? && !index_build_in_progress?
       end
 
       # Mark this migration as ready to be processed again.

data/lib/online_migrations/background_schema_migrations/migration_runner.rb

@@ -44,9 +44,8 @@ module OnlineMigrations
         # Background schema migrations could take a while to run. It is possible, that the process
         # never reaches this (or the rescue below) line of code. E.g., when it is force quitted
         # (SIGKILL etc.) and so the migration will end up in the "running" state and the query is
-        # still executing (or already finished) in the database. This migration can either be safely
-        # manually retried or will be picked up in the future by scheduler when it decides that
-        # this migration is stuck.
+        # still executing (or already finished) in the database. This migration can be safely
+        # manually retried.
 
         migration.update!(status: :succeeded, finished_at: Time.current)
 

data/lib/online_migrations/background_schema_migrations/scheduler.rb

@@ -35,8 +35,8 @@ module OnlineMigrations
 
       private
         def find_migration(**options)
-          stuck_migrations, active_migrations = Migration.running.partition(&:stuck?)
-          runnable_migrations = (Migration.pending + Migration.errored + stuck_migrations).sort_by(&:created_at)
+          active_migrations = Migration.running.to_a
+          runnable_migrations = (Migration.pending + Migration.errored).sort_by(&:created_at)
 
           if options.key?(:shard)
             runnable_migrations = runnable_migrations.select { |migration| migration.shard.to_s == options[:shard].to_s }

data/lib/online_migrations/change_column_type_helpers.rb

@@ -212,7 +212,14 @@ module OnlineMigrations
       conversions = column_names.map do |column_name|
         tmp_column = __change_type_column(column_name)
 
-        old_value = Arel::Table.new(table_name)[column_name]
+        old_value = begin
+          # Delete after supporting only ActiveRecord >= 8.2
+          Arel::Table.new(table_name)[column_name]
+        rescue ArgumentError
+          # https://github.com/rails/rails/commit/b1650993b02497ae7d0d8b984d40bc036e62c681
+          Arel::Table.new(name: table_name)[column_name]
+        end
+
         if (type_cast_function = type_cast_functions.with_indifferent_access[column_name])
           old_value =
             case type_cast_function
@@ -393,10 +400,8 @@ module OnlineMigrations
       def __options_from_column(column, options)
         result = {}
         options.each do |option|
-          if column.respond_to?(option)
-            value = column.public_send(option)
-            result[option] = value if !value.nil?
-          end
+          value = column.public_send(option)
+          result[option] = value if !value.nil?
         end
         result
       end

data/lib/online_migrations/command_checker.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "erb"
-require "set"
 
 module OnlineMigrations
   # @private
@@ -46,7 +45,7 @@ module OnlineMigrations
 
       true
     end
-    ruby2_keywords(:check) if respond_to?(:ruby2_keywords, true)
+    ruby2_keywords(:check)
 
     def version_safe?
       version && version <= OnlineMigrations.config.start_after
@@ -574,13 +573,21 @@ module OnlineMigrations
             Array(index.columns).map(&:to_s) == Array(options[:column]).map(&:to_s)
         end
 
-        if index_def
-          existing_options = [:name, :columns, :unique, :where, :type, :using, :opclasses].filter_map do |option|
-            [option, index_def.public_send(option)] if index_def.respond_to?(option)
-          end.to_h
+        # IndexDefinition expects "columns", but "remove_index" API uses "column".
+        options[:columns] = options[:column]
 
-          @removed_indexes << IndexDefinition.new(table: table_name, **existing_options)
-        end
+        # Fall back to the declared options when the index isn't present in the database,
+        # so an index remove → add is still recorded and caught.
+        existing_options =
+          [:name, :columns, :unique, :where, :type, :using, :opclasses].index_with do |option|
+            if index_def
+              index_def.public_send(option)
+            else
+              options[option]
+            end
+          end.compact
+
+        @removed_indexes << IndexDefinition.new(table: table_name, **existing_options)
       end
 
       def add_foreign_key(from_table, to_table, **options)
@@ -844,8 +851,8 @@ module OnlineMigrations
 
       def index_include_column?(index, column)
         index.columns.include?(column) ||
-          (index.include && index.include.include?(column)) ||
-          (index.where && index.where.include?(column))
+          index.include&.include?(column) ||
+          index.where&.include?(column)
       end
 
       def run_custom_checks(method, args)

data/lib/online_migrations/command_recorder.rb

@@ -48,7 +48,7 @@ module OnlineMigrations
           record(:"#{method}", args, &block)  #   record(:create_table, args, &block)
         end                                   # end
       RUBY
-      ruby2_keywords(method) if respond_to?(:ruby2_keywords, true)
+      ruby2_keywords(method)
     end
 
     private

data/lib/online_migrations/index_definition.rb

@@ -3,10 +3,11 @@
 module OnlineMigrations
   # @private
   class IndexDefinition
-    attr_reader :table, :columns, :unique, :opclasses, :where, :type, :using
+    attr_reader :table, :name, :columns, :unique, :opclasses, :where, :type, :using
 
     def initialize(**options)
-      @table = options[:table]
+      @table = options[:table]&.to_s
+      @name = options[:name]&.to_s
       @columns = Array(options[:columns]).map(&:to_s)
       @unique = options[:unique]
       @opclasses = options[:opclass] || {}
@@ -20,7 +21,7 @@ module OnlineMigrations
       # For ActiveRecord::ConnectionAdapters::IndexDefinition is for expression indexes,
       # `columns` is a string
       table == other.table &&
-        columns.intersect?(Array(other.columns))
+        ((name && name == other.name) || columns.intersect?(Array(other.columns)))
     end
 
     # @param other [OnlineMigrations::IndexDefinition, ActiveRecord::ConnectionAdapters::IndexDefinition]
@@ -28,7 +29,7 @@ module OnlineMigrations
       return false if type != other.type
       return false if using != other.using
       return false if where != other.where
-      return false if other.respond_to?(:opclasses) && opclasses != other.opclasses
+      return false if opclasses != other.opclasses
 
       if unique && !other.unique
         false

data/lib/online_migrations/migration.rb

@@ -29,7 +29,7 @@ module OnlineMigrations
         end
       end
     end
-    ruby2_keywords(:method_missing) if respond_to?(:ruby2_keywords, true)
+    ruby2_keywords(:method_missing)
 
     # @private
     def revert(*args)

data/lib/online_migrations/schema_statements.rb

@@ -111,7 +111,7 @@ module OnlineMigrations
 
         relation.update_all(updates)
 
-        progress.call(relation) if progress
+        progress.call(relation) if progress.present?
 
         sleep(pause_ms * 0.001) if pause_ms > 0
       end
@@ -635,7 +635,7 @@ module OnlineMigrations
     # @see https://api.rubyonrails.org/v8.1/classes/ActiveRecord/ConnectionAdapters/SchemaStatements.html#method-i-add_column
     #
     def add_column(table_name, column_name, type, **options)
-      if column_exists?(table_name, column_name, type, **options)
+      if column_exists?(table_name, column_name)
         Utils.say("Column was not added because it already exists (this may be due to an aborted migration " \
                   "or similar) table_name: #{table_name}, column_name: #{column_name}")
       else

data/lib/online_migrations/utils.rb

@@ -145,7 +145,7 @@ module OnlineMigrations
 
       def run_background_migrations_inline?
         run_inline = OnlineMigrations.config.run_background_migrations_inline
-        run_inline && run_inline.call
+        run_inline.call if run_inline.present?
       end
     end
   end

data/lib/online_migrations/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module OnlineMigrations
-  VERSION = "0.34.0"
+  VERSION = "0.36.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: online_migrations
 version: !ruby/object:Gem::Version
-  version: 0.34.0
+  version: 0.36.0
 platform: ruby
 authors:
 - fatkodima
@@ -41,6 +41,7 @@ files:
 - lib/generators/online_migrations/templates/add_sharding_to_online_migrations.rb.tt
 - lib/generators/online_migrations/templates/add_timestamps_to_background_migrations.rb.tt
 - lib/generators/online_migrations/templates/background_data_migrations_add_iteration_pause.rb.tt
+- lib/generators/online_migrations/templates/background_data_migrations_convert_cursor.rb.tt
 - lib/generators/online_migrations/templates/background_data_migrations_remove_iteration_pause_default.rb.tt
 - lib/generators/online_migrations/templates/background_migrations_change_status_default.rb.tt
 - lib/generators/online_migrations/templates/background_schema_migrations_change_unique_index.rb.tt
@@ -108,14 +109,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '3.1'
+      version: '3.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.10
+rubygems_version: 4.0.14
 specification_version: 4
 summary: Catch unsafe PostgreSQL migrations in development and run them easier in
   production