online_migrations 0.12.0 → 0.13.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a5db4da5657a6887113af46baa5c5ec9a6906ba4f8e3d33afc3d81f378f9b5ae
-  data.tar.gz: 0acf2706e6b453055d41f338908323a0672066b6e742724b8a6d588e969ca05f
+  metadata.gz: 8952366397c54505f364a353dde1d68e988ac4773e9b3663de790a75c8b860f3
+  data.tar.gz: 376dc6e760a2b07d55968a11097f9f81067bc72291718938d1102fafb46a73ea
 SHA512:
-  metadata.gz: 4a1b97dceb858d5c6ebd6ddccab4ea0b3ed8d257074f1e7f6562f101b3ae002ac19576053a44fe765624350c40d6e269d753fd24be5b5419474036a308b5cce5
-  data.tar.gz: 9b6a3ece26c9e3e679c5d8d0f827eda7aa10b591242440a18d2a0fb18aa8f16b16935db6654859aab9b6f5008160e117c2eb32e98571261127d1a6ac1e936f2c
+  metadata.gz: 3e40ed5ab49e5d702c7d7cfebdb4264621375ad9affb67e8c3592814c4a832e75929c4ff718ce90709e64497226df3d9503a42babf56cfaad5ce973026f71d82
+  data.tar.gz: cdcbc9c26ac23fb975f21dd58c9826dab59c9571d074da06211dea2ffae72c3c57372722110ecd729ba649f21db551f58e8ca43689214f1235cd20669f033cf6

data/CHANGELOG.md

@@ -1,5 +1,22 @@
 ## master (unreleased)
 
+## 0.13.1 (2024-01-23)
+
+- Fix calculation of batch ranges for background migration created with explicit ranges
+
+## 0.13.0 (2024-01-22)
+
+- Add ability to configure the path where generated background migrations will be placed
+
+    ```ruby
+    # It is placed in lib/ by default.
+    config.background_migrations.migrations_path = "app/lib"
+    ```
+
+- Reduce number of queries needed to calculate batch ranges for background migrations
+- Fix `finalize_column_type_change` to not recreate already existing indexes on the temporary column
+- Remove potentially heavy queries used to get the ranges of a background migration
+
 ## 0.12.0 (2024-01-18)
 
 - Require passing model name for background migration helpers when using multiple databases

data/docs/background_migrations.md

@@ -255,7 +255,7 @@ Specify the throttle condition as a block:
 ```ruby
 # config/initializers/online_migrations.rb
 
-OnlineMigrations.config.background_migrations.throttler = -> { DatabaseStatus.unhealthy? }
+config.background_migrations.throttler = -> { DatabaseStatus.unhealthy? }
 ```
 
 Note that it's up to you to define a throttling condition that makes sense for your app. For example, you can check various PostgreSQL metrics such as replication lag, DB threads, whether DB writes are available, etc.
@@ -269,7 +269,7 @@ If you want to integrate with an exception monitoring service (e.g. Bugsnag), yo
 ```ruby
 # config/initializers/online_migrations.rb
 
-OnlineMigrations.config.background_migrations.error_handler = ->(error, errored_job) do
+config.background_migrations.error_handler = ->(error, errored_job) do
   Bugsnag.notify(error) do |notification|
     notification.add_metadata(:background_migration, { name: errored_job.migration_name })
   end
@@ -281,22 +281,34 @@ The error handler should be a lambda that accepts 2 arguments:
 * `error`: The exception that was raised.
 * `errored_job`: An `OnlineMigrations::BackgroundMigrations::MigrationJob` object that represents a failed batch.
 
+### Customizing the background migrations path
+
+`OnlineMigrations.config.background_migrations.migrations_path` can be configured to define where generated background migrations will be placed.
+
+```ruby
+# config/initializers/online_migrations.rb
+
+config.background_migrations.migrations_path = "app/lib"
+```
+
+If no value is specified, it will default to `"lib"`.
+
 ### Customizing the background migrations module
 
-`OnlineMigrations.config.background_migrations.migrations_module` can be configured to define the module in which
+`config.background_migrations.migrations_module` can be configured to define the module in which
 background migrations will be placed.
 
 ```ruby
 # config/initializers/online_migrations.rb
 
-OnlineMigrations.config.background_migrations.migrations_module = "BackgroundMigrationsModule"
+config.background_migrations.migrations_module = "BackgroundMigrationsModule"
 ```
 
-If no value is specified, it will default to `OnlineMigrations::BackgroundMigrations`.
+If no value is specified, it will default to `"OnlineMigrations::BackgroundMigrations"`.
 
 ### Customizing the backtrace cleaner
 
-`OnlineMigrations.config.background_migrations.backtrace_cleaner` can be configured to specify a backtrace cleaner to use when a Background Migration errors and the backtrace is cleaned and persisted. An `ActiveSupport::BacktraceCleaner` should be used.
+`config.background_migrations.backtrace_cleaner` can be configured to specify a backtrace cleaner to use when a Background Migration errors and the backtrace is cleaned and persisted. An `ActiveSupport::BacktraceCleaner` should be used.
 
 ```ruby
 # config/initializers/online_migrations.rb
@@ -304,7 +316,7 @@ If no value is specified, it will default to `OnlineMigrations::BackgroundMigrat
 cleaner = ActiveSupport::BacktraceCleaner.new
 cleaner.add_silencer { |line| line =~ /ignore_this_dir/ }
 
-OnlineMigrations.config.background_migrations.backtrace_cleaner = cleaner
+config.background_migrations.backtrace_cleaner = cleaner
 ```
 
 If none is specified, the default `Rails.backtrace_cleaner` will be used to clean backtraces.

data/lib/generators/online_migrations/background_migration_generator.rb

@@ -9,8 +9,11 @@ module OnlineMigrations
     desc "This generator creates a background migration file."
 
     def create_background_migration_file
+      migrations_module_file_path = migrations_module.underscore
+
       template_file = File.join(
-        "lib/#{migrations_module_file_path}",
+        config.migrations_path,
+        migrations_module_file_path,
         class_path,
         "#{file_name}.rb"
       )
@@ -18,12 +21,12 @@ module OnlineMigrations
     end
 
     private
-      def migrations_module_file_path
-        migrations_module.underscore
+      def migrations_module
+        config.migrations_module
       end
 
-      def migrations_module
-        OnlineMigrations.config.background_migrations.migrations_module
+      def config
+        OnlineMigrations.config.background_migrations
       end
   end
 end

data/lib/generators/online_migrations/templates/initializer.rb.tt

@@ -79,6 +79,9 @@ OnlineMigrations.configure do |config|
 
   # ==> Background migrations configuration
 
+  # The path where generated background migrations will be placed.
+  # config.background_migrations.migrations_path = "lib"
+
   # The module in which background migrations will be placed.
   # config.background_migrations.migrations_module = "OnlineMigrations::BackgroundMigrations"
 

data/lib/online_migrations/background_migrations/config.rb

@@ -4,6 +4,10 @@ module OnlineMigrations
   module BackgroundMigrations
     # Class representing configuration options for background migrations.
     class Config
+      # The path where generated background migrations will be placed
+      # @return [String] defaults to "lib"
+      attr_accessor :migrations_path
+
       # The module in which background migrations will be placed
       # @return [String] defaults to "OnlineMigrations::BackgroundMigrations"
       attr_accessor :migrations_module
@@ -75,6 +79,7 @@ module OnlineMigrations
       attr_accessor :error_handler
 
       def initialize
+        @migrations_path = "lib"
         @migrations_module = "OnlineMigrations::BackgroundMigrations"
         @batch_size = 20_000
         @sub_batch_size = 1000

data/lib/online_migrations/background_migrations/migration.rb

@@ -28,6 +28,8 @@ module OnlineMigrations
         for_migration_name(migration_name).where("arguments = ?", arguments.to_json)
       end
 
+      alias_attribute :name, :migration_name
+
       enum status: STATUSES.index_with(&:to_s)
 
       belongs_to :parent, class_name: name, optional: true
@@ -64,6 +66,7 @@ module OnlineMigrations
         class_name = class_name.name if class_name.is_a?(Class)
         write_attribute(:migration_name, self.class.normalize_migration_name(class_name))
       end
+      alias name= migration_name=
 
       def completed?
         succeeded? || failed?
@@ -194,10 +197,8 @@ module OnlineMigrations
 
         on_shard do
           # rubocop:disable Lint/UnreachableLoop
-          iterator.each_batch(of: batch_size, column: batch_column_name, start: next_min_value) do |relation|
-            min = relation.arel_table[batch_column_name].minimum
-            max = relation.arel_table[batch_column_name].maximum
-            batch_range = relation.pick(min, max)
+          iterator.each_batch(of: batch_size, column: batch_column_name, start: next_min_value, finish: max_value) do |_relation, min_value, max_value|
+            batch_range = [min_value, max_value]
 
             break
           end
@@ -249,14 +250,10 @@ module OnlineMigrations
               self.min_value = self.max_value = self.rows_count = -1 # not relevant
             else
               on_shard do
-                self.min_value ||= migration_relation.minimum(batch_column_name)
-                self.max_value ||= migration_relation.maximum(batch_column_name)
-
-                # This can be the case when run in development on empty tables
-                if min_value.nil?
-                  # integer IDs minimum value is 1
-                  self.min_value = self.max_value = 1
-                end
+                # Getting exact min/max values can be a very heavy operation
+                # and is not needed practically.
+                self.min_value ||= 1
+                self.max_value ||= migration_model.unscoped.maximum(batch_column_name) || self.min_value
 
                 count = migration_object.count
                 self.rows_count = count if count != :no_count

data/lib/online_migrations/batch_iterator.rb

@@ -19,37 +19,42 @@ module OnlineMigrations
       end
 
       relation = apply_limits(self.relation, column, start, finish, order)
+      base_relation = relation.reselect(column).reorder(column => order)
 
-      base_relation = relation.except(:select)
-        .select(column)
-        .reorder(column => order)
-
-      start_row = base_relation.uncached { base_relation.first }
-
-      return if !start_row
+      start_id = start || begin
+        start_row = base_relation.uncached { base_relation.first }
+        start_row[column] if start_row
+      end
 
-      start_id = start_row[column]
       arel_table = relation.arel_table
 
-      0.step do |index|
+      while start_id
         if order == :asc
           start_cond = arel_table[column].gteq(start_id)
         else
           start_cond = arel_table[column].lteq(start_id)
         end
 
-        stop_row = base_relation.uncached do
+        last_row, stop_row = base_relation.uncached do
           base_relation
             .where(start_cond)
-            .offset(of)
-            .first
+            .offset(of - 1)
+            .first(2)
+        end
+
+        if last_row.nil?
+          # We are at the end of the table.
+          last_row, stop_row = base_relation.uncached do
+            base_relation
+              .where(start_cond)
+              .last(2)
+          end
         end
 
         batch_relation = relation.where(start_cond)
 
         if stop_row
           stop_id = stop_row[column]
-          start_id = stop_id
 
           if order == :asc
             stop_cond = arel_table[column].lt(stop_id)
@@ -64,10 +69,14 @@ module OnlineMigrations
         # efficient UPDATE queries, hence we get rid of it.
         batch_relation = batch_relation.except(:order)
 
+        last_id = (last_row && last_row[column]) || finish
+
         # Retaining the results in the query cache would undermine the point of batching.
-        batch_relation.uncached { yield batch_relation, index }
+        batch_relation.uncached { yield batch_relation, start_id, last_id }
+
+        break if stop_row.nil?
 
-        break if !stop_row
+        start_id = stop_id
       end
     end
 

data/lib/online_migrations/change_column_type_helpers.rb

@@ -416,15 +416,8 @@ module OnlineMigrations
               end
             end
 
-          if index.name.include?(from_column)
-            name = index.name.gsub(from_column, to_column)
-          end
-
-          name = index_name(table_name, new_columns) if !name || name.length > max_identifier_length
-
           options = {
             unique: index.unique,
-            name: name,
             length: index.lengths,
             order: index.orders,
           }

data/lib/online_migrations/schema_statements.rb

@@ -681,24 +681,20 @@ module OnlineMigrations
     # @see https://edgeapi.rubyonrails.org/classes/ActiveRecord/ConnectionAdapters/SchemaStatements.html#method-i-add_index
     #
     def add_index(table_name, column_name, **options)
-      algorithm = options[:algorithm]
-
-      __ensure_not_in_transaction! if algorithm == :concurrently
-
-      column_names = index_column_names(column_name || options[:column])
-
-      index_name = options[:name]
-      index_name ||= index_name(table_name, column_names)
+      __ensure_not_in_transaction! if options[:algorithm] == :concurrently
 
-      if index_exists?(table_name, column_name, **options)
+      # Rewrite this with `IndexDefinition#defined_for?` when Active Record >= 7.1 is supported.
+      # See https://github.com/rails/rails/pull/45160.
+      index = indexes(table_name).find { |i| __index_defined_for?(i, column_name, **options) }
+      if index
         schema = __schema_for_table(table_name)
 
-        if __index_valid?(index_name, schema: schema)
+        if __index_valid?(index.name, schema: schema)
           Utils.say("Index was not created because it already exists.")
           return
         else
           Utils.say("Recreating invalid index: table_name: #{table_name}, column_name: #{column_name}")
-          remove_index(table_name, column_name, name: index_name, algorithm: algorithm)
+          remove_index(table_name, column_name, **options)
         end
       end
 
@@ -706,7 +702,7 @@ module OnlineMigrations
         # "CREATE INDEX CONCURRENTLY" requires a "SHARE UPDATE EXCLUSIVE" lock.
         # It only conflicts with constraint validations, creating/removing indexes,
         # and some other "ALTER TABLE"s.
-        super(table_name, column_name, **options.merge(name: index_name))
+        super
       else
         OnlineMigrations.deprecator.warn(<<~MSG)
           Running `add_index` without a statement timeout is deprecated.
@@ -716,7 +712,7 @@ module OnlineMigrations
         MSG
 
         disable_statement_timeout do
-          super(table_name, column_name, **options.merge(name: index_name))
+          super
         end
       end
     end
@@ -944,6 +940,17 @@ module OnlineMigrations
         end
       end
 
+      # Will not be needed for Active Record >= 7.1
+      def __index_defined_for?(index, columns = nil, name: nil, unique: nil, valid: nil, include: nil, nulls_not_distinct: nil, **options)
+        columns = options[:column] if columns.blank?
+        (columns.nil? || Array(index.columns) == Array(columns).map(&:to_s)) &&
+          (name.nil? || index.name == name.to_s) &&
+          (unique.nil? || index.unique == unique) &&
+          (valid.nil? || index.valid == valid) &&
+          (include.nil? || Array(index.include) == Array(include).map(&:to_s)) &&
+          (nulls_not_distinct.nil? || index.nulls_not_distinct == nulls_not_distinct)
+      end
+
       def __not_null_constraint_exists?(table_name, column_name, name: nil)
         name ||= __not_null_constraint_name(table_name, column_name)
         __check_constraint_exists?(table_name, name: name)

data/lib/online_migrations/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module OnlineMigrations
-  VERSION = "0.12.0"
+  VERSION = "0.13.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: online_migrations
 version: !ruby/object:Gem::Version
-  version: 0.12.0
+  version: 0.13.1
 platform: ruby
 authors:
 - fatkodima
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-01-17 00:00:00.000000000 Z
+date: 2024-01-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord