online_migrations 0.12.0 → 0.13.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: a5db4da5657a6887113af46baa5c5ec9a6906ba4f8e3d33afc3d81f378f9b5ae
4
- data.tar.gz: 0acf2706e6b453055d41f338908323a0672066b6e742724b8a6d588e969ca05f
3
+ metadata.gz: 8952366397c54505f364a353dde1d68e988ac4773e9b3663de790a75c8b860f3
4
+ data.tar.gz: 376dc6e760a2b07d55968a11097f9f81067bc72291718938d1102fafb46a73ea
5
5
  SHA512:
6
- metadata.gz: 4a1b97dceb858d5c6ebd6ddccab4ea0b3ed8d257074f1e7f6562f101b3ae002ac19576053a44fe765624350c40d6e269d753fd24be5b5419474036a308b5cce5
7
- data.tar.gz: 9b6a3ece26c9e3e679c5d8d0f827eda7aa10b591242440a18d2a0fb18aa8f16b16935db6654859aab9b6f5008160e117c2eb32e98571261127d1a6ac1e936f2c
6
+ metadata.gz: 3e40ed5ab49e5d702c7d7cfebdb4264621375ad9affb67e8c3592814c4a832e75929c4ff718ce90709e64497226df3d9503a42babf56cfaad5ce973026f71d82
7
+ data.tar.gz: cdcbc9c26ac23fb975f21dd58c9826dab59c9571d074da06211dea2ffae72c3c57372722110ecd729ba649f21db551f58e8ca43689214f1235cd20669f033cf6
data/CHANGELOG.md CHANGED
@@ -1,5 +1,22 @@
1
1
  ## master (unreleased)
2
2
 
3
+ ## 0.13.1 (2024-01-23)
4
+
5
+ - Fix calculation of batch ranges for background migration created with explicit ranges
6
+
7
+ ## 0.13.0 (2024-01-22)
8
+
9
+ - Add ability to configure the path where generated background migrations will be placed
10
+
11
+ ```ruby
12
+ # It is placed in lib/ by default.
13
+ config.background_migrations.migrations_path = "app/lib"
14
+ ```
15
+
16
+ - Reduce number of queries needed to calculate batch ranges for background migrations
17
+ - Fix `finalize_column_type_change` to not recreate already existing indexes on the temporary column
18
+ - Remove potentially heavy queries used to get the ranges of a background migration
19
+
3
20
  ## 0.12.0 (2024-01-18)
4
21
 
5
22
  - Require passing model name for background migration helpers when using multiple databases
@@ -255,7 +255,7 @@ Specify the throttle condition as a block:
255
255
  ```ruby
256
256
  # config/initializers/online_migrations.rb
257
257
 
258
- OnlineMigrations.config.background_migrations.throttler = -> { DatabaseStatus.unhealthy? }
258
+ config.background_migrations.throttler = -> { DatabaseStatus.unhealthy? }
259
259
  ```
260
260
 
261
261
  Note that it's up to you to define a throttling condition that makes sense for your app. For example, you can check various PostgreSQL metrics such as replication lag, DB threads, whether DB writes are available, etc.
@@ -269,7 +269,7 @@ If you want to integrate with an exception monitoring service (e.g. Bugsnag), yo
269
269
  ```ruby
270
270
  # config/initializers/online_migrations.rb
271
271
 
272
- OnlineMigrations.config.background_migrations.error_handler = ->(error, errored_job) do
272
+ config.background_migrations.error_handler = ->(error, errored_job) do
273
273
  Bugsnag.notify(error) do |notification|
274
274
  notification.add_metadata(:background_migration, { name: errored_job.migration_name })
275
275
  end
@@ -281,22 +281,34 @@ The error handler should be a lambda that accepts 2 arguments:
281
281
  * `error`: The exception that was raised.
282
282
  * `errored_job`: An `OnlineMigrations::BackgroundMigrations::MigrationJob` object that represents a failed batch.
283
283
 
284
+ ### Customizing the background migrations path
285
+
286
+ `OnlineMigrations.config.background_migrations.migrations_path` can be configured to define where generated background migrations will be placed.
287
+
288
+ ```ruby
289
+ # config/initializers/online_migrations.rb
290
+
291
+ config.background_migrations.migrations_path = "app/lib"
292
+ ```
293
+
294
+ If no value is specified, it will default to `"lib"`.
295
+
284
296
  ### Customizing the background migrations module
285
297
 
286
- `OnlineMigrations.config.background_migrations.migrations_module` can be configured to define the module in which
298
+ `config.background_migrations.migrations_module` can be configured to define the module in which
287
299
  background migrations will be placed.
288
300
 
289
301
  ```ruby
290
302
  # config/initializers/online_migrations.rb
291
303
 
292
- OnlineMigrations.config.background_migrations.migrations_module = "BackgroundMigrationsModule"
304
+ config.background_migrations.migrations_module = "BackgroundMigrationsModule"
293
305
  ```
294
306
 
295
- If no value is specified, it will default to `OnlineMigrations::BackgroundMigrations`.
307
+ If no value is specified, it will default to `"OnlineMigrations::BackgroundMigrations"`.
296
308
 
297
309
  ### Customizing the backtrace cleaner
298
310
 
299
- `OnlineMigrations.config.background_migrations.backtrace_cleaner` can be configured to specify a backtrace cleaner to use when a Background Migration errors and the backtrace is cleaned and persisted. An `ActiveSupport::BacktraceCleaner` should be used.
311
+ `config.background_migrations.backtrace_cleaner` can be configured to specify a backtrace cleaner to use when a Background Migration errors and the backtrace is cleaned and persisted. An `ActiveSupport::BacktraceCleaner` should be used.
300
312
 
301
313
  ```ruby
302
314
  # config/initializers/online_migrations.rb
@@ -304,7 +316,7 @@ If no value is specified, it will default to `OnlineMigrations::BackgroundMigrat
304
316
  cleaner = ActiveSupport::BacktraceCleaner.new
305
317
  cleaner.add_silencer { |line| line =~ /ignore_this_dir/ }
306
318
 
307
- OnlineMigrations.config.background_migrations.backtrace_cleaner = cleaner
319
+ config.background_migrations.backtrace_cleaner = cleaner
308
320
  ```
309
321
 
310
322
  If none is specified, the default `Rails.backtrace_cleaner` will be used to clean backtraces.
@@ -9,8 +9,11 @@ module OnlineMigrations
9
9
  desc "This generator creates a background migration file."
10
10
 
11
11
  def create_background_migration_file
12
+ migrations_module_file_path = migrations_module.underscore
13
+
12
14
  template_file = File.join(
13
- "lib/#{migrations_module_file_path}",
15
+ config.migrations_path,
16
+ migrations_module_file_path,
14
17
  class_path,
15
18
  "#{file_name}.rb"
16
19
  )
@@ -18,12 +21,12 @@ module OnlineMigrations
18
21
  end
19
22
 
20
23
  private
21
- def migrations_module_file_path
22
- migrations_module.underscore
24
+ def migrations_module
25
+ config.migrations_module
23
26
  end
24
27
 
25
- def migrations_module
26
- OnlineMigrations.config.background_migrations.migrations_module
28
+ def config
29
+ OnlineMigrations.config.background_migrations
27
30
  end
28
31
  end
29
32
  end
@@ -79,6 +79,9 @@ OnlineMigrations.configure do |config|
79
79
 
80
80
  # ==> Background migrations configuration
81
81
 
82
+ # The path where generated background migrations will be placed.
83
+ # config.background_migrations.migrations_path = "lib"
84
+
82
85
  # The module in which background migrations will be placed.
83
86
  # config.background_migrations.migrations_module = "OnlineMigrations::BackgroundMigrations"
84
87
 
@@ -4,6 +4,10 @@ module OnlineMigrations
4
4
  module BackgroundMigrations
5
5
  # Class representing configuration options for background migrations.
6
6
  class Config
7
+ # The path where generated background migrations will be placed
8
+ # @return [String] defaults to "lib"
9
+ attr_accessor :migrations_path
10
+
7
11
  # The module in which background migrations will be placed
8
12
  # @return [String] defaults to "OnlineMigrations::BackgroundMigrations"
9
13
  attr_accessor :migrations_module
@@ -75,6 +79,7 @@ module OnlineMigrations
75
79
  attr_accessor :error_handler
76
80
 
77
81
  def initialize
82
+ @migrations_path = "lib"
78
83
  @migrations_module = "OnlineMigrations::BackgroundMigrations"
79
84
  @batch_size = 20_000
80
85
  @sub_batch_size = 1000
@@ -28,6 +28,8 @@ module OnlineMigrations
28
28
  for_migration_name(migration_name).where("arguments = ?", arguments.to_json)
29
29
  end
30
30
 
31
+ alias_attribute :name, :migration_name
32
+
31
33
  enum status: STATUSES.index_with(&:to_s)
32
34
 
33
35
  belongs_to :parent, class_name: name, optional: true
@@ -64,6 +66,7 @@ module OnlineMigrations
64
66
  class_name = class_name.name if class_name.is_a?(Class)
65
67
  write_attribute(:migration_name, self.class.normalize_migration_name(class_name))
66
68
  end
69
+ alias name= migration_name=
67
70
 
68
71
  def completed?
69
72
  succeeded? || failed?
@@ -194,10 +197,8 @@ module OnlineMigrations
194
197
 
195
198
  on_shard do
196
199
  # rubocop:disable Lint/UnreachableLoop
197
- iterator.each_batch(of: batch_size, column: batch_column_name, start: next_min_value) do |relation|
198
- min = relation.arel_table[batch_column_name].minimum
199
- max = relation.arel_table[batch_column_name].maximum
200
- batch_range = relation.pick(min, max)
200
+ iterator.each_batch(of: batch_size, column: batch_column_name, start: next_min_value, finish: max_value) do |_relation, min_value, max_value|
201
+ batch_range = [min_value, max_value]
201
202
 
202
203
  break
203
204
  end
@@ -249,14 +250,10 @@ module OnlineMigrations
249
250
  self.min_value = self.max_value = self.rows_count = -1 # not relevant
250
251
  else
251
252
  on_shard do
252
- self.min_value ||= migration_relation.minimum(batch_column_name)
253
- self.max_value ||= migration_relation.maximum(batch_column_name)
254
-
255
- # This can be the case when run in development on empty tables
256
- if min_value.nil?
257
- # integer IDs minimum value is 1
258
- self.min_value = self.max_value = 1
259
- end
253
+ # Getting exact min/max values can be a very heavy operation
254
+ # and is not needed practically.
255
+ self.min_value ||= 1
256
+ self.max_value ||= migration_model.unscoped.maximum(batch_column_name) || self.min_value
260
257
 
261
258
  count = migration_object.count
262
259
  self.rows_count = count if count != :no_count
@@ -19,37 +19,42 @@ module OnlineMigrations
19
19
  end
20
20
 
21
21
  relation = apply_limits(self.relation, column, start, finish, order)
22
+ base_relation = relation.reselect(column).reorder(column => order)
22
23
 
23
- base_relation = relation.except(:select)
24
- .select(column)
25
- .reorder(column => order)
26
-
27
- start_row = base_relation.uncached { base_relation.first }
28
-
29
- return if !start_row
24
+ start_id = start || begin
25
+ start_row = base_relation.uncached { base_relation.first }
26
+ start_row[column] if start_row
27
+ end
30
28
 
31
- start_id = start_row[column]
32
29
  arel_table = relation.arel_table
33
30
 
34
- 0.step do |index|
31
+ while start_id
35
32
  if order == :asc
36
33
  start_cond = arel_table[column].gteq(start_id)
37
34
  else
38
35
  start_cond = arel_table[column].lteq(start_id)
39
36
  end
40
37
 
41
- stop_row = base_relation.uncached do
38
+ last_row, stop_row = base_relation.uncached do
42
39
  base_relation
43
40
  .where(start_cond)
44
- .offset(of)
45
- .first
41
+ .offset(of - 1)
42
+ .first(2)
43
+ end
44
+
45
+ if last_row.nil?
46
+ # We are at the end of the table.
47
+ last_row, stop_row = base_relation.uncached do
48
+ base_relation
49
+ .where(start_cond)
50
+ .last(2)
51
+ end
46
52
  end
47
53
 
48
54
  batch_relation = relation.where(start_cond)
49
55
 
50
56
  if stop_row
51
57
  stop_id = stop_row[column]
52
- start_id = stop_id
53
58
 
54
59
  if order == :asc
55
60
  stop_cond = arel_table[column].lt(stop_id)
@@ -64,10 +69,14 @@ module OnlineMigrations
64
69
  # efficient UPDATE queries, hence we get rid of it.
65
70
  batch_relation = batch_relation.except(:order)
66
71
 
72
+ last_id = (last_row && last_row[column]) || finish
73
+
67
74
  # Retaining the results in the query cache would undermine the point of batching.
68
- batch_relation.uncached { yield batch_relation, index }
75
+ batch_relation.uncached { yield batch_relation, start_id, last_id }
76
+
77
+ break if stop_row.nil?
69
78
 
70
- break if !stop_row
79
+ start_id = stop_id
71
80
  end
72
81
  end
73
82
 
@@ -416,15 +416,8 @@ module OnlineMigrations
416
416
  end
417
417
  end
418
418
 
419
- if index.name.include?(from_column)
420
- name = index.name.gsub(from_column, to_column)
421
- end
422
-
423
- name = index_name(table_name, new_columns) if !name || name.length > max_identifier_length
424
-
425
419
  options = {
426
420
  unique: index.unique,
427
- name: name,
428
421
  length: index.lengths,
429
422
  order: index.orders,
430
423
  }
@@ -681,24 +681,20 @@ module OnlineMigrations
681
681
  # @see https://edgeapi.rubyonrails.org/classes/ActiveRecord/ConnectionAdapters/SchemaStatements.html#method-i-add_index
682
682
  #
683
683
  def add_index(table_name, column_name, **options)
684
- algorithm = options[:algorithm]
685
-
686
- __ensure_not_in_transaction! if algorithm == :concurrently
687
-
688
- column_names = index_column_names(column_name || options[:column])
689
-
690
- index_name = options[:name]
691
- index_name ||= index_name(table_name, column_names)
684
+ __ensure_not_in_transaction! if options[:algorithm] == :concurrently
692
685
 
693
- if index_exists?(table_name, column_name, **options)
686
+ # Rewrite this with `IndexDefinition#defined_for?` when Active Record >= 7.1 is supported.
687
+ # See https://github.com/rails/rails/pull/45160.
688
+ index = indexes(table_name).find { |i| __index_defined_for?(i, column_name, **options) }
689
+ if index
694
690
  schema = __schema_for_table(table_name)
695
691
 
696
- if __index_valid?(index_name, schema: schema)
692
+ if __index_valid?(index.name, schema: schema)
697
693
  Utils.say("Index was not created because it already exists.")
698
694
  return
699
695
  else
700
696
  Utils.say("Recreating invalid index: table_name: #{table_name}, column_name: #{column_name}")
701
- remove_index(table_name, column_name, name: index_name, algorithm: algorithm)
697
+ remove_index(table_name, column_name, **options)
702
698
  end
703
699
  end
704
700
 
@@ -706,7 +702,7 @@ module OnlineMigrations
706
702
  # "CREATE INDEX CONCURRENTLY" requires a "SHARE UPDATE EXCLUSIVE" lock.
707
703
  # It only conflicts with constraint validations, creating/removing indexes,
708
704
  # and some other "ALTER TABLE"s.
709
- super(table_name, column_name, **options.merge(name: index_name))
705
+ super
710
706
  else
711
707
  OnlineMigrations.deprecator.warn(<<~MSG)
712
708
  Running `add_index` without a statement timeout is deprecated.
@@ -716,7 +712,7 @@ module OnlineMigrations
716
712
  MSG
717
713
 
718
714
  disable_statement_timeout do
719
- super(table_name, column_name, **options.merge(name: index_name))
715
+ super
720
716
  end
721
717
  end
722
718
  end
@@ -944,6 +940,17 @@ module OnlineMigrations
944
940
  end
945
941
  end
946
942
 
943
+ # Will not be needed for Active Record >= 7.1
944
+ def __index_defined_for?(index, columns = nil, name: nil, unique: nil, valid: nil, include: nil, nulls_not_distinct: nil, **options)
945
+ columns = options[:column] if columns.blank?
946
+ (columns.nil? || Array(index.columns) == Array(columns).map(&:to_s)) &&
947
+ (name.nil? || index.name == name.to_s) &&
948
+ (unique.nil? || index.unique == unique) &&
949
+ (valid.nil? || index.valid == valid) &&
950
+ (include.nil? || Array(index.include) == Array(include).map(&:to_s)) &&
951
+ (nulls_not_distinct.nil? || index.nulls_not_distinct == nulls_not_distinct)
952
+ end
953
+
947
954
  def __not_null_constraint_exists?(table_name, column_name, name: nil)
948
955
  name ||= __not_null_constraint_name(table_name, column_name)
949
956
  __check_constraint_exists?(table_name, name: name)
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module OnlineMigrations
4
- VERSION = "0.12.0"
4
+ VERSION = "0.13.1"
5
5
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: online_migrations
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.12.0
4
+ version: 0.13.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - fatkodima
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2024-01-17 00:00:00.000000000 Z
11
+ date: 2024-01-23 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activerecord