online_migrations 0.4.1 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/BACKGROUND_MIGRATIONS.md +6 -5
- data/CHANGELOG.md +69 -0
- data/README.md +1 -1
- data/lib/online_migrations/background_migrations/delete_associated_records.rb +28 -0
- data/lib/online_migrations/background_migrations/delete_orphaned_records.rb +45 -0
- data/lib/online_migrations/background_migrations/migration_helpers.rb +113 -4
- data/lib/online_migrations/background_migrations/perform_action_on_relation.rb +33 -0
- data/lib/online_migrations/change_column_type_helpers.rb +1 -1
- data/lib/online_migrations/command_checker.rb +18 -6
- data/lib/online_migrations/command_recorder.rb +7 -2
- data/lib/online_migrations/error_messages.rb +6 -1
- data/lib/online_migrations/schema_statements.rb +2 -2
- data/lib/online_migrations/verbose_sql_logs.rb +2 -2
- data/lib/online_migrations/version.rb +1 -1
- data/lib/online_migrations.rb +3 -0
- metadata +6 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c2515b6dd51e983fbcbdb448d33273e77d5b7f30ef53bc42dacb5127ac390fc8
|
4
|
+
data.tar.gz: b099a129bfea91da13d1dbde5c08d18fb09ae195dd83b56495c5513a881f609f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: '0220675939d3a08c0ecb16ba3e4a212bb40106d34478b7008c2458d2eebbead2c897873634589a6ae55b0665fdf0a02fb691432c639198c7c3a59e378c10d023'
|
7
|
+
data.tar.gz: b3d9a76edb0f4e220790a7de2a2ba5f7930c9106599dd488f585f4003ae61166341feccd66cbc63965b5a181217fe5639eb17e836be48542b0070591c74b8ab6
|
data/BACKGROUND_MIGRATIONS.md
CHANGED
@@ -116,9 +116,12 @@ enqueue_background_migration("MyMigrationWithArgs", arg1, arg2, ...)
|
|
116
116
|
|
117
117
|
## Predefined background migrations
|
118
118
|
|
119
|
-
* `BackfillColumn` - backfills column(s) with scalar values
|
120
|
-
* `CopyColumn` - copies data from one column(s) to other(s)
|
121
|
-
* `
|
119
|
+
* `BackfillColumn` - backfills column(s) with scalar values (enqueue using `backfill_column_in_background`)
|
120
|
+
* `CopyColumn` - copies data from one column(s) to other(s) (enqueue using `copy_column_in_background`)
|
121
|
+
* `DeleteAssociatedRecords` - deletes records associated with a parent object (enqueue using `delete_associated_records_in_background`)
|
122
|
+
* `DeleteOrphanedRecords` - deletes records with one or more missing relations (enqueue using `delete_orphaned_records_in_background`)
|
123
|
+
* `PerformActionOnRelation` - performs specific action on a relation or indvidual records (enqueue using `perform_action_on_relation_in_background`)
|
124
|
+
* `ResetCounters` - resets one or more counter caches to their correct value (enqueue using `reset_counters_in_background`)
|
122
125
|
|
123
126
|
## Testing
|
124
127
|
|
@@ -262,8 +265,6 @@ The error handler should be a lambda that accepts 2 arguments:
|
|
262
265
|
|
263
266
|
* `error`: The exception that was raised.
|
264
267
|
* `errored_job`: An `OnlineMigrations::BackgroundMigrations::MigrationJob` object that represents a failed batch.
|
265
|
-
* `errored_element`: The `OnlineMigrations::BackgroundMigrations::MigrationJob` object representing a batch,
|
266
|
-
that was being processed when the Background Migration raised an exception.
|
267
268
|
|
268
269
|
### Customizing the background migrations module
|
269
270
|
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,74 @@
|
|
1
1
|
## master (unreleased)
|
2
2
|
|
3
|
+
## 0.5.0 (2022-06-23)
|
4
|
+
|
5
|
+
- Added check for index corruption with PostgreSQL 14.0 to 14.3
|
6
|
+
|
7
|
+
- No need to separately remove indexes when removing a column from the small table
|
8
|
+
|
9
|
+
- Add ability to perform specific action on a relation or individual records using background migrations
|
10
|
+
|
11
|
+
Example, assuming you have lots and lots of fraud likes:
|
12
|
+
|
13
|
+
```ruby
|
14
|
+
class DeleteFraudLikes < ActiveRecord::Migration[7.0]
|
15
|
+
def up
|
16
|
+
perform_action_on_relation_in_background("Like", { fraud: true }, :delete_all)
|
17
|
+
end
|
18
|
+
end
|
19
|
+
```
|
20
|
+
|
21
|
+
Example, assuming you added a new column to the users and want to populate it:
|
22
|
+
|
23
|
+
```ruby
|
24
|
+
class User < ApplicationRecord
|
25
|
+
def generate_invite_token
|
26
|
+
self.invite_token = # some complex logic
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
perform_action_on_relation_in_background("User", { invite_token: nil }, :generate_invite_token)
|
31
|
+
```
|
32
|
+
|
33
|
+
You can use `delete_all`/`destroy_all`/`update_all` for the whole relation or run specific methods on individual records.
|
34
|
+
|
35
|
+
- Add ability to delete records associated with a parent object using background migrations
|
36
|
+
|
37
|
+
```ruby
|
38
|
+
class Link < ActiveRecord::Base
|
39
|
+
has_many :clicks
|
40
|
+
end
|
41
|
+
|
42
|
+
class Click < ActiveRecord::Base
|
43
|
+
belongs_to :link
|
44
|
+
end
|
45
|
+
|
46
|
+
class DeleteSomeLinkClicks < ActiveRecord::Migration[7.0]
|
47
|
+
def up
|
48
|
+
some_link = ...
|
49
|
+
delete_associated_records_in_background("Link", some_link.id, :clicks)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
```
|
53
|
+
|
54
|
+
- Add ability to delete orphaned records using background migrations
|
55
|
+
|
56
|
+
```ruby
|
57
|
+
class User < ApplicationRecord
|
58
|
+
has_many :posts
|
59
|
+
end
|
60
|
+
|
61
|
+
class Post < ApplicationRecord
|
62
|
+
belongs_to :author, class_name: 'User'
|
63
|
+
end
|
64
|
+
|
65
|
+
class DeleteOrphanedPosts < ActiveRecord::Migration[7.0]
|
66
|
+
def up
|
67
|
+
delete_orphaned_records_in_background("Post", :author)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
```
|
71
|
+
|
3
72
|
## 0.4.1 (2022-03-21)
|
4
73
|
|
5
74
|
- Fix missing options in suggested command for columns removal
|
data/README.md
CHANGED
@@ -351,7 +351,7 @@ A safer approach can be accomplished in several steps:
|
|
351
351
|
```ruby
|
352
352
|
class CleanupChangeFilesSizeType < ActiveRecord::Migration[7.0]
|
353
353
|
def up
|
354
|
-
|
354
|
+
cleanup_column_type_change :files, :size
|
355
355
|
end
|
356
356
|
|
357
357
|
def down
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module OnlineMigrations
|
4
|
+
module BackgroundMigrations
|
5
|
+
# @private
|
6
|
+
class DeleteAssociatedRecords < BackgroundMigration
|
7
|
+
attr_reader :record, :association
|
8
|
+
|
9
|
+
def initialize(model_name, record_id, association, _options = {})
|
10
|
+
model = Object.const_get(model_name, false)
|
11
|
+
@record = model.find(record_id)
|
12
|
+
@association = association
|
13
|
+
end
|
14
|
+
|
15
|
+
def relation
|
16
|
+
unless @record.respond_to?(association)
|
17
|
+
raise ArgumentError, "'#{@record.class.name}' has no association called '#{association}'"
|
18
|
+
end
|
19
|
+
|
20
|
+
record.public_send(association)
|
21
|
+
end
|
22
|
+
|
23
|
+
def process_batch(relation)
|
24
|
+
relation.delete_all(:delete_all)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module OnlineMigrations
|
4
|
+
module BackgroundMigrations
|
5
|
+
# @private
|
6
|
+
class DeleteOrphanedRecords < BackgroundMigration
|
7
|
+
attr_reader :model, :associations
|
8
|
+
|
9
|
+
def initialize(model_name, associations, _options = {})
|
10
|
+
@model = Object.const_get(model_name, false)
|
11
|
+
@associations = associations.map(&:to_sym)
|
12
|
+
end
|
13
|
+
|
14
|
+
def relation
|
15
|
+
# For ActiveRecord 6.1+ we can use `where.missing`
|
16
|
+
# https://github.com/rails/rails/pull/34727
|
17
|
+
associations.inject(model.unscoped) do |relation, association|
|
18
|
+
reflection = model.reflect_on_association(association)
|
19
|
+
unless reflection
|
20
|
+
raise ArgumentError, "'#{model.name}' has no association called '#{association}'"
|
21
|
+
end
|
22
|
+
|
23
|
+
# left_joins was added in ActiveRecord 5.0 - https://github.com/rails/rails/pull/12071
|
24
|
+
relation
|
25
|
+
.left_joins(association)
|
26
|
+
.where(reflection.table_name => { reflection.association_primary_key => nil })
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def process_batch(relation)
|
31
|
+
if Utils.ar_version > 5.0
|
32
|
+
relation.delete_all
|
33
|
+
else
|
34
|
+
# Older ActiveRecord generates incorrect query when running delete_all
|
35
|
+
primary_key = model.primary_key
|
36
|
+
model.unscoped.where(primary_key => relation.select(primary_key)).delete_all
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def count
|
41
|
+
Utils.estimated_count(model.connection, model.table_name)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -21,7 +21,7 @@ module OnlineMigrations
|
|
21
21
|
# @example Additional background migration options
|
22
22
|
# backfill_column_in_background(:users, :admin, false, batch_size: 10_000)
|
23
23
|
#
|
24
|
-
# @note This method is better suited for
|
24
|
+
# @note This method is better suited for large tables (10/100s of millions of records).
|
25
25
|
# For smaller tables it is probably better and easier to use more flexible `update_column_in_batches`.
|
26
26
|
#
|
27
27
|
# @note Consider `backfill_columns_in_background` when backfilling multiple columns
|
@@ -76,7 +76,7 @@ module OnlineMigrations
|
|
76
76
|
# @example Additional background migration options
|
77
77
|
# backfill_column_for_type_change_in_background(:files, :size, batch_size: 10_000)
|
78
78
|
#
|
79
|
-
# @note This method is better suited for
|
79
|
+
# @note This method is better suited for large tables (10/100s of millions of records).
|
80
80
|
# For smaller tables it is probably better and easier to use more flexible `backfill_column_for_type_change`.
|
81
81
|
#
|
82
82
|
def backfill_column_for_type_change_in_background(table_name, column_name, model_name: nil,
|
@@ -131,7 +131,7 @@ module OnlineMigrations
|
|
131
131
|
# @example
|
132
132
|
# copy_column_in_background(:users, :id, :id_for_type_change)
|
133
133
|
#
|
134
|
-
# @note This method is better suited for
|
134
|
+
# @note This method is better suited for large tables (10/100s of millions of records).
|
135
135
|
# For smaller tables it is probably better and easier to use more flexible `update_column_in_batches`.
|
136
136
|
#
|
137
137
|
def copy_column_in_background(table_name, copy_from, copy_to, model_name: nil, type_cast_function: nil, **options)
|
@@ -190,7 +190,7 @@ module OnlineMigrations
|
|
190
190
|
#
|
191
191
|
# @see https://api.rubyonrails.org/classes/ActiveRecord/CounterCache/ClassMethods.html#method-i-reset_counters
|
192
192
|
#
|
193
|
-
# @note This method is better suited for
|
193
|
+
# @note This method is better suited for large tables (10/100s of millions of records).
|
194
194
|
# For smaller tables it is probably better and easier to use `reset_counters` from the ActiveRecord.
|
195
195
|
#
|
196
196
|
def reset_counters_in_background(model_name, *counters, touch: nil, **options)
|
@@ -205,6 +205,115 @@ module OnlineMigrations
|
|
205
205
|
)
|
206
206
|
end
|
207
207
|
|
208
|
+
# Deletes records with one or more missing relations using background migrations.
|
209
|
+
# This is useful when some referential integrity in the database is broken and
|
210
|
+
# you want to delete orphaned records.
|
211
|
+
#
|
212
|
+
# @param model_name [String]
|
213
|
+
# @param associations [Array]
|
214
|
+
# @param options [Hash] used to control the behavior of background migration.
|
215
|
+
# See `#enqueue_background_migration`
|
216
|
+
#
|
217
|
+
# @return [OnlineMigrations::BackgroundMigrations::Migration]
|
218
|
+
#
|
219
|
+
# @example
|
220
|
+
# delete_orphaned_records_in_background("Post", :author)
|
221
|
+
#
|
222
|
+
# @note This method is better suited for large tables (10/100s of millions of records).
|
223
|
+
# For smaller tables it is probably better and easier to directly find and delete orpahed records.
|
224
|
+
#
|
225
|
+
def delete_orphaned_records_in_background(model_name, *associations, **options)
|
226
|
+
if Utils.ar_version <= 4.2
|
227
|
+
raise "#{__method__} does not support ActiveRecord <= 4.2 yet"
|
228
|
+
end
|
229
|
+
|
230
|
+
model_name = model_name.name if model_name.is_a?(Class)
|
231
|
+
|
232
|
+
enqueue_background_migration(
|
233
|
+
"DeleteOrphanedRecords",
|
234
|
+
model_name,
|
235
|
+
associations,
|
236
|
+
**options
|
237
|
+
)
|
238
|
+
end
|
239
|
+
|
240
|
+
# Deletes associated records for a specific parent record using background migrations.
|
241
|
+
# This is useful when you are planning to remove a parent object (user, account etc)
|
242
|
+
# and needs to remove lots of its associated objects.
|
243
|
+
#
|
244
|
+
# @param model_name [String]
|
245
|
+
# @param record_id [Integer, String] parent record primary key's value
|
246
|
+
# @param association [String, Symbol] association name for which records will be removed
|
247
|
+
# @param options [Hash] used to control the behavior of background migration.
|
248
|
+
# See `#enqueue_background_migration`
|
249
|
+
#
|
250
|
+
# @return [OnlineMigrations::BackgroundMigrations::Migration]
|
251
|
+
#
|
252
|
+
# @example
|
253
|
+
# delete_associated_records_in_background("Link", 1, :clicks)
|
254
|
+
#
|
255
|
+
# @note This method is better suited for large tables (10/100s of millions of records).
|
256
|
+
# For smaller tables it is probably better and easier to directly delete associated records.
|
257
|
+
#
|
258
|
+
def delete_associated_records_in_background(model_name, record_id, association, **options)
|
259
|
+
model_name = model_name.name if model_name.is_a?(Class)
|
260
|
+
|
261
|
+
enqueue_background_migration(
|
262
|
+
"DeleteAssociatedRecords",
|
263
|
+
model_name,
|
264
|
+
record_id,
|
265
|
+
association,
|
266
|
+
**options
|
267
|
+
)
|
268
|
+
end
|
269
|
+
|
270
|
+
# Performs specific action on a relation or individual records.
|
271
|
+
# This is useful when you want to delete/destroy/update/etc records based on some conditions.
|
272
|
+
#
|
273
|
+
# @param model_name [String]
|
274
|
+
# @param conditions [Array, Hash, String] conditions to filter the relation
|
275
|
+
# @param action [String, Symbol] action to perform on the relation or individual records.
|
276
|
+
# Relation-wide available actions: `:delete_all`, `:destroy_all`, and `:update_all`.
|
277
|
+
# @param updates [Hash] updates to perform when `action` is set to `:update_all`
|
278
|
+
# @param options [Hash] used to control the behavior of background migration.
|
279
|
+
# See `#enqueue_background_migration`
|
280
|
+
#
|
281
|
+
# @return [OnlineMigrations::BackgroundMigrations::Migration]
|
282
|
+
#
|
283
|
+
# @example Delete records
|
284
|
+
# perform_action_on_relation_in_background("User", { banned: true }, :delete_all)
|
285
|
+
#
|
286
|
+
# @example Destroy records
|
287
|
+
# perform_action_on_relation_in_background("User", { banned: true }, :destroy_all)
|
288
|
+
#
|
289
|
+
# @example Update records
|
290
|
+
# perform_action_on_relation_in_background("User", { banned: nil }, :update_all, updates: { banned: false })
|
291
|
+
#
|
292
|
+
# @example Perform custom method on individual records
|
293
|
+
# class User < ApplicationRecord
|
294
|
+
# def generate_invite_token
|
295
|
+
# self.invite_token = # some complex logic
|
296
|
+
# end
|
297
|
+
# end
|
298
|
+
#
|
299
|
+
# perform_action_on_relation_in_background("User", { invite_token: nil }, :generate_invite_token)
|
300
|
+
#
|
301
|
+
# @note This method is better suited for large tables (10/100s of millions of records).
|
302
|
+
# For smaller tables it is probably better and easier to directly delete associated records.
|
303
|
+
#
|
304
|
+
def perform_action_on_relation_in_background(model_name, conditions, action, updates: nil, **options)
|
305
|
+
model_name = model_name.name if model_name.is_a?(Class)
|
306
|
+
|
307
|
+
enqueue_background_migration(
|
308
|
+
"PerformActionOnRelation",
|
309
|
+
model_name,
|
310
|
+
conditions,
|
311
|
+
action,
|
312
|
+
{ updates: updates },
|
313
|
+
**options
|
314
|
+
)
|
315
|
+
end
|
316
|
+
|
208
317
|
# Creates a background migration for the given job class name.
|
209
318
|
#
|
210
319
|
# A background migration runs one job at a time, computing the bounds of the next batch
|
@@ -0,0 +1,33 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module OnlineMigrations
|
4
|
+
module BackgroundMigrations
|
5
|
+
# @private
|
6
|
+
class PerformActionOnRelation < BackgroundMigration
|
7
|
+
attr_reader :model, :conditions, :action, :options
|
8
|
+
|
9
|
+
def initialize(model_name, conditions, action, options = {})
|
10
|
+
@model = Object.const_get(model_name, false)
|
11
|
+
@conditions = conditions
|
12
|
+
@action = action.to_sym
|
13
|
+
@options = options.symbolize_keys
|
14
|
+
end
|
15
|
+
|
16
|
+
def relation
|
17
|
+
model.unscoped.where(conditions)
|
18
|
+
end
|
19
|
+
|
20
|
+
def process_batch(relation)
|
21
|
+
case action
|
22
|
+
when :update_all
|
23
|
+
updates = options.fetch(:updates)
|
24
|
+
relation.public_send(action, updates)
|
25
|
+
when :delete_all, :destroy_all
|
26
|
+
relation.public_send(action)
|
27
|
+
else
|
28
|
+
relation.each(&action)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -173,7 +173,7 @@ module OnlineMigrations
|
|
173
173
|
# backfill_column_for_type_change(:files, :size, batch_size: 10_000)
|
174
174
|
#
|
175
175
|
# @note This method should not be run within a transaction
|
176
|
-
# @note For
|
176
|
+
# @note For large tables (10/100s of millions of records)
|
177
177
|
# it is recommended to use `backfill_column_for_type_change_in_background`.
|
178
178
|
#
|
179
179
|
def backfill_column_for_type_change(table_name, column_name, type_cast_function: nil, **options)
|
@@ -316,7 +316,7 @@ module OnlineMigrations
|
|
316
316
|
initialize_change_code: command_str(:initialize_column_type_change, table_name, column_name, type, **options),
|
317
317
|
backfill_code: command_str(:backfill_column_for_type_change, table_name, column_name, **options),
|
318
318
|
finalize_code: command_str(:finalize_column_type_change, table_name, column_name),
|
319
|
-
cleanup_code: command_str(:
|
319
|
+
cleanup_code: command_str(:cleanup_column_type_change, table_name, column_name),
|
320
320
|
cleanup_down_code: command_str(:initialize_column_type_change, table_name, column_name, existing_type)
|
321
321
|
end
|
322
322
|
end
|
@@ -383,7 +383,8 @@ module OnlineMigrations
|
|
383
383
|
columns: columns.inspect,
|
384
384
|
command: command_str(command, *args, options),
|
385
385
|
table_name: table_name.inspect,
|
386
|
-
indexes: indexes.map { |i| i.name.to_sym.inspect }
|
386
|
+
indexes: indexes.map { |i| i.name.to_sym.inspect },
|
387
|
+
small_table: small_table?(table_name)
|
387
388
|
end
|
388
389
|
end
|
389
390
|
|
@@ -450,6 +451,10 @@ module OnlineMigrations
|
|
450
451
|
command: command_str(:add_index, table_name, column_name, **options.merge(algorithm: :concurrently))
|
451
452
|
end
|
452
453
|
|
454
|
+
if options[:algorithm] == :concurrently && index_corruption?
|
455
|
+
raise_error :add_index_corruption
|
456
|
+
end
|
457
|
+
|
453
458
|
if @removed_indexes.any?
|
454
459
|
index = IndexDefinition.new(table: table_name, columns: column_name, **options)
|
455
460
|
existing_indexes = connection.indexes(table_name)
|
@@ -576,16 +581,17 @@ module OnlineMigrations
|
|
576
581
|
end
|
577
582
|
|
578
583
|
def new_or_small_table?(table_name)
|
579
|
-
|
580
|
-
|
581
|
-
new_table?(table_name) ||
|
582
|
-
small_tables.include?(table_name.to_s)
|
584
|
+
new_table?(table_name) || small_table?(table_name)
|
583
585
|
end
|
584
586
|
|
585
587
|
def new_table?(table_name)
|
586
588
|
@new_tables.include?(table_name.to_s)
|
587
589
|
end
|
588
590
|
|
591
|
+
def small_table?(table_name)
|
592
|
+
OnlineMigrations.config.small_tables.include?(table_name.to_s)
|
593
|
+
end
|
594
|
+
|
589
595
|
def postgresql_version
|
590
596
|
version =
|
591
597
|
if Utils.developer_env? && (target_version = OnlineMigrations.config.target_version)
|
@@ -737,6 +743,12 @@ module OnlineMigrations
|
|
737
743
|
[table1.to_s, table2.to_s].sort.join("\0").gsub(/^(.*_)(.+)\0\1(.+)/, '\1\2_\3').tr("\0", "_")
|
738
744
|
end
|
739
745
|
|
746
|
+
def index_corruption?
|
747
|
+
postgresql_version >= Gem::Version.new("14.0") &&
|
748
|
+
postgresql_version < Gem::Version.new("14.4") &&
|
749
|
+
!Utils.developer_env?
|
750
|
+
end
|
751
|
+
|
740
752
|
def run_custom_checks(method, args)
|
741
753
|
OnlineMigrations.config.checks.each do |options, check|
|
742
754
|
if !options[:start_after] || version > options[:start_after]
|
@@ -16,7 +16,9 @@ module OnlineMigrations
|
|
16
16
|
:swap_column_names,
|
17
17
|
:add_column_with_default,
|
18
18
|
:add_not_null_constraint,
|
19
|
+
:remove_not_null_constraint,
|
19
20
|
:add_text_limit_constraint,
|
21
|
+
:remove_text_limit_constraint,
|
20
22
|
:add_reference_concurrently,
|
21
23
|
:change_column_type_in_background,
|
22
24
|
:enqueue_background_migration,
|
@@ -127,11 +129,14 @@ module OnlineMigrations
|
|
127
129
|
end
|
128
130
|
|
129
131
|
def invert_remove_text_limit_constraint(args)
|
130
|
-
|
132
|
+
options = args.extract_options!
|
133
|
+
table_name, column, limit = args
|
134
|
+
|
135
|
+
unless limit
|
131
136
|
raise ActiveRecord::IrreversibleMigration, "remove_text_limit_constraint is only reversible if given a limit."
|
132
137
|
end
|
133
138
|
|
134
|
-
|
139
|
+
[:add_text_limit_constraint, [table_name, column, limit, **options]]
|
135
140
|
end
|
136
141
|
end
|
137
142
|
end
|
@@ -233,7 +233,7 @@ class <%= migration_name %> < <%= migration_parent %>
|
|
233
233
|
end",
|
234
234
|
|
235
235
|
remove_column:
|
236
|
-
"<% if indexes.any? %>
|
236
|
+
"<% if !small_table && indexes.any? %>
|
237
237
|
Removing a column will automatically remove all of the indexes that involved the removed column.
|
238
238
|
But the indexes would be removed non-concurrently, so you need to safely remove the indexes first:
|
239
239
|
|
@@ -333,6 +333,11 @@ class <%= migration_name %> < <%= migration_parent %>
|
|
333
333
|
end
|
334
334
|
end",
|
335
335
|
|
336
|
+
add_index_corruption:
|
337
|
+
"Adding an index concurrently can cause silent data corruption in PostgreSQL 14.0 to 14.3.
|
338
|
+
Upgrade PostgreSQL before adding new indexes, or wrap this step in a safety_assured { ... }
|
339
|
+
block to accept the risk.",
|
340
|
+
|
336
341
|
remove_index:
|
337
342
|
"Removing an index non-concurrently blocks writes. Instead, use:
|
338
343
|
|
@@ -48,7 +48,7 @@ module OnlineMigrations
|
|
48
48
|
# @note This method should not be run within a transaction
|
49
49
|
# @note Consider `update_columns_in_batches` when updating multiple columns
|
50
50
|
# to avoid rewriting the table multiple times.
|
51
|
-
# @note For
|
51
|
+
# @note For large tables (10/100s of millions of records)
|
52
52
|
# you may consider using `backfill_column_in_background` or `copy_column_in_background`.
|
53
53
|
#
|
54
54
|
def update_column_in_batches(table_name, column_name, value, **options, &block)
|
@@ -361,7 +361,7 @@ module OnlineMigrations
|
|
361
361
|
# These steps ensure a column can be added to a large and commonly used table
|
362
362
|
# without locking the entire table for the duration of the table modification.
|
363
363
|
#
|
364
|
-
# For
|
364
|
+
# For large tables (10/100s of millions of records) you may consider implementing
|
365
365
|
# the steps from this helper method yourself as a separate migrations, replacing step #3
|
366
366
|
# with the help of background migrations (see `backfill_column_in_background`).
|
367
367
|
#
|
@@ -26,7 +26,7 @@ module OnlineMigrations
|
|
26
26
|
|
27
27
|
private
|
28
28
|
def verbose_query_logs
|
29
|
-
if Utils.ar_version
|
29
|
+
if Utils.ar_version >= 7.0
|
30
30
|
ActiveRecord.verbose_query_logs
|
31
31
|
elsif Utils.ar_version >= 5.2
|
32
32
|
ActiveRecord::Base.verbose_query_logs
|
@@ -34,7 +34,7 @@ module OnlineMigrations
|
|
34
34
|
end
|
35
35
|
|
36
36
|
def set_verbose_query_logs(value) # rubocop:disable Naming/AccessorMethodName
|
37
|
-
if Utils.ar_version
|
37
|
+
if Utils.ar_version >= 7.0
|
38
38
|
ActiveRecord.verbose_query_logs = value
|
39
39
|
elsif Utils.ar_version >= 5.2
|
40
40
|
ActiveRecord::Base.verbose_query_logs = value
|
data/lib/online_migrations.rb
CHANGED
@@ -46,6 +46,9 @@ module OnlineMigrations
|
|
46
46
|
autoload :BackgroundMigrationClassValidator
|
47
47
|
autoload :BackfillColumn
|
48
48
|
autoload :CopyColumn
|
49
|
+
autoload :DeleteAssociatedRecords
|
50
|
+
autoload :DeleteOrphanedRecords
|
51
|
+
autoload :PerformActionOnRelation
|
49
52
|
autoload :ResetCounters
|
50
53
|
autoload :MigrationJob
|
51
54
|
autoload :Migration
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: online_migrations
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- fatkodima
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-
|
11
|
+
date: 2022-06-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activerecord
|
@@ -47,6 +47,8 @@ files:
|
|
47
47
|
- lib/online_migrations/background_migrations/background_migration_class_validator.rb
|
48
48
|
- lib/online_migrations/background_migrations/config.rb
|
49
49
|
- lib/online_migrations/background_migrations/copy_column.rb
|
50
|
+
- lib/online_migrations/background_migrations/delete_associated_records.rb
|
51
|
+
- lib/online_migrations/background_migrations/delete_orphaned_records.rb
|
50
52
|
- lib/online_migrations/background_migrations/migration.rb
|
51
53
|
- lib/online_migrations/background_migrations/migration_helpers.rb
|
52
54
|
- lib/online_migrations/background_migrations/migration_job.rb
|
@@ -54,6 +56,7 @@ files:
|
|
54
56
|
- lib/online_migrations/background_migrations/migration_job_status_validator.rb
|
55
57
|
- lib/online_migrations/background_migrations/migration_runner.rb
|
56
58
|
- lib/online_migrations/background_migrations/migration_status_validator.rb
|
59
|
+
- lib/online_migrations/background_migrations/perform_action_on_relation.rb
|
57
60
|
- lib/online_migrations/background_migrations/reset_counters.rb
|
58
61
|
- lib/online_migrations/background_migrations/scheduler.rb
|
59
62
|
- lib/online_migrations/batch_iterator.rb
|
@@ -98,7 +101,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
98
101
|
- !ruby/object:Gem::Version
|
99
102
|
version: '0'
|
100
103
|
requirements: []
|
101
|
-
rubygems_version: 3.
|
104
|
+
rubygems_version: 3.3.7
|
102
105
|
signing_key:
|
103
106
|
specification_version: 4
|
104
107
|
summary: Catch unsafe PostgreSQL migrations in development and run them easier in
|