online_migrations 0.4.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/BACKGROUND_MIGRATIONS.md +6 -5
- data/CHANGELOG.md +81 -0
- data/README.md +1 -1
- data/lib/online_migrations/background_migrations/delete_associated_records.rb +28 -0
- data/lib/online_migrations/background_migrations/delete_orphaned_records.rb +45 -0
- data/lib/online_migrations/background_migrations/migration_helpers.rb +113 -4
- data/lib/online_migrations/background_migrations/perform_action_on_relation.rb +33 -0
- data/lib/online_migrations/change_column_type_helpers.rb +4 -4
- data/lib/online_migrations/command_checker.rb +26 -16
- data/lib/online_migrations/command_recorder.rb +7 -2
- data/lib/online_migrations/error_messages.rb +6 -1
- data/lib/online_migrations/schema_statements.rb +8 -17
- data/lib/online_migrations/verbose_sql_logs.rb +2 -2
- data/lib/online_migrations/version.rb +1 -1
- data/lib/online_migrations.rb +3 -0
- metadata +6 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 95f3b31c9fe8edb868fade7dbdaed0ebf78d53da7a1ff52786a748663bc93bf5
|
|
4
|
+
data.tar.gz: 94c2fed042d39993d85f6641db3b4a86c3a360b3c858f48aeb2da052bb2c52cc
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: f844aa5e502a91739923039a8fb84d10ae9682404bd14bd4b92352400c6dae82963b21166178c72c47ea4ecbd0792b6f46da151d92ce205e217a5788c9a8b27a
|
|
7
|
+
data.tar.gz: 2db40b6b3cf81f923251f16a023aedbb18f8f24d28abb9e6b638090030c013c24dd6a41942df3e4f4cd1264c84c997eaaeb6adcb1c2d8d8844bc4934406f9ebc
|
data/BACKGROUND_MIGRATIONS.md
CHANGED
|
@@ -116,9 +116,12 @@ enqueue_background_migration("MyMigrationWithArgs", arg1, arg2, ...)
|
|
|
116
116
|
|
|
117
117
|
## Predefined background migrations
|
|
118
118
|
|
|
119
|
-
* `BackfillColumn` - backfills column(s) with scalar values
|
|
120
|
-
* `CopyColumn` - copies data from one column(s) to other(s)
|
|
121
|
-
* `
|
|
119
|
+
* `BackfillColumn` - backfills column(s) with scalar values (enqueue using `backfill_column_in_background`)
|
|
120
|
+
* `CopyColumn` - copies data from one column(s) to other(s) (enqueue using `copy_column_in_background`)
|
|
121
|
+
* `DeleteAssociatedRecords` - deletes records associated with a parent object (enqueue using `delete_associated_records_in_background`)
|
|
122
|
+
* `DeleteOrphanedRecords` - deletes records with one or more missing relations (enqueue using `delete_orphaned_records_in_background`)
|
|
123
|
+
* `PerformActionOnRelation` - performs specific action on a relation or indvidual records (enqueue using `perform_action_on_relation_in_background`)
|
|
124
|
+
* `ResetCounters` - resets one or more counter caches to their correct value (enqueue using `reset_counters_in_background`)
|
|
122
125
|
|
|
123
126
|
## Testing
|
|
124
127
|
|
|
@@ -262,8 +265,6 @@ The error handler should be a lambda that accepts 2 arguments:
|
|
|
262
265
|
|
|
263
266
|
* `error`: The exception that was raised.
|
|
264
267
|
* `errored_job`: An `OnlineMigrations::BackgroundMigrations::MigrationJob` object that represents a failed batch.
|
|
265
|
-
* `errored_element`: The `OnlineMigrations::BackgroundMigrations::MigrationJob` object representing a batch,
|
|
266
|
-
that was being processed when the Background Migration raised an exception.
|
|
267
268
|
|
|
268
269
|
### Customizing the background migrations module
|
|
269
270
|
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,86 @@
|
|
|
1
1
|
## master (unreleased)
|
|
2
2
|
|
|
3
|
+
## 0.5.1 (2022-07-19)
|
|
4
|
+
|
|
5
|
+
- Raise for possible index corruption in all environments (previously, the check was made only
|
|
6
|
+
in the production environment)
|
|
7
|
+
|
|
8
|
+
## 0.5.0 (2022-06-23)
|
|
9
|
+
|
|
10
|
+
- Added check for index corruption with PostgreSQL 14.0 to 14.3
|
|
11
|
+
|
|
12
|
+
- No need to separately remove indexes when removing a column from the small table
|
|
13
|
+
|
|
14
|
+
- Add ability to perform specific action on a relation or individual records using background migrations
|
|
15
|
+
|
|
16
|
+
Example, assuming you have lots and lots of fraud likes:
|
|
17
|
+
|
|
18
|
+
```ruby
|
|
19
|
+
class DeleteFraudLikes < ActiveRecord::Migration[7.0]
|
|
20
|
+
def up
|
|
21
|
+
perform_action_on_relation_in_background("Like", { fraud: true }, :delete_all)
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
Example, assuming you added a new column to the users and want to populate it:
|
|
27
|
+
|
|
28
|
+
```ruby
|
|
29
|
+
class User < ApplicationRecord
|
|
30
|
+
def generate_invite_token
|
|
31
|
+
self.invite_token = # some complex logic
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
perform_action_on_relation_in_background("User", { invite_token: nil }, :generate_invite_token)
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
You can use `delete_all`/`destroy_all`/`update_all` for the whole relation or run specific methods on individual records.
|
|
39
|
+
|
|
40
|
+
- Add ability to delete records associated with a parent object using background migrations
|
|
41
|
+
|
|
42
|
+
```ruby
|
|
43
|
+
class Link < ActiveRecord::Base
|
|
44
|
+
has_many :clicks
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
class Click < ActiveRecord::Base
|
|
48
|
+
belongs_to :link
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
class DeleteSomeLinkClicks < ActiveRecord::Migration[7.0]
|
|
52
|
+
def up
|
|
53
|
+
some_link = ...
|
|
54
|
+
delete_associated_records_in_background("Link", some_link.id, :clicks)
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
- Add ability to delete orphaned records using background migrations
|
|
60
|
+
|
|
61
|
+
```ruby
|
|
62
|
+
class User < ApplicationRecord
|
|
63
|
+
has_many :posts
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
class Post < ApplicationRecord
|
|
67
|
+
belongs_to :author, class_name: 'User'
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
class DeleteOrphanedPosts < ActiveRecord::Migration[7.0]
|
|
71
|
+
def up
|
|
72
|
+
delete_orphaned_records_in_background("Post", :author)
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
## 0.4.1 (2022-03-21)
|
|
78
|
+
|
|
79
|
+
- Fix missing options in suggested command for columns removal
|
|
80
|
+
- Fix retrieving raw postgresql connection
|
|
81
|
+
|
|
82
|
+
## 0.4.0 (2022-03-17)
|
|
83
|
+
|
|
3
84
|
- Lazy load this gem
|
|
4
85
|
|
|
5
86
|
- Add ability to reset counter caches using background migrations
|
data/README.md
CHANGED
|
@@ -351,7 +351,7 @@ A safer approach can be accomplished in several steps:
|
|
|
351
351
|
```ruby
|
|
352
352
|
class CleanupChangeFilesSizeType < ActiveRecord::Migration[7.0]
|
|
353
353
|
def up
|
|
354
|
-
|
|
354
|
+
cleanup_column_type_change :files, :size
|
|
355
355
|
end
|
|
356
356
|
|
|
357
357
|
def down
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module OnlineMigrations
|
|
4
|
+
module BackgroundMigrations
|
|
5
|
+
# @private
|
|
6
|
+
class DeleteAssociatedRecords < BackgroundMigration
|
|
7
|
+
attr_reader :record, :association
|
|
8
|
+
|
|
9
|
+
def initialize(model_name, record_id, association, _options = {})
|
|
10
|
+
model = Object.const_get(model_name, false)
|
|
11
|
+
@record = model.find(record_id)
|
|
12
|
+
@association = association
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def relation
|
|
16
|
+
unless @record.respond_to?(association)
|
|
17
|
+
raise ArgumentError, "'#{@record.class.name}' has no association called '#{association}'"
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
record.public_send(association)
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def process_batch(relation)
|
|
24
|
+
relation.delete_all(:delete_all)
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module OnlineMigrations
|
|
4
|
+
module BackgroundMigrations
|
|
5
|
+
# @private
|
|
6
|
+
class DeleteOrphanedRecords < BackgroundMigration
|
|
7
|
+
attr_reader :model, :associations
|
|
8
|
+
|
|
9
|
+
def initialize(model_name, associations, _options = {})
|
|
10
|
+
@model = Object.const_get(model_name, false)
|
|
11
|
+
@associations = associations.map(&:to_sym)
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def relation
|
|
15
|
+
# For ActiveRecord 6.1+ we can use `where.missing`
|
|
16
|
+
# https://github.com/rails/rails/pull/34727
|
|
17
|
+
associations.inject(model.unscoped) do |relation, association|
|
|
18
|
+
reflection = model.reflect_on_association(association)
|
|
19
|
+
unless reflection
|
|
20
|
+
raise ArgumentError, "'#{model.name}' has no association called '#{association}'"
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# left_joins was added in ActiveRecord 5.0 - https://github.com/rails/rails/pull/12071
|
|
24
|
+
relation
|
|
25
|
+
.left_joins(association)
|
|
26
|
+
.where(reflection.table_name => { reflection.association_primary_key => nil })
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def process_batch(relation)
|
|
31
|
+
if Utils.ar_version > 5.0
|
|
32
|
+
relation.delete_all
|
|
33
|
+
else
|
|
34
|
+
# Older ActiveRecord generates incorrect query when running delete_all
|
|
35
|
+
primary_key = model.primary_key
|
|
36
|
+
model.unscoped.where(primary_key => relation.select(primary_key)).delete_all
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def count
|
|
41
|
+
Utils.estimated_count(model.connection, model.table_name)
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
@@ -21,7 +21,7 @@ module OnlineMigrations
|
|
|
21
21
|
# @example Additional background migration options
|
|
22
22
|
# backfill_column_in_background(:users, :admin, false, batch_size: 10_000)
|
|
23
23
|
#
|
|
24
|
-
# @note This method is better suited for
|
|
24
|
+
# @note This method is better suited for large tables (10/100s of millions of records).
|
|
25
25
|
# For smaller tables it is probably better and easier to use more flexible `update_column_in_batches`.
|
|
26
26
|
#
|
|
27
27
|
# @note Consider `backfill_columns_in_background` when backfilling multiple columns
|
|
@@ -76,7 +76,7 @@ module OnlineMigrations
|
|
|
76
76
|
# @example Additional background migration options
|
|
77
77
|
# backfill_column_for_type_change_in_background(:files, :size, batch_size: 10_000)
|
|
78
78
|
#
|
|
79
|
-
# @note This method is better suited for
|
|
79
|
+
# @note This method is better suited for large tables (10/100s of millions of records).
|
|
80
80
|
# For smaller tables it is probably better and easier to use more flexible `backfill_column_for_type_change`.
|
|
81
81
|
#
|
|
82
82
|
def backfill_column_for_type_change_in_background(table_name, column_name, model_name: nil,
|
|
@@ -131,7 +131,7 @@ module OnlineMigrations
|
|
|
131
131
|
# @example
|
|
132
132
|
# copy_column_in_background(:users, :id, :id_for_type_change)
|
|
133
133
|
#
|
|
134
|
-
# @note This method is better suited for
|
|
134
|
+
# @note This method is better suited for large tables (10/100s of millions of records).
|
|
135
135
|
# For smaller tables it is probably better and easier to use more flexible `update_column_in_batches`.
|
|
136
136
|
#
|
|
137
137
|
def copy_column_in_background(table_name, copy_from, copy_to, model_name: nil, type_cast_function: nil, **options)
|
|
@@ -190,7 +190,7 @@ module OnlineMigrations
|
|
|
190
190
|
#
|
|
191
191
|
# @see https://api.rubyonrails.org/classes/ActiveRecord/CounterCache/ClassMethods.html#method-i-reset_counters
|
|
192
192
|
#
|
|
193
|
-
# @note This method is better suited for
|
|
193
|
+
# @note This method is better suited for large tables (10/100s of millions of records).
|
|
194
194
|
# For smaller tables it is probably better and easier to use `reset_counters` from the ActiveRecord.
|
|
195
195
|
#
|
|
196
196
|
def reset_counters_in_background(model_name, *counters, touch: nil, **options)
|
|
@@ -205,6 +205,115 @@ module OnlineMigrations
|
|
|
205
205
|
)
|
|
206
206
|
end
|
|
207
207
|
|
|
208
|
+
# Deletes records with one or more missing relations using background migrations.
|
|
209
|
+
# This is useful when some referential integrity in the database is broken and
|
|
210
|
+
# you want to delete orphaned records.
|
|
211
|
+
#
|
|
212
|
+
# @param model_name [String]
|
|
213
|
+
# @param associations [Array]
|
|
214
|
+
# @param options [Hash] used to control the behavior of background migration.
|
|
215
|
+
# See `#enqueue_background_migration`
|
|
216
|
+
#
|
|
217
|
+
# @return [OnlineMigrations::BackgroundMigrations::Migration]
|
|
218
|
+
#
|
|
219
|
+
# @example
|
|
220
|
+
# delete_orphaned_records_in_background("Post", :author)
|
|
221
|
+
#
|
|
222
|
+
# @note This method is better suited for large tables (10/100s of millions of records).
|
|
223
|
+
# For smaller tables it is probably better and easier to directly find and delete orpahed records.
|
|
224
|
+
#
|
|
225
|
+
def delete_orphaned_records_in_background(model_name, *associations, **options)
|
|
226
|
+
if Utils.ar_version <= 4.2
|
|
227
|
+
raise "#{__method__} does not support ActiveRecord <= 4.2 yet"
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
model_name = model_name.name if model_name.is_a?(Class)
|
|
231
|
+
|
|
232
|
+
enqueue_background_migration(
|
|
233
|
+
"DeleteOrphanedRecords",
|
|
234
|
+
model_name,
|
|
235
|
+
associations,
|
|
236
|
+
**options
|
|
237
|
+
)
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
# Deletes associated records for a specific parent record using background migrations.
|
|
241
|
+
# This is useful when you are planning to remove a parent object (user, account etc)
|
|
242
|
+
# and needs to remove lots of its associated objects.
|
|
243
|
+
#
|
|
244
|
+
# @param model_name [String]
|
|
245
|
+
# @param record_id [Integer, String] parent record primary key's value
|
|
246
|
+
# @param association [String, Symbol] association name for which records will be removed
|
|
247
|
+
# @param options [Hash] used to control the behavior of background migration.
|
|
248
|
+
# See `#enqueue_background_migration`
|
|
249
|
+
#
|
|
250
|
+
# @return [OnlineMigrations::BackgroundMigrations::Migration]
|
|
251
|
+
#
|
|
252
|
+
# @example
|
|
253
|
+
# delete_associated_records_in_background("Link", 1, :clicks)
|
|
254
|
+
#
|
|
255
|
+
# @note This method is better suited for large tables (10/100s of millions of records).
|
|
256
|
+
# For smaller tables it is probably better and easier to directly delete associated records.
|
|
257
|
+
#
|
|
258
|
+
def delete_associated_records_in_background(model_name, record_id, association, **options)
|
|
259
|
+
model_name = model_name.name if model_name.is_a?(Class)
|
|
260
|
+
|
|
261
|
+
enqueue_background_migration(
|
|
262
|
+
"DeleteAssociatedRecords",
|
|
263
|
+
model_name,
|
|
264
|
+
record_id,
|
|
265
|
+
association,
|
|
266
|
+
**options
|
|
267
|
+
)
|
|
268
|
+
end
|
|
269
|
+
|
|
270
|
+
# Performs specific action on a relation or individual records.
|
|
271
|
+
# This is useful when you want to delete/destroy/update/etc records based on some conditions.
|
|
272
|
+
#
|
|
273
|
+
# @param model_name [String]
|
|
274
|
+
# @param conditions [Array, Hash, String] conditions to filter the relation
|
|
275
|
+
# @param action [String, Symbol] action to perform on the relation or individual records.
|
|
276
|
+
# Relation-wide available actions: `:delete_all`, `:destroy_all`, and `:update_all`.
|
|
277
|
+
# @param updates [Hash] updates to perform when `action` is set to `:update_all`
|
|
278
|
+
# @param options [Hash] used to control the behavior of background migration.
|
|
279
|
+
# See `#enqueue_background_migration`
|
|
280
|
+
#
|
|
281
|
+
# @return [OnlineMigrations::BackgroundMigrations::Migration]
|
|
282
|
+
#
|
|
283
|
+
# @example Delete records
|
|
284
|
+
# perform_action_on_relation_in_background("User", { banned: true }, :delete_all)
|
|
285
|
+
#
|
|
286
|
+
# @example Destroy records
|
|
287
|
+
# perform_action_on_relation_in_background("User", { banned: true }, :destroy_all)
|
|
288
|
+
#
|
|
289
|
+
# @example Update records
|
|
290
|
+
# perform_action_on_relation_in_background("User", { banned: nil }, :update_all, updates: { banned: false })
|
|
291
|
+
#
|
|
292
|
+
# @example Perform custom method on individual records
|
|
293
|
+
# class User < ApplicationRecord
|
|
294
|
+
# def generate_invite_token
|
|
295
|
+
# self.invite_token = # some complex logic
|
|
296
|
+
# end
|
|
297
|
+
# end
|
|
298
|
+
#
|
|
299
|
+
# perform_action_on_relation_in_background("User", { invite_token: nil }, :generate_invite_token)
|
|
300
|
+
#
|
|
301
|
+
# @note This method is better suited for large tables (10/100s of millions of records).
|
|
302
|
+
# For smaller tables it is probably better and easier to directly delete associated records.
|
|
303
|
+
#
|
|
304
|
+
def perform_action_on_relation_in_background(model_name, conditions, action, updates: nil, **options)
|
|
305
|
+
model_name = model_name.name if model_name.is_a?(Class)
|
|
306
|
+
|
|
307
|
+
enqueue_background_migration(
|
|
308
|
+
"PerformActionOnRelation",
|
|
309
|
+
model_name,
|
|
310
|
+
conditions,
|
|
311
|
+
action,
|
|
312
|
+
{ updates: updates },
|
|
313
|
+
**options
|
|
314
|
+
)
|
|
315
|
+
end
|
|
316
|
+
|
|
208
317
|
# Creates a background migration for the given job class name.
|
|
209
318
|
#
|
|
210
319
|
# A background migration runs one job at a time, computing the bounds of the next batch
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module OnlineMigrations
|
|
4
|
+
module BackgroundMigrations
|
|
5
|
+
# @private
|
|
6
|
+
class PerformActionOnRelation < BackgroundMigration
|
|
7
|
+
attr_reader :model, :conditions, :action, :options
|
|
8
|
+
|
|
9
|
+
def initialize(model_name, conditions, action, options = {})
|
|
10
|
+
@model = Object.const_get(model_name, false)
|
|
11
|
+
@conditions = conditions
|
|
12
|
+
@action = action.to_sym
|
|
13
|
+
@options = options.symbolize_keys
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def relation
|
|
17
|
+
model.unscoped.where(conditions)
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def process_batch(relation)
|
|
21
|
+
case action
|
|
22
|
+
when :update_all
|
|
23
|
+
updates = options.fetch(:updates)
|
|
24
|
+
relation.public_send(action, updates)
|
|
25
|
+
when :delete_all, :destroy_all
|
|
26
|
+
relation.public_send(action)
|
|
27
|
+
else
|
|
28
|
+
relation.each(&action)
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
@@ -100,7 +100,7 @@ module OnlineMigrations
|
|
|
100
100
|
end.to_h
|
|
101
101
|
|
|
102
102
|
if (extra_keys = (options.keys - conversions.keys)).any?
|
|
103
|
-
raise ArgumentError, "Options has unknown keys: #{extra_keys.map(&:inspect).join(', ')}. "\
|
|
103
|
+
raise ArgumentError, "Options has unknown keys: #{extra_keys.map(&:inspect).join(', ')}. " \
|
|
104
104
|
"Can contain only column names: #{conversions.keys.map(&:inspect).join(', ')}."
|
|
105
105
|
end
|
|
106
106
|
|
|
@@ -110,7 +110,7 @@ module OnlineMigrations
|
|
|
110
110
|
column_options = options[column_name] || {}
|
|
111
111
|
tmp_column_name = conversions[column_name]
|
|
112
112
|
|
|
113
|
-
if
|
|
113
|
+
if raw_connection.server_version >= 11_00_00 &&
|
|
114
114
|
primary_key(table_name) == column_name.to_s && old_col.type == :integer
|
|
115
115
|
# If the column to be converted is a Primary Key, set it to
|
|
116
116
|
# `NOT NULL DEFAULT 0` and we'll copy the correct values when backfilling.
|
|
@@ -173,7 +173,7 @@ module OnlineMigrations
|
|
|
173
173
|
# backfill_column_for_type_change(:files, :size, batch_size: 10_000)
|
|
174
174
|
#
|
|
175
175
|
# @note This method should not be run within a transaction
|
|
176
|
-
# @note For
|
|
176
|
+
# @note For large tables (10/100s of millions of records)
|
|
177
177
|
# it is recommended to use `backfill_column_for_type_change_in_background`.
|
|
178
178
|
#
|
|
179
179
|
def backfill_column_for_type_change(table_name, column_name, type_cast_function: nil, **options)
|
|
@@ -400,7 +400,7 @@ module OnlineMigrations
|
|
|
400
400
|
|
|
401
401
|
# This is necessary as we can't properly rename indexes such as "taggings_idx".
|
|
402
402
|
unless index.name.include?(from_column)
|
|
403
|
-
raise "The index #{index.name} can not be copied as it does not "\
|
|
403
|
+
raise "The index #{index.name} can not be copied as it does not " \
|
|
404
404
|
"mention the old column. You have to rename this index manually first."
|
|
405
405
|
end
|
|
406
406
|
|
|
@@ -316,7 +316,7 @@ module OnlineMigrations
|
|
|
316
316
|
initialize_change_code: command_str(:initialize_column_type_change, table_name, column_name, type, **options),
|
|
317
317
|
backfill_code: command_str(:backfill_column_for_type_change, table_name, column_name, **options),
|
|
318
318
|
finalize_code: command_str(:finalize_column_type_change, table_name, column_name),
|
|
319
|
-
cleanup_code: command_str(:
|
|
319
|
+
cleanup_code: command_str(:cleanup_column_type_change, table_name, column_name),
|
|
320
320
|
cleanup_down_code: command_str(:initialize_column_type_change, table_name, column_name, existing_type)
|
|
321
321
|
end
|
|
322
322
|
end
|
|
@@ -381,9 +381,10 @@ module OnlineMigrations
|
|
|
381
381
|
raise_error :remove_column,
|
|
382
382
|
model: table_name.to_s.classify,
|
|
383
383
|
columns: columns.inspect,
|
|
384
|
-
command: command_str(command, *args),
|
|
384
|
+
command: command_str(command, *args, options),
|
|
385
385
|
table_name: table_name.inspect,
|
|
386
|
-
indexes: indexes.map { |i| i.name.to_sym.inspect }
|
|
386
|
+
indexes: indexes.map { |i| i.name.to_sym.inspect },
|
|
387
|
+
small_table: small_table?(table_name)
|
|
387
388
|
end
|
|
388
389
|
end
|
|
389
390
|
|
|
@@ -450,6 +451,10 @@ module OnlineMigrations
|
|
|
450
451
|
command: command_str(:add_index, table_name, column_name, **options.merge(algorithm: :concurrently))
|
|
451
452
|
end
|
|
452
453
|
|
|
454
|
+
if options[:algorithm] == :concurrently && index_corruption?
|
|
455
|
+
raise_error :add_index_corruption
|
|
456
|
+
end
|
|
457
|
+
|
|
453
458
|
if @removed_indexes.any?
|
|
454
459
|
index = IndexDefinition.new(table: table_name, columns: column_name, **options)
|
|
455
460
|
existing_indexes = connection.indexes(table_name)
|
|
@@ -576,30 +581,30 @@ module OnlineMigrations
|
|
|
576
581
|
end
|
|
577
582
|
|
|
578
583
|
def new_or_small_table?(table_name)
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
new_table?(table_name) ||
|
|
582
|
-
small_tables.include?(table_name.to_s)
|
|
584
|
+
new_table?(table_name) || small_table?(table_name)
|
|
583
585
|
end
|
|
584
586
|
|
|
585
587
|
def new_table?(table_name)
|
|
586
588
|
@new_tables.include?(table_name.to_s)
|
|
587
589
|
end
|
|
588
590
|
|
|
591
|
+
def small_table?(table_name)
|
|
592
|
+
OnlineMigrations.config.small_tables.include?(table_name.to_s)
|
|
593
|
+
end
|
|
594
|
+
|
|
589
595
|
def postgresql_version
|
|
590
596
|
version =
|
|
591
597
|
if Utils.developer_env? && (target_version = OnlineMigrations.config.target_version)
|
|
592
598
|
target_version.to_s
|
|
593
599
|
else
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
database_version
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
major
|
|
602
|
-
"#{major}.#{minor}.#{patch}"
|
|
600
|
+
database_version = connection.select_value("SHOW server_version_num").to_i
|
|
601
|
+
major = database_version / 10000
|
|
602
|
+
if database_version >= 100000
|
|
603
|
+
minor = database_version % 10000
|
|
604
|
+
else
|
|
605
|
+
minor = (database_version % 10000) / 100
|
|
606
|
+
end
|
|
607
|
+
"#{major}.#{minor}"
|
|
603
608
|
end
|
|
604
609
|
|
|
605
610
|
Gem::Version.new(version)
|
|
@@ -737,6 +742,11 @@ module OnlineMigrations
|
|
|
737
742
|
[table1.to_s, table2.to_s].sort.join("\0").gsub(/^(.*_)(.+)\0\1(.+)/, '\1\2_\3').tr("\0", "_")
|
|
738
743
|
end
|
|
739
744
|
|
|
745
|
+
def index_corruption?
|
|
746
|
+
postgresql_version >= Gem::Version.new("14.0") &&
|
|
747
|
+
postgresql_version < Gem::Version.new("14.4")
|
|
748
|
+
end
|
|
749
|
+
|
|
740
750
|
def run_custom_checks(method, args)
|
|
741
751
|
OnlineMigrations.config.checks.each do |options, check|
|
|
742
752
|
if !options[:start_after] || version > options[:start_after]
|
|
@@ -16,7 +16,9 @@ module OnlineMigrations
|
|
|
16
16
|
:swap_column_names,
|
|
17
17
|
:add_column_with_default,
|
|
18
18
|
:add_not_null_constraint,
|
|
19
|
+
:remove_not_null_constraint,
|
|
19
20
|
:add_text_limit_constraint,
|
|
21
|
+
:remove_text_limit_constraint,
|
|
20
22
|
:add_reference_concurrently,
|
|
21
23
|
:change_column_type_in_background,
|
|
22
24
|
:enqueue_background_migration,
|
|
@@ -127,11 +129,14 @@ module OnlineMigrations
|
|
|
127
129
|
end
|
|
128
130
|
|
|
129
131
|
def invert_remove_text_limit_constraint(args)
|
|
130
|
-
|
|
132
|
+
options = args.extract_options!
|
|
133
|
+
table_name, column, limit = args
|
|
134
|
+
|
|
135
|
+
unless limit
|
|
131
136
|
raise ActiveRecord::IrreversibleMigration, "remove_text_limit_constraint is only reversible if given a limit."
|
|
132
137
|
end
|
|
133
138
|
|
|
134
|
-
|
|
139
|
+
[:add_text_limit_constraint, [table_name, column, limit, **options]]
|
|
135
140
|
end
|
|
136
141
|
end
|
|
137
142
|
end
|
|
@@ -233,7 +233,7 @@ class <%= migration_name %> < <%= migration_parent %>
|
|
|
233
233
|
end",
|
|
234
234
|
|
|
235
235
|
remove_column:
|
|
236
|
-
"<% if indexes.any? %>
|
|
236
|
+
"<% if !small_table && indexes.any? %>
|
|
237
237
|
Removing a column will automatically remove all of the indexes that involved the removed column.
|
|
238
238
|
But the indexes would be removed non-concurrently, so you need to safely remove the indexes first:
|
|
239
239
|
|
|
@@ -333,6 +333,11 @@ class <%= migration_name %> < <%= migration_parent %>
|
|
|
333
333
|
end
|
|
334
334
|
end",
|
|
335
335
|
|
|
336
|
+
add_index_corruption:
|
|
337
|
+
"Adding an index concurrently can cause silent data corruption in PostgreSQL 14.0 to 14.3.
|
|
338
|
+
Upgrade PostgreSQL before adding new indexes, or wrap this step in a safety_assured { ... }
|
|
339
|
+
block to accept the risk.",
|
|
340
|
+
|
|
336
341
|
remove_index:
|
|
337
342
|
"Removing an index non-concurrently blocks writes. Instead, use:
|
|
338
343
|
|
|
@@ -48,7 +48,7 @@ module OnlineMigrations
|
|
|
48
48
|
# @note This method should not be run within a transaction
|
|
49
49
|
# @note Consider `update_columns_in_batches` when updating multiple columns
|
|
50
50
|
# to avoid rewriting the table multiple times.
|
|
51
|
-
# @note For
|
|
51
|
+
# @note For large tables (10/100s of millions of records)
|
|
52
52
|
# you may consider using `backfill_column_in_background` or `copy_column_in_background`.
|
|
53
53
|
#
|
|
54
54
|
def update_column_in_batches(table_name, column_name, value, **options, &block)
|
|
@@ -361,7 +361,7 @@ module OnlineMigrations
|
|
|
361
361
|
# These steps ensure a column can be added to a large and commonly used table
|
|
362
362
|
# without locking the entire table for the duration of the table modification.
|
|
363
363
|
#
|
|
364
|
-
# For
|
|
364
|
+
# For large tables (10/100s of millions of records) you may consider implementing
|
|
365
365
|
# the steps from this helper method yourself as a separate migrations, replacing step #3
|
|
366
366
|
# with the help of background migrations (see `backfill_column_in_background`).
|
|
367
367
|
#
|
|
@@ -396,7 +396,7 @@ module OnlineMigrations
|
|
|
396
396
|
raise ArgumentError, "Expressions as default are not supported"
|
|
397
397
|
end
|
|
398
398
|
|
|
399
|
-
if
|
|
399
|
+
if raw_connection.server_version >= 11_00_00 && !Utils.volatile_default?(self, type, default)
|
|
400
400
|
add_column(table_name, column_name, type, **options)
|
|
401
401
|
else
|
|
402
402
|
__ensure_not_in_transaction!
|
|
@@ -404,7 +404,7 @@ module OnlineMigrations
|
|
|
404
404
|
batch_options = options.extract!(:batch_size, :batch_column_name, :progress, :pause_ms)
|
|
405
405
|
|
|
406
406
|
if column_exists?(table_name, column_name)
|
|
407
|
-
Utils.say("Column was not created because it already exists (this may be due to an aborted migration "\
|
|
407
|
+
Utils.say("Column was not created because it already exists (this may be due to an aborted migration " \
|
|
408
408
|
"or similar) table_name: #{table_name}, column_name: #{column_name}")
|
|
409
409
|
else
|
|
410
410
|
transaction do
|
|
@@ -422,7 +422,7 @@ module OnlineMigrations
|
|
|
422
422
|
add_not_null_constraint(table_name, column_name, validate: false)
|
|
423
423
|
validate_not_null_constraint(table_name, column_name)
|
|
424
424
|
|
|
425
|
-
if
|
|
425
|
+
if raw_connection.server_version >= 12_00_00
|
|
426
426
|
# In PostgreSQL 12+ it is safe to "promote" a CHECK constraint to `NOT NULL` for the column
|
|
427
427
|
change_column_null(table_name, column_name, false)
|
|
428
428
|
remove_not_null_constraint(table_name, column_name)
|
|
@@ -655,7 +655,7 @@ module OnlineMigrations
|
|
|
655
655
|
schema = __schema_for_table(table_name)
|
|
656
656
|
|
|
657
657
|
if __index_valid?(index_name, schema: schema)
|
|
658
|
-
Utils.say("Index was not created because it already exists (this may be due to an aborted migration "\
|
|
658
|
+
Utils.say("Index was not created because it already exists (this may be due to an aborted migration " \
|
|
659
659
|
"or similar): table_name: #{table_name}, column_name: #{column_name}")
|
|
660
660
|
return
|
|
661
661
|
else
|
|
@@ -699,7 +699,7 @@ module OnlineMigrations
|
|
|
699
699
|
end
|
|
700
700
|
end
|
|
701
701
|
else
|
|
702
|
-
Utils.say("Index was not removed because it does not exist (this may be due to an aborted migration "\
|
|
702
|
+
Utils.say("Index was not removed because it does not exist (this may be due to an aborted migration " \
|
|
703
703
|
"or similar): table_name: #{table_name}, column_name: #{column_names}")
|
|
704
704
|
end
|
|
705
705
|
end
|
|
@@ -768,7 +768,7 @@ module OnlineMigrations
|
|
|
768
768
|
constraint_name = __check_constraint_name(table_name, expression: expression, **options)
|
|
769
769
|
|
|
770
770
|
if __check_constraint_exists?(table_name, constraint_name)
|
|
771
|
-
Utils.say("Check constraint was not created because it already exists (this may be due to an aborted migration "\
|
|
771
|
+
Utils.say("Check constraint was not created because it already exists (this may be due to an aborted migration " \
|
|
772
772
|
"or similar) table_name: #{table_name}, expression: #{expression}, constraint name: #{constraint_name}")
|
|
773
773
|
else
|
|
774
774
|
query = "ALTER TABLE #{table_name} ADD CONSTRAINT #{constraint_name} CHECK (#{expression})"
|
|
@@ -1046,14 +1046,5 @@ module OnlineMigrations
|
|
|
1046
1046
|
_, schema = table_name.to_s.split(".").reverse
|
|
1047
1047
|
schema ? quote(schema) : "current_schema()"
|
|
1048
1048
|
end
|
|
1049
|
-
|
|
1050
|
-
def __raw_connection
|
|
1051
|
-
# ActiveRecord > 7.0.2.2 (https://github.com/rails/rails/pull/44530)
|
|
1052
|
-
if defined?(@raw_connection)
|
|
1053
|
-
@raw_connection
|
|
1054
|
-
else
|
|
1055
|
-
@connection
|
|
1056
|
-
end
|
|
1057
|
-
end
|
|
1058
1049
|
end
|
|
1059
1050
|
end
|
|
@@ -26,7 +26,7 @@ module OnlineMigrations
|
|
|
26
26
|
|
|
27
27
|
private
|
|
28
28
|
def verbose_query_logs
|
|
29
|
-
if Utils.ar_version
|
|
29
|
+
if Utils.ar_version >= 7.0
|
|
30
30
|
ActiveRecord.verbose_query_logs
|
|
31
31
|
elsif Utils.ar_version >= 5.2
|
|
32
32
|
ActiveRecord::Base.verbose_query_logs
|
|
@@ -34,7 +34,7 @@ module OnlineMigrations
|
|
|
34
34
|
end
|
|
35
35
|
|
|
36
36
|
def set_verbose_query_logs(value) # rubocop:disable Naming/AccessorMethodName
|
|
37
|
-
if Utils.ar_version
|
|
37
|
+
if Utils.ar_version >= 7.0
|
|
38
38
|
ActiveRecord.verbose_query_logs = value
|
|
39
39
|
elsif Utils.ar_version >= 5.2
|
|
40
40
|
ActiveRecord::Base.verbose_query_logs = value
|
data/lib/online_migrations.rb
CHANGED
|
@@ -46,6 +46,9 @@ module OnlineMigrations
|
|
|
46
46
|
autoload :BackgroundMigrationClassValidator
|
|
47
47
|
autoload :BackfillColumn
|
|
48
48
|
autoload :CopyColumn
|
|
49
|
+
autoload :DeleteAssociatedRecords
|
|
50
|
+
autoload :DeleteOrphanedRecords
|
|
51
|
+
autoload :PerformActionOnRelation
|
|
49
52
|
autoload :ResetCounters
|
|
50
53
|
autoload :MigrationJob
|
|
51
54
|
autoload :Migration
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: online_migrations
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.5.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- fatkodima
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2022-
|
|
11
|
+
date: 2022-07-18 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activerecord
|
|
@@ -47,6 +47,8 @@ files:
|
|
|
47
47
|
- lib/online_migrations/background_migrations/background_migration_class_validator.rb
|
|
48
48
|
- lib/online_migrations/background_migrations/config.rb
|
|
49
49
|
- lib/online_migrations/background_migrations/copy_column.rb
|
|
50
|
+
- lib/online_migrations/background_migrations/delete_associated_records.rb
|
|
51
|
+
- lib/online_migrations/background_migrations/delete_orphaned_records.rb
|
|
50
52
|
- lib/online_migrations/background_migrations/migration.rb
|
|
51
53
|
- lib/online_migrations/background_migrations/migration_helpers.rb
|
|
52
54
|
- lib/online_migrations/background_migrations/migration_job.rb
|
|
@@ -54,6 +56,7 @@ files:
|
|
|
54
56
|
- lib/online_migrations/background_migrations/migration_job_status_validator.rb
|
|
55
57
|
- lib/online_migrations/background_migrations/migration_runner.rb
|
|
56
58
|
- lib/online_migrations/background_migrations/migration_status_validator.rb
|
|
59
|
+
- lib/online_migrations/background_migrations/perform_action_on_relation.rb
|
|
57
60
|
- lib/online_migrations/background_migrations/reset_counters.rb
|
|
58
61
|
- lib/online_migrations/background_migrations/scheduler.rb
|
|
59
62
|
- lib/online_migrations/batch_iterator.rb
|
|
@@ -98,7 +101,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
98
101
|
- !ruby/object:Gem::Version
|
|
99
102
|
version: '0'
|
|
100
103
|
requirements: []
|
|
101
|
-
rubygems_version: 3.
|
|
104
|
+
rubygems_version: 3.3.7
|
|
102
105
|
signing_key:
|
|
103
106
|
specification_version: 4
|
|
104
107
|
summary: Catch unsafe PostgreSQL migrations in development and run them easier in
|