online_migrations 0.11.1 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +24 -0
- data/docs/background_migrations.md +19 -7
- data/docs/configuring.md +21 -10
- data/lib/generators/online_migrations/background_migration_generator.rb +8 -5
- data/lib/generators/online_migrations/templates/initializer.rb.tt +12 -2
- data/lib/online_migrations/background_migrations/config.rb +6 -1
- data/lib/online_migrations/background_migrations/migration.rb +20 -36
- data/lib/online_migrations/background_migrations/migration_helpers.rb +37 -7
- data/lib/online_migrations/background_migrations/migration_runner.rb +4 -2
- data/lib/online_migrations/batch_iterator.rb +24 -15
- data/lib/online_migrations/change_column_type_helpers.rb +0 -7
- data/lib/online_migrations/command_checker.rb +11 -0
- data/lib/online_migrations/config.rb +7 -1
- data/lib/online_migrations/lock_retrier.rb +4 -6
- data/lib/online_migrations/schema_statements.rb +91 -45
- data/lib/online_migrations/utils.rb +15 -1
- data/lib/online_migrations/version.rb +1 -1
- data/lib/online_migrations.rb +21 -11
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 34bd3f3fc2c18bc49962183603c7f2c85e167d978eafd3fb840b639fa98de60d
|
4
|
+
data.tar.gz: 0ae0b82440ea7dcec1183c987395742fab0c93618b384f770d7acd3fecd41cdf
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 321b877ffe09ccf2edb94b43d1060f5ebc8d83c1c058a41dc1d248f4d52c161b5a4576186363349c6c027d20890375728590271d24bdb75647d0556a57202aa9
|
7
|
+
data.tar.gz: d6a25cf51e31b772e4556b18de85ed88c5019b0469a8d6299e955063042c2e939f4f3f228e05ce4fdc8a4060b2ee2f4160a1e5d477a8769439b103f03b103fb8
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,29 @@
|
|
1
1
|
## master (unreleased)
|
2
2
|
|
3
|
+
## 0.13.0 (2024-01-22)
|
4
|
+
|
5
|
+
- Add ability to configure the path where generated background migrations will be placed
|
6
|
+
|
7
|
+
```ruby
|
8
|
+
# It is placed in lib/ by default.
|
9
|
+
config.background_migrations.migrations_path = "app/lib"
|
10
|
+
```
|
11
|
+
|
12
|
+
- Reduce number of queries needed to calculate batch ranges for background migrations
|
13
|
+
- Fix `finalize_column_type_change` to not recreate already existing indexes on the temporary column
|
14
|
+
- Remove potentially heavy queries used to get the ranges of a background migration
|
15
|
+
|
16
|
+
## 0.12.0 (2024-01-18)
|
17
|
+
|
18
|
+
- Require passing model name for background migration helpers when using multiple databases
|
19
|
+
- Add `statement_timeout` configuration option
|
20
|
+
|
21
|
+
- Make `lock_timeout` argument optional for `config.lock_retrier`
|
22
|
+
|
23
|
+
This way, a default lock timeout value will be used (configured in `database.yml` or for the database user).
|
24
|
+
|
25
|
+
- Fix a bug that can lead to unfinished children of a sharded background migration
|
26
|
+
|
3
27
|
## 0.11.1 (2024-01-11)
|
4
28
|
|
5
29
|
- Fix calculation of batch ranges for sharded background migrations
|
@@ -255,7 +255,7 @@ Specify the throttle condition as a block:
|
|
255
255
|
```ruby
|
256
256
|
# config/initializers/online_migrations.rb
|
257
257
|
|
258
|
-
|
258
|
+
config.background_migrations.throttler = -> { DatabaseStatus.unhealthy? }
|
259
259
|
```
|
260
260
|
|
261
261
|
Note that it's up to you to define a throttling condition that makes sense for your app. For example, you can check various PostgreSQL metrics such as replication lag, DB threads, whether DB writes are available, etc.
|
@@ -269,7 +269,7 @@ If you want to integrate with an exception monitoring service (e.g. Bugsnag), yo
|
|
269
269
|
```ruby
|
270
270
|
# config/initializers/online_migrations.rb
|
271
271
|
|
272
|
-
|
272
|
+
config.background_migrations.error_handler = ->(error, errored_job) do
|
273
273
|
Bugsnag.notify(error) do |notification|
|
274
274
|
notification.add_metadata(:background_migration, { name: errored_job.migration_name })
|
275
275
|
end
|
@@ -281,22 +281,34 @@ The error handler should be a lambda that accepts 2 arguments:
|
|
281
281
|
* `error`: The exception that was raised.
|
282
282
|
* `errored_job`: An `OnlineMigrations::BackgroundMigrations::MigrationJob` object that represents a failed batch.
|
283
283
|
|
284
|
+
### Customizing the background migrations path
|
285
|
+
|
286
|
+
`OnlineMigrations.config.background_migrations.migrations_path` can be configured to define where generated background migrations will be placed.
|
287
|
+
|
288
|
+
```ruby
|
289
|
+
# config/initializers/online_migrations.rb
|
290
|
+
|
291
|
+
config.background_migrations.migrations_path = "app/lib"
|
292
|
+
```
|
293
|
+
|
294
|
+
If no value is specified, it will default to `"lib"`.
|
295
|
+
|
284
296
|
### Customizing the background migrations module
|
285
297
|
|
286
|
-
`
|
298
|
+
`config.background_migrations.migrations_module` can be configured to define the module in which
|
287
299
|
background migrations will be placed.
|
288
300
|
|
289
301
|
```ruby
|
290
302
|
# config/initializers/online_migrations.rb
|
291
303
|
|
292
|
-
|
304
|
+
config.background_migrations.migrations_module = "BackgroundMigrationsModule"
|
293
305
|
```
|
294
306
|
|
295
|
-
If no value is specified, it will default to `OnlineMigrations::BackgroundMigrations`.
|
307
|
+
If no value is specified, it will default to `"OnlineMigrations::BackgroundMigrations"`.
|
296
308
|
|
297
309
|
### Customizing the backtrace cleaner
|
298
310
|
|
299
|
-
`
|
311
|
+
`config.background_migrations.backtrace_cleaner` can be configured to specify a backtrace cleaner to use when a Background Migration errors and the backtrace is cleaned and persisted. An `ActiveSupport::BacktraceCleaner` should be used.
|
300
312
|
|
301
313
|
```ruby
|
302
314
|
# config/initializers/online_migrations.rb
|
@@ -304,7 +316,7 @@ If no value is specified, it will default to `OnlineMigrations::BackgroundMigrat
|
|
304
316
|
cleaner = ActiveSupport::BacktraceCleaner.new
|
305
317
|
cleaner.add_silencer { |line| line =~ /ignore_this_dir/ }
|
306
318
|
|
307
|
-
|
319
|
+
config.background_migrations.backtrace_cleaner = cleaner
|
308
320
|
```
|
309
321
|
|
310
322
|
If none is specified, the default `Rails.backtrace_cleaner` will be used to clean backtraces.
|
data/docs/configuring.md
CHANGED
@@ -59,22 +59,33 @@ Check the [source code](https://github.com/fatkodima/online_migrations/blob/mast
|
|
59
59
|
## Migration Timeouts
|
60
60
|
|
61
61
|
It’s extremely important to set a short lock timeout for migrations. This way, if a migration can't acquire a lock in a timely manner, other statements won't be stuck behind it.
|
62
|
+
We also recommend setting a long statement timeout so migrations can run for a while.
|
62
63
|
|
63
|
-
|
64
|
+
You can configure a statement timeout for migrations via:
|
64
65
|
|
65
|
-
```
|
66
|
-
|
67
|
-
connect_timeout: 5
|
68
|
-
variables:
|
69
|
-
lock_timeout: 10s
|
70
|
-
statement_timeout: 15s
|
66
|
+
```ruby
|
67
|
+
config.statement_timeout = 1.hour
|
71
68
|
```
|
72
69
|
|
70
|
+
and a lock timeout for migrations can be configured via the `lock_retrier`.
|
71
|
+
|
73
72
|
Or set the timeouts directly on the database user that runs migrations:
|
74
73
|
|
75
74
|
```sql
|
76
75
|
ALTER ROLE myuser SET lock_timeout = '10s';
|
77
|
-
ALTER ROLE myuser SET statement_timeout = '
|
76
|
+
ALTER ROLE myuser SET statement_timeout = '1h';
|
77
|
+
```
|
78
|
+
|
79
|
+
## App Timeouts
|
80
|
+
|
81
|
+
We recommend adding timeouts to `config/database.yml` to prevent connections from hanging and individual queries from taking up too many resources in controllers, jobs, the Rails console, and other places.
|
82
|
+
|
83
|
+
```yml
|
84
|
+
production:
|
85
|
+
connect_timeout: 5
|
86
|
+
variables:
|
87
|
+
lock_timeout: 10s
|
88
|
+
statement_timeout: 15s
|
78
89
|
```
|
79
90
|
|
80
91
|
## Lock Timeout Retries
|
@@ -86,7 +97,7 @@ config.lock_retrier = OnlineMigrations::ExponentialLockRetrier.new(
|
|
86
97
|
attempts: 30, # attempt 30 retries
|
87
98
|
base_delay: 0.01.seconds, # starting with delay of 10ms between each unsuccessful try, increasing exponentially
|
88
99
|
max_delay: 1.minute, # maximum delay is 1 minute
|
89
|
-
lock_timeout: 0.2.seconds # and 200ms set as lock timeout for each try
|
100
|
+
lock_timeout: 0.2.seconds # and 200ms set as lock timeout for each try. Remove this line to use a default lock timeout.
|
90
101
|
)
|
91
102
|
```
|
92
103
|
|
@@ -192,7 +203,7 @@ To enable verbose sql logs:
|
|
192
203
|
config.verbose_sql_logs = true
|
193
204
|
```
|
194
205
|
|
195
|
-
This feature is enabled by default in a production Rails
|
206
|
+
This feature is enabled by default in a staging and production Rails environments. You can override this setting via `ONLINE_MIGRATIONS_VERBOSE_SQL_LOGS` environment variable.
|
196
207
|
|
197
208
|
## Analyze Tables
|
198
209
|
|
@@ -9,8 +9,11 @@ module OnlineMigrations
|
|
9
9
|
desc "This generator creates a background migration file."
|
10
10
|
|
11
11
|
def create_background_migration_file
|
12
|
+
migrations_module_file_path = migrations_module.underscore
|
13
|
+
|
12
14
|
template_file = File.join(
|
13
|
-
|
15
|
+
config.migrations_path,
|
16
|
+
migrations_module_file_path,
|
14
17
|
class_path,
|
15
18
|
"#{file_name}.rb"
|
16
19
|
)
|
@@ -18,12 +21,12 @@ module OnlineMigrations
|
|
18
21
|
end
|
19
22
|
|
20
23
|
private
|
21
|
-
def
|
22
|
-
migrations_module
|
24
|
+
def migrations_module
|
25
|
+
config.migrations_module
|
23
26
|
end
|
24
27
|
|
25
|
-
def
|
26
|
-
OnlineMigrations.config.background_migrations
|
28
|
+
def config
|
29
|
+
OnlineMigrations.config.background_migrations
|
27
30
|
end
|
28
31
|
end
|
29
32
|
end
|
@@ -4,6 +4,9 @@ OnlineMigrations.configure do |config|
|
|
4
4
|
# Configure the migration version starting after which checks are performed.
|
5
5
|
# config.start_after = <%= start_after %>
|
6
6
|
|
7
|
+
# Configure statement timeout used for migrations.
|
8
|
+
config.statement_timeout = 1.hour
|
9
|
+
|
7
10
|
# Set the version of the production database so the right checks are run in development.
|
8
11
|
# config.target_version = 10
|
9
12
|
|
@@ -48,7 +51,7 @@ OnlineMigrations.configure do |config|
|
|
48
51
|
# a better grasp of what is going on for high-level statements like add_column_with_default.
|
49
52
|
#
|
50
53
|
# Note: It can be overridden by `ONLINE_MIGRATIONS_VERBOSE_SQL_LOGS` environment variable.
|
51
|
-
config.verbose_sql_logs = defined?(Rails) && Rails.env.production?
|
54
|
+
config.verbose_sql_logs = defined?(Rails.env) && (Rails.env.production? || Rails.env.staging?)
|
52
55
|
|
53
56
|
# Lock retries.
|
54
57
|
# Configure your custom lock retrier (see LockRetrier).
|
@@ -57,7 +60,7 @@ OnlineMigrations.configure do |config|
|
|
57
60
|
attempts: 30, # attempt 30 retries
|
58
61
|
base_delay: 0.01.seconds, # starting with delay of 10ms between each unsuccessful try, increasing exponentially
|
59
62
|
max_delay: 1.minute, # up to the maximum delay of 1 minute
|
60
|
-
lock_timeout: 0.2.seconds # and 200ms set as lock timeout for each try
|
63
|
+
lock_timeout: 0.2.seconds # and 200ms set as lock timeout for each try. Remove this line to use a default lock timeout.
|
61
64
|
)
|
62
65
|
|
63
66
|
# Configure tables that are in the process of being renamed.
|
@@ -75,6 +78,13 @@ OnlineMigrations.configure do |config|
|
|
75
78
|
# end
|
76
79
|
|
77
80
|
# ==> Background migrations configuration
|
81
|
+
|
82
|
+
# The path where generated background migrations will be placed.
|
83
|
+
# config.background_migrations.migrations_path = "lib"
|
84
|
+
|
85
|
+
# The module in which background migrations will be placed.
|
86
|
+
# config.background_migrations.migrations_module = "OnlineMigrations::BackgroundMigrations"
|
87
|
+
|
78
88
|
# The number of rows to process in a single background migration run.
|
79
89
|
# config.background_migrations.batch_size = 20_000
|
80
90
|
|
@@ -4,7 +4,11 @@ module OnlineMigrations
|
|
4
4
|
module BackgroundMigrations
|
5
5
|
# Class representing configuration options for background migrations.
|
6
6
|
class Config
|
7
|
-
# The
|
7
|
+
# The path where generated background migrations will be placed
|
8
|
+
# @return [String] defaults to "lib"
|
9
|
+
attr_accessor :migrations_path
|
10
|
+
|
11
|
+
# The module in which background migrations will be placed
|
8
12
|
# @return [String] defaults to "OnlineMigrations::BackgroundMigrations"
|
9
13
|
attr_accessor :migrations_module
|
10
14
|
|
@@ -75,6 +79,7 @@ module OnlineMigrations
|
|
75
79
|
attr_accessor :error_handler
|
76
80
|
|
77
81
|
def initialize
|
82
|
+
@migrations_path = "lib"
|
78
83
|
@migrations_module = "OnlineMigrations::BackgroundMigrations"
|
79
84
|
@batch_size = 20_000
|
80
85
|
@sub_batch_size = 1000
|
@@ -2,6 +2,11 @@
|
|
2
2
|
|
3
3
|
module OnlineMigrations
|
4
4
|
module BackgroundMigrations
|
5
|
+
# Class representing background data migration.
|
6
|
+
#
|
7
|
+
# @note The records of this class should not be created manually, but via
|
8
|
+
# `enqueue_background_migration` helper inside migrations.
|
9
|
+
#
|
5
10
|
class Migration < ApplicationRecord
|
6
11
|
STATUSES = [
|
7
12
|
:enqueued, # The migration has been enqueued by the user.
|
@@ -23,11 +28,13 @@ module OnlineMigrations
|
|
23
28
|
for_migration_name(migration_name).where("arguments = ?", arguments.to_json)
|
24
29
|
end
|
25
30
|
|
31
|
+
alias_attribute :name, :migration_name
|
32
|
+
|
26
33
|
enum status: STATUSES.index_with(&:to_s)
|
27
34
|
|
28
35
|
belongs_to :parent, class_name: name, optional: true
|
29
|
-
has_many :children, class_name: name, foreign_key: :parent_id
|
30
|
-
has_many :migration_jobs
|
36
|
+
has_many :children, class_name: name, foreign_key: :parent_id, dependent: :delete_all
|
37
|
+
has_many :migration_jobs, dependent: :delete_all
|
31
38
|
|
32
39
|
validates :migration_name, :batch_column_name, presence: true
|
33
40
|
|
@@ -47,7 +54,6 @@ module OnlineMigrations
|
|
47
54
|
validates_with MigrationStatusValidator, on: :update
|
48
55
|
|
49
56
|
before_validation :set_defaults
|
50
|
-
before_create :create_child_migrations, if: :composite?
|
51
57
|
before_update :copy_attributes_to_children, if: :composite?
|
52
58
|
|
53
59
|
# @private
|
@@ -57,8 +63,10 @@ module OnlineMigrations
|
|
57
63
|
end
|
58
64
|
|
59
65
|
def migration_name=(class_name)
|
66
|
+
class_name = class_name.name if class_name.is_a?(Class)
|
60
67
|
write_attribute(:migration_name, self.class.normalize_migration_name(class_name))
|
61
68
|
end
|
69
|
+
alias name= migration_name=
|
62
70
|
|
63
71
|
def completed?
|
64
72
|
succeeded? || failed?
|
@@ -189,10 +197,11 @@ module OnlineMigrations
|
|
189
197
|
|
190
198
|
on_shard do
|
191
199
|
# rubocop:disable Lint/UnreachableLoop
|
192
|
-
iterator.each_batch(of: batch_size, column: batch_column_name, start: next_min_value) do |relation|
|
193
|
-
|
194
|
-
|
195
|
-
|
200
|
+
iterator.each_batch(of: batch_size, column: batch_column_name, start: next_min_value) do |relation, min_value, max_value|
|
201
|
+
if max_value.nil?
|
202
|
+
max_value = relation.pick(relation.arel_table[batch_column_name].maximum)
|
203
|
+
end
|
204
|
+
batch_range = [min_value, max_value]
|
196
205
|
|
197
206
|
break
|
198
207
|
end
|
@@ -209,10 +218,6 @@ module OnlineMigrations
|
|
209
218
|
[min_value, max_value]
|
210
219
|
end
|
211
220
|
|
212
|
-
protected
|
213
|
-
attr_accessor :child
|
214
|
-
alias child? child
|
215
|
-
|
216
221
|
private
|
217
222
|
def validate_batch_column_values
|
218
223
|
if max_value.to_i < min_value.to_i
|
@@ -242,25 +247,16 @@ module OnlineMigrations
|
|
242
247
|
|
243
248
|
def set_defaults
|
244
249
|
if migration_relation.is_a?(ActiveRecord::Relation)
|
245
|
-
if !child?
|
246
|
-
shards = Utils.shard_names(migration_model)
|
247
|
-
self.composite = shards.size > 1
|
248
|
-
end
|
249
|
-
|
250
250
|
self.batch_column_name ||= migration_relation.primary_key
|
251
251
|
|
252
252
|
if composite?
|
253
253
|
self.min_value = self.max_value = self.rows_count = -1 # not relevant
|
254
254
|
else
|
255
255
|
on_shard do
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
if min_value.nil?
|
261
|
-
# integer IDs minimum value is 1
|
262
|
-
self.min_value = self.max_value = 1
|
263
|
-
end
|
256
|
+
# Getting exact min/max values can be a very heavy operation
|
257
|
+
# and is not needed practically.
|
258
|
+
self.min_value ||= 1
|
259
|
+
self.max_value ||= migration_model.unscoped.maximum(batch_column_name) || self.min_value
|
264
260
|
|
265
261
|
count = migration_object.count
|
266
262
|
self.rows_count = count if count != :no_count
|
@@ -276,18 +272,6 @@ module OnlineMigrations
|
|
276
272
|
self.batch_max_attempts ||= config.batch_max_attempts
|
277
273
|
end
|
278
274
|
|
279
|
-
def create_child_migrations
|
280
|
-
shards = Utils.shard_names(migration_model)
|
281
|
-
|
282
|
-
children = shards.map do |shard|
|
283
|
-
child = Migration.new(migration_name: migration_name, arguments: arguments, shard: shard)
|
284
|
-
child.child = true
|
285
|
-
child
|
286
|
-
end
|
287
|
-
|
288
|
-
self.children = children
|
289
|
-
end
|
290
|
-
|
291
275
|
def copy_attributes_to_children
|
292
276
|
attributes = [:batch_size, :sub_batch_size, :batch_pause, :sub_batch_pause_ms, :batch_max_attempts]
|
293
277
|
updates = {}
|
@@ -42,6 +42,10 @@ module OnlineMigrations
|
|
42
42
|
# @see #backfill_column_in_background
|
43
43
|
#
|
44
44
|
def backfill_columns_in_background(table_name, updates, model_name: nil, **options)
|
45
|
+
if model_name.nil? && Utils.multiple_databases?
|
46
|
+
raise ArgumentError, "You must pass a :model_name when using multiple databases."
|
47
|
+
end
|
48
|
+
|
45
49
|
model_name = model_name.name if model_name.is_a?(Class)
|
46
50
|
|
47
51
|
enqueue_background_migration(
|
@@ -99,6 +103,10 @@ module OnlineMigrations
|
|
99
103
|
#
|
100
104
|
def backfill_columns_for_type_change_in_background(table_name, *column_names, model_name: nil,
|
101
105
|
type_cast_functions: {}, **options)
|
106
|
+
if model_name.nil? && Utils.multiple_databases?
|
107
|
+
raise ArgumentError, "You must pass a :model_name when using multiple databases."
|
108
|
+
end
|
109
|
+
|
102
110
|
tmp_columns = column_names.map { |column_name| "#{column_name}_for_type_change" }
|
103
111
|
model_name = model_name.name if model_name.is_a?(Class)
|
104
112
|
|
@@ -153,6 +161,10 @@ module OnlineMigrations
|
|
153
161
|
# @see #copy_column_in_background
|
154
162
|
#
|
155
163
|
def copy_columns_in_background(table_name, copy_from, copy_to, model_name: nil, type_cast_functions: {}, **options)
|
164
|
+
if model_name.nil? && Utils.multiple_databases?
|
165
|
+
raise ArgumentError, "You must pass a :model_name when using multiple databases."
|
166
|
+
end
|
167
|
+
|
156
168
|
model_name = model_name.name if model_name.is_a?(Class)
|
157
169
|
|
158
170
|
enqueue_background_migration(
|
@@ -358,23 +370,41 @@ module OnlineMigrations
|
|
358
370
|
# in development and test environments
|
359
371
|
#
|
360
372
|
def enqueue_background_migration(migration_name, *arguments, **options)
|
373
|
+
migration = create_background_migration(migration_name, *arguments, **options)
|
374
|
+
|
375
|
+
# For convenience in dev/test environments
|
376
|
+
if Utils.developer_env?
|
377
|
+
runner = MigrationRunner.new(migration)
|
378
|
+
runner.run_all_migration_jobs
|
379
|
+
end
|
380
|
+
|
381
|
+
migration
|
382
|
+
end
|
383
|
+
|
384
|
+
# @private
|
385
|
+
def create_background_migration(migration_name, *arguments, **options)
|
361
386
|
options.assert_valid_keys(:batch_column_name, :min_value, :max_value, :batch_size, :sub_batch_size,
|
362
387
|
:batch_pause, :sub_batch_pause_ms, :batch_max_attempts)
|
363
388
|
|
364
|
-
|
365
|
-
|
366
|
-
migration = Migration.create!(
|
389
|
+
migration = Migration.new(
|
367
390
|
migration_name: migration_name,
|
368
391
|
arguments: arguments,
|
369
392
|
**options
|
370
393
|
)
|
371
394
|
|
372
|
-
|
373
|
-
if
|
374
|
-
|
375
|
-
|
395
|
+
shards = Utils.shard_names(migration.migration_model)
|
396
|
+
if shards.size > 1
|
397
|
+
migration.children = shards.map do |shard|
|
398
|
+
child = migration.dup
|
399
|
+
child.shard = shard
|
400
|
+
child
|
401
|
+
end
|
402
|
+
|
403
|
+
migration.composite = true
|
376
404
|
end
|
377
405
|
|
406
|
+
# This will save all the records using a transaction.
|
407
|
+
migration.save!
|
378
408
|
migration
|
379
409
|
end
|
380
410
|
end
|
@@ -94,8 +94,10 @@ module OnlineMigrations
|
|
94
94
|
|
95
95
|
private
|
96
96
|
def mark_as_running
|
97
|
-
|
98
|
-
|
97
|
+
Migration.transaction do
|
98
|
+
migration.running!
|
99
|
+
migration.parent.running! if migration.parent && migration.parent.enqueued?
|
100
|
+
end
|
99
101
|
end
|
100
102
|
|
101
103
|
def should_throttle?
|
@@ -19,37 +19,42 @@ module OnlineMigrations
|
|
19
19
|
end
|
20
20
|
|
21
21
|
relation = apply_limits(self.relation, column, start, finish, order)
|
22
|
+
base_relation = relation.reselect(column).reorder(column => order)
|
22
23
|
|
23
|
-
|
24
|
-
.
|
25
|
-
|
26
|
-
|
27
|
-
start_row = base_relation.uncached { base_relation.first }
|
28
|
-
|
29
|
-
return if !start_row
|
24
|
+
start_id = start || begin
|
25
|
+
start_row = base_relation.uncached { base_relation.first }
|
26
|
+
start_row[column] if start_row
|
27
|
+
end
|
30
28
|
|
31
|
-
start_id = start_row[column]
|
32
29
|
arel_table = relation.arel_table
|
33
30
|
|
34
|
-
|
31
|
+
while start_id
|
35
32
|
if order == :asc
|
36
33
|
start_cond = arel_table[column].gteq(start_id)
|
37
34
|
else
|
38
35
|
start_cond = arel_table[column].lteq(start_id)
|
39
36
|
end
|
40
37
|
|
41
|
-
stop_row = base_relation.uncached do
|
38
|
+
last_row, stop_row = base_relation.uncached do
|
42
39
|
base_relation
|
43
40
|
.where(start_cond)
|
44
|
-
.offset(of)
|
45
|
-
.first
|
41
|
+
.offset(of - 1)
|
42
|
+
.first(2)
|
43
|
+
end
|
44
|
+
|
45
|
+
if last_row.nil?
|
46
|
+
# We are at the end of the table.
|
47
|
+
last_row, stop_row = base_relation.uncached do
|
48
|
+
base_relation
|
49
|
+
.where(start_cond)
|
50
|
+
.last(2)
|
51
|
+
end
|
46
52
|
end
|
47
53
|
|
48
54
|
batch_relation = relation.where(start_cond)
|
49
55
|
|
50
56
|
if stop_row
|
51
57
|
stop_id = stop_row[column]
|
52
|
-
start_id = stop_id
|
53
58
|
|
54
59
|
if order == :asc
|
55
60
|
stop_cond = arel_table[column].lt(stop_id)
|
@@ -64,10 +69,14 @@ module OnlineMigrations
|
|
64
69
|
# efficient UPDATE queries, hence we get rid of it.
|
65
70
|
batch_relation = batch_relation.except(:order)
|
66
71
|
|
72
|
+
last_id = (last_row && last_row[column]) || start_id
|
73
|
+
|
67
74
|
# Retaining the results in the query cache would undermine the point of batching.
|
68
|
-
batch_relation.uncached { yield batch_relation,
|
75
|
+
batch_relation.uncached { yield batch_relation, start_id, last_id }
|
76
|
+
|
77
|
+
break if stop_row.nil?
|
69
78
|
|
70
|
-
|
79
|
+
start_id = stop_id
|
71
80
|
end
|
72
81
|
end
|
73
82
|
|
@@ -416,15 +416,8 @@ module OnlineMigrations
|
|
416
416
|
end
|
417
417
|
end
|
418
418
|
|
419
|
-
if index.name.include?(from_column)
|
420
|
-
name = index.name.gsub(from_column, to_column)
|
421
|
-
end
|
422
|
-
|
423
|
-
name = index_name(table_name, new_columns) if !name || name.length > max_identifier_length
|
424
|
-
|
425
419
|
options = {
|
426
420
|
unique: index.unique,
|
427
|
-
name: name,
|
428
421
|
length: index.lengths,
|
429
422
|
order: index.orders,
|
430
423
|
}
|
@@ -31,6 +31,7 @@ module OnlineMigrations
|
|
31
31
|
|
32
32
|
def check(command, *args, &block)
|
33
33
|
check_database_version
|
34
|
+
set_statement_timeout
|
34
35
|
check_lock_timeout
|
35
36
|
|
36
37
|
if !safe?
|
@@ -98,6 +99,16 @@ module OnlineMigrations
|
|
98
99
|
@database_version_checked = true
|
99
100
|
end
|
100
101
|
|
102
|
+
def set_statement_timeout
|
103
|
+
if !@statement_timeout_set
|
104
|
+
if (statement_timeout = OnlineMigrations.config.statement_timeout)
|
105
|
+
# TODO: inline this method call after deprecated `disable_statement_timeout` method removal.
|
106
|
+
connection.__set_statement_timeout(statement_timeout)
|
107
|
+
end
|
108
|
+
@statement_timeout_set = true
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
101
112
|
def check_lock_timeout
|
102
113
|
limit = OnlineMigrations.config.lock_timeout_limit
|
103
114
|
|
@@ -35,6 +35,12 @@ module OnlineMigrations
|
|
35
35
|
end
|
36
36
|
end
|
37
37
|
|
38
|
+
# Statement timeout used for migrations (in seconds)
|
39
|
+
#
|
40
|
+
# @return [Numeric]
|
41
|
+
#
|
42
|
+
attr_accessor :statement_timeout
|
43
|
+
|
38
44
|
# Set the database version against which the checks will be performed
|
39
45
|
#
|
40
46
|
# If your development database version is different from production, you can specify
|
@@ -158,7 +164,7 @@ module OnlineMigrations
|
|
158
164
|
# migration failure in production. This is also useful in development to get
|
159
165
|
# a better grasp of what is going on for high-level statements like add_column_with_default.
|
160
166
|
#
|
161
|
-
# This feature is enabled by default in a production Rails
|
167
|
+
# This feature is enabled by default in a staging and production Rails environments.
|
162
168
|
# @return [Boolean]
|
163
169
|
#
|
164
170
|
# @note: It can be overridden by `ONLINE_MIGRATIONS_VERBOSE_SQL_LOGS` environment variable.
|
@@ -60,9 +60,7 @@ module OnlineMigrations
|
|
60
60
|
#
|
61
61
|
# @param _attempt [Integer] attempt number
|
62
62
|
#
|
63
|
-
def lock_timeout(_attempt)
|
64
|
-
raise NotImplementedError
|
65
|
-
end
|
63
|
+
def lock_timeout(_attempt); end
|
66
64
|
|
67
65
|
# Returns sleep time after unsuccessful lock attempt (in seconds)
|
68
66
|
#
|
@@ -143,9 +141,9 @@ module OnlineMigrations
|
|
143
141
|
#
|
144
142
|
# @param attempts [Integer] Maximum number of attempts
|
145
143
|
# @param delay [Numeric] Sleep time after unsuccessful lock attempt (in seconds)
|
146
|
-
# @param lock_timeout [Numeric] Database lock timeout value (in seconds)
|
144
|
+
# @param lock_timeout [Numeric, nil] Database lock timeout value (in seconds)
|
147
145
|
#
|
148
|
-
def initialize(attempts:, delay:, lock_timeout:)
|
146
|
+
def initialize(attempts:, delay:, lock_timeout: nil)
|
149
147
|
super()
|
150
148
|
@attempts = attempts
|
151
149
|
@delay = delay
|
@@ -196,7 +194,7 @@ module OnlineMigrations
|
|
196
194
|
# @param max_delay [Numeric] Maximum sleep time after unsuccessful lock attempt (in seconds)
|
197
195
|
# @param lock_timeout [Numeric] Database lock timeout value (in seconds)
|
198
196
|
#
|
199
|
-
def initialize(attempts:, base_delay:, max_delay:, lock_timeout:)
|
197
|
+
def initialize(attempts:, base_delay:, max_delay:, lock_timeout: nil)
|
200
198
|
super()
|
201
199
|
@attempts = attempts
|
202
200
|
@base_delay = base_delay
|
@@ -680,34 +680,40 @@ module OnlineMigrations
|
|
680
680
|
#
|
681
681
|
# @see https://edgeapi.rubyonrails.org/classes/ActiveRecord/ConnectionAdapters/SchemaStatements.html#method-i-add_index
|
682
682
|
#
|
683
|
-
def add_index(table_name, column_name, options
|
684
|
-
|
683
|
+
def add_index(table_name, column_name, **options)
|
684
|
+
__ensure_not_in_transaction! if options[:algorithm] == :concurrently
|
685
685
|
|
686
|
-
|
687
|
-
|
688
|
-
|
689
|
-
|
690
|
-
index_name = options[:name]
|
691
|
-
index_name ||= index_name(table_name, column_names)
|
692
|
-
|
693
|
-
if index_exists?(table_name, column_name, **options)
|
686
|
+
# Rewrite this with `IndexDefinition#defined_for?` when Active Record >= 7.1 is supported.
|
687
|
+
# See https://github.com/rails/rails/pull/45160.
|
688
|
+
index = indexes(table_name).find { |i| __index_defined_for?(i, column_name, **options) }
|
689
|
+
if index
|
694
690
|
schema = __schema_for_table(table_name)
|
695
691
|
|
696
|
-
if __index_valid?(
|
697
|
-
Utils.say("Index was not created because it already exists
|
698
|
-
"or similar): table_name: #{table_name}, column_name: #{column_name}")
|
692
|
+
if __index_valid?(index.name, schema: schema)
|
693
|
+
Utils.say("Index was not created because it already exists.")
|
699
694
|
return
|
700
695
|
else
|
701
696
|
Utils.say("Recreating invalid index: table_name: #{table_name}, column_name: #{column_name}")
|
702
|
-
remove_index(table_name, column_name,
|
697
|
+
remove_index(table_name, column_name, **options)
|
703
698
|
end
|
704
699
|
end
|
705
700
|
|
706
|
-
|
701
|
+
if OnlineMigrations.config.statement_timeout
|
707
702
|
# "CREATE INDEX CONCURRENTLY" requires a "SHARE UPDATE EXCLUSIVE" lock.
|
708
703
|
# It only conflicts with constraint validations, creating/removing indexes,
|
709
704
|
# and some other "ALTER TABLE"s.
|
710
|
-
super
|
705
|
+
super
|
706
|
+
else
|
707
|
+
OnlineMigrations.deprecator.warn(<<~MSG)
|
708
|
+
Running `add_index` without a statement timeout is deprecated.
|
709
|
+
Configure an explicit statement timeout in the initializer file via `config.statement_timeout`
|
710
|
+
or the default database statement timeout will be used.
|
711
|
+
Example, `config.statement_timeout = 1.hour`.
|
712
|
+
MSG
|
713
|
+
|
714
|
+
disable_statement_timeout do
|
715
|
+
super
|
716
|
+
end
|
711
717
|
end
|
712
718
|
end
|
713
719
|
|
@@ -716,23 +722,32 @@ module OnlineMigrations
|
|
716
722
|
# @see https://edgeapi.rubyonrails.org/classes/ActiveRecord/ConnectionAdapters/SchemaStatements.html#method-i-remove_index
|
717
723
|
#
|
718
724
|
def remove_index(table_name, column_name = nil, **options)
|
719
|
-
|
720
|
-
|
721
|
-
|
725
|
+
if column_name.blank? && options[:column].blank? && options[:name].blank?
|
726
|
+
raise ArgumentError, "No name or columns specified"
|
727
|
+
end
|
722
728
|
|
723
|
-
|
729
|
+
__ensure_not_in_transaction! if options[:algorithm] == :concurrently
|
724
730
|
|
725
|
-
if index_exists?(table_name,
|
726
|
-
|
731
|
+
if index_exists?(table_name, column_name, **options)
|
732
|
+
if OnlineMigrations.config.statement_timeout
|
727
733
|
# "DROP INDEX CONCURRENTLY" requires a "SHARE UPDATE EXCLUSIVE" lock.
|
728
734
|
# It only conflicts with constraint validations, other creating/removing indexes,
|
729
735
|
# and some "ALTER TABLE"s.
|
736
|
+
super(table_name, column_name, **options)
|
737
|
+
else
|
738
|
+
OnlineMigrations.deprecator.warn(<<~MSG)
|
739
|
+
Running `remove_index` without a statement timeout is deprecated.
|
740
|
+
Configure an explicit statement timeout in the initializer file via `config.statement_timeout`
|
741
|
+
or the default database statement timeout will be used.
|
742
|
+
Example, `config.statement_timeout = 1.hour`.
|
743
|
+
MSG
|
730
744
|
|
731
|
-
|
745
|
+
disable_statement_timeout do
|
746
|
+
super(table_name, column_name, **options)
|
747
|
+
end
|
732
748
|
end
|
733
749
|
else
|
734
|
-
Utils.say("Index was not removed because it does not exist
|
735
|
-
"or similar): table_name: #{table_name}, column_name: #{column_names}")
|
750
|
+
Utils.say("Index was not removed because it does not exist.")
|
736
751
|
end
|
737
752
|
end
|
738
753
|
|
@@ -778,11 +793,22 @@ module OnlineMigrations
|
|
778
793
|
# Skip costly operation if already validated.
|
779
794
|
return if foreign_key.validated?
|
780
795
|
|
781
|
-
|
796
|
+
if OnlineMigrations.config.statement_timeout
|
782
797
|
# "VALIDATE CONSTRAINT" requires a "SHARE UPDATE EXCLUSIVE" lock.
|
783
798
|
# It only conflicts with other validations, creating/removing indexes,
|
784
799
|
# and some other "ALTER TABLE"s.
|
785
800
|
super
|
801
|
+
else
|
802
|
+
OnlineMigrations.deprecator.warn(<<~MSG)
|
803
|
+
Running `validate_foreign_key` without a statement timeout is deprecated.
|
804
|
+
Configure an explicit statement timeout in the initializer file via `config.statement_timeout`
|
805
|
+
or the default database statement timeout will be used.
|
806
|
+
Example, `config.statement_timeout = 1.hour`.
|
807
|
+
MSG
|
808
|
+
|
809
|
+
disable_statement_timeout do
|
810
|
+
super
|
811
|
+
end
|
786
812
|
end
|
787
813
|
end
|
788
814
|
|
@@ -811,11 +837,22 @@ module OnlineMigrations
|
|
811
837
|
# Skip costly operation if already validated.
|
812
838
|
return if check_constraint.validated?
|
813
839
|
|
814
|
-
|
840
|
+
if OnlineMigrations.config.statement_timeout
|
815
841
|
# "VALIDATE CONSTRAINT" requires a "SHARE UPDATE EXCLUSIVE" lock.
|
816
842
|
# It only conflicts with other validations, creating/removing indexes,
|
817
843
|
# and some other "ALTER TABLE"s.
|
818
844
|
super
|
845
|
+
else
|
846
|
+
OnlineMigrations.deprecator.warn(<<~MSG)
|
847
|
+
Running `validate_check_constraint` without a statement timeout is deprecated.
|
848
|
+
Configure an explicit statement timeout in the initializer file via `config.statement_timeout`
|
849
|
+
or the default database statement timeout will be used.
|
850
|
+
Example, `config.statement_timeout = 1.hour`.
|
851
|
+
MSG
|
852
|
+
|
853
|
+
disable_statement_timeout do
|
854
|
+
super
|
855
|
+
end
|
819
856
|
end
|
820
857
|
end
|
821
858
|
|
@@ -855,28 +892,26 @@ module OnlineMigrations
|
|
855
892
|
end
|
856
893
|
end
|
857
894
|
|
858
|
-
#
|
859
|
-
#
|
860
|
-
# Long-running migrations may take more than the timeout allowed by the database.
|
861
|
-
# Disable the session's statement timeout to ensure migrations don't get killed prematurely.
|
862
|
-
#
|
863
|
-
# Statement timeouts are already disabled in `add_index`, `remove_index`,
|
864
|
-
# `validate_foreign_key`, and `validate_check_constraint` helpers.
|
865
|
-
#
|
866
|
-
# @return [void]
|
867
|
-
#
|
868
|
-
# @example
|
869
|
-
# disable_statement_timeout do
|
870
|
-
# add_index(:users, :email, unique: true, algorithm: :concurrently)
|
871
|
-
# end
|
872
|
-
#
|
895
|
+
# @private
|
873
896
|
def disable_statement_timeout
|
874
|
-
|
875
|
-
|
897
|
+
OnlineMigrations.deprecator.warn(<<~MSG)
|
898
|
+
`disable_statement_timeout` is deprecated and will be removed. Configure an explicit
|
899
|
+
statement timeout in the initializer file via `config.statement_timeout` or the default
|
900
|
+
database statement timeout will be used. Example, `config.statement_timeout = 1.hour`.
|
901
|
+
MSG
|
876
902
|
|
903
|
+
prev_value = select_value("SHOW statement_timeout")
|
904
|
+
__set_statement_timeout(0)
|
877
905
|
yield
|
878
906
|
ensure
|
879
|
-
|
907
|
+
__set_statement_timeout(prev_value)
|
908
|
+
end
|
909
|
+
|
910
|
+
# @private
|
911
|
+
def __set_statement_timeout(timeout)
|
912
|
+
# use ceil to prevent no timeout for values under 1 ms
|
913
|
+
timeout = (timeout.to_f * 1000).ceil if !timeout.is_a?(String)
|
914
|
+
execute("SET statement_timeout TO #{quote(timeout)}")
|
880
915
|
end
|
881
916
|
|
882
917
|
# @private
|
@@ -905,6 +940,17 @@ module OnlineMigrations
|
|
905
940
|
end
|
906
941
|
end
|
907
942
|
|
943
|
+
# Will not be needed for Active Record >= 7.1
|
944
|
+
def __index_defined_for?(index, columns = nil, name: nil, unique: nil, valid: nil, include: nil, nulls_not_distinct: nil, **options)
|
945
|
+
columns = options[:column] if columns.blank?
|
946
|
+
(columns.nil? || Array(index.columns) == Array(columns).map(&:to_s)) &&
|
947
|
+
(name.nil? || index.name == name.to_s) &&
|
948
|
+
(unique.nil? || index.unique == unique) &&
|
949
|
+
(valid.nil? || index.valid == valid) &&
|
950
|
+
(include.nil? || Array(index.include) == Array(include).map(&:to_s)) &&
|
951
|
+
(nulls_not_distinct.nil? || index.nulls_not_distinct == nulls_not_distinct)
|
952
|
+
end
|
953
|
+
|
908
954
|
def __not_null_constraint_exists?(table_name, column_name, name: nil)
|
909
955
|
name ||= __not_null_constraint_name(table_name, column_name)
|
910
956
|
__check_constraint_exists?(table_name, name: name)
|
@@ -10,8 +10,17 @@ module OnlineMigrations
|
|
10
10
|
ActiveRecord.version.to_s.to_f
|
11
11
|
end
|
12
12
|
|
13
|
+
def env
|
14
|
+
if defined?(Rails.env)
|
15
|
+
Rails.env
|
16
|
+
else
|
17
|
+
# default to production for safety
|
18
|
+
ENV["RACK_ENV"] || "production"
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
13
22
|
def developer_env?
|
14
|
-
|
23
|
+
env == "development" || env == "test"
|
15
24
|
end
|
16
25
|
|
17
26
|
def say(message)
|
@@ -137,6 +146,11 @@ module OnlineMigrations
|
|
137
146
|
return pool_manager.shard_names.uniq if pool_manager
|
138
147
|
end
|
139
148
|
end
|
149
|
+
|
150
|
+
def multiple_databases?
|
151
|
+
db_config = ActiveRecord::Base.configurations.configs_for(env_name: env)
|
152
|
+
db_config.reject(&:replica?).size > 1
|
153
|
+
end
|
140
154
|
end
|
141
155
|
end
|
142
156
|
end
|
data/lib/online_migrations.rb
CHANGED
@@ -1,7 +1,19 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require "active_record"
|
4
|
+
|
4
5
|
require "online_migrations/version"
|
6
|
+
require "online_migrations/utils"
|
7
|
+
require "online_migrations/change_column_type_helpers"
|
8
|
+
require "online_migrations/background_migrations/migration_helpers"
|
9
|
+
require "online_migrations/schema_statements"
|
10
|
+
require "online_migrations/migration"
|
11
|
+
require "online_migrations/migrator"
|
12
|
+
require "online_migrations/schema_dumper"
|
13
|
+
require "online_migrations/database_tasks"
|
14
|
+
require "online_migrations/command_recorder"
|
15
|
+
require "online_migrations/error_messages"
|
16
|
+
require "online_migrations/config"
|
5
17
|
|
6
18
|
module OnlineMigrations
|
7
19
|
class Error < StandardError; end
|
@@ -9,15 +21,8 @@ module OnlineMigrations
|
|
9
21
|
|
10
22
|
extend ActiveSupport::Autoload
|
11
23
|
|
12
|
-
autoload :Utils
|
13
|
-
autoload :ErrorMessages
|
14
|
-
autoload :Config
|
15
24
|
autoload :BatchIterator
|
16
25
|
autoload :VerboseSqlLogs
|
17
|
-
autoload :Migration
|
18
|
-
autoload :Migrator
|
19
|
-
autoload :SchemaDumper
|
20
|
-
autoload :DatabaseTasks
|
21
26
|
autoload :ForeignKeysCollector
|
22
27
|
autoload :IndexDefinition
|
23
28
|
autoload :IndexesCollector
|
@@ -36,10 +41,7 @@ module OnlineMigrations
|
|
36
41
|
autoload :NullLockRetrier
|
37
42
|
end
|
38
43
|
|
39
|
-
autoload :CommandRecorder
|
40
44
|
autoload :CopyTrigger
|
41
|
-
autoload :ChangeColumnTypeHelpers
|
42
|
-
autoload :SchemaStatements
|
43
45
|
|
44
46
|
module BackgroundMigrations
|
45
47
|
extend ActiveSupport::Autoload
|
@@ -59,7 +61,6 @@ module OnlineMigrations
|
|
59
61
|
autoload :Migration
|
60
62
|
autoload :MigrationJobRunner
|
61
63
|
autoload :MigrationRunner
|
62
|
-
autoload :MigrationHelpers
|
63
64
|
autoload :Scheduler
|
64
65
|
end
|
65
66
|
|
@@ -80,6 +81,15 @@ module OnlineMigrations
|
|
80
81
|
BackgroundMigrations::Scheduler.run
|
81
82
|
end
|
82
83
|
|
84
|
+
def deprecator
|
85
|
+
@deprecator ||=
|
86
|
+
if Utils.ar_version >= 7.1
|
87
|
+
ActiveSupport::Deprecation.new(nil, "online_migrations")
|
88
|
+
else
|
89
|
+
ActiveSupport::Deprecation
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
83
93
|
# @private
|
84
94
|
def load
|
85
95
|
require "active_record/connection_adapters/postgresql_adapter"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: online_migrations
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.13.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- fatkodima
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-01-
|
11
|
+
date: 2024-01-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activerecord
|