online_migrations 0.25.0 → 0.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +31 -0
  3. data/README.md +18 -73
  4. data/docs/0.27-upgrade.md +24 -0
  5. data/docs/background_data_migrations.md +200 -101
  6. data/docs/background_schema_migrations.md +2 -2
  7. data/docs/configuring.md +8 -0
  8. data/lib/generators/online_migrations/{background_migration_generator.rb → data_migration_generator.rb} +4 -4
  9. data/lib/generators/online_migrations/templates/add_sharding_to_online_migrations.rb.tt +1 -1
  10. data/lib/generators/online_migrations/templates/add_timestamps_to_background_migrations.rb.tt +1 -1
  11. data/lib/generators/online_migrations/templates/background_schema_migrations_change_unique_index.rb.tt +1 -1
  12. data/lib/generators/online_migrations/templates/change_background_data_migrations.rb.tt +34 -0
  13. data/lib/generators/online_migrations/templates/{background_data_migration.rb.tt → data_migration.rb.tt} +8 -9
  14. data/lib/generators/online_migrations/templates/initializer.rb.tt +22 -25
  15. data/lib/generators/online_migrations/templates/install_migration.rb.tt +9 -40
  16. data/lib/generators/online_migrations/upgrade_generator.rb +16 -8
  17. data/lib/online_migrations/active_record_batch_enumerator.rb +8 -0
  18. data/lib/online_migrations/background_data_migrations/backfill_column.rb +50 -0
  19. data/lib/online_migrations/background_data_migrations/config.rb +62 -0
  20. data/lib/online_migrations/{background_migrations → background_data_migrations}/copy_column.rb +15 -28
  21. data/lib/online_migrations/{background_migrations → background_data_migrations}/delete_associated_records.rb +9 -5
  22. data/lib/online_migrations/{background_migrations → background_data_migrations}/delete_orphaned_records.rb +5 -9
  23. data/lib/online_migrations/background_data_migrations/migration.rb +312 -0
  24. data/lib/online_migrations/{background_migrations → background_data_migrations}/migration_helpers.rb +72 -61
  25. data/lib/online_migrations/background_data_migrations/migration_job.rb +158 -0
  26. data/lib/online_migrations/background_data_migrations/migration_status_validator.rb +65 -0
  27. data/lib/online_migrations/{background_migrations → background_data_migrations}/perform_action_on_relation.rb +5 -5
  28. data/lib/online_migrations/{background_migrations → background_data_migrations}/reset_counters.rb +5 -5
  29. data/lib/online_migrations/background_data_migrations/scheduler.rb +78 -0
  30. data/lib/online_migrations/background_data_migrations/ticker.rb +62 -0
  31. data/lib/online_migrations/background_schema_migrations/config.rb +2 -2
  32. data/lib/online_migrations/background_schema_migrations/migration.rb +57 -127
  33. data/lib/online_migrations/background_schema_migrations/migration_helpers.rb +26 -47
  34. data/lib/online_migrations/background_schema_migrations/migration_runner.rb +43 -97
  35. data/lib/online_migrations/background_schema_migrations/scheduler.rb +2 -2
  36. data/lib/online_migrations/batch_iterator.rb +7 -4
  37. data/lib/online_migrations/change_column_type_helpers.rb +75 -14
  38. data/lib/online_migrations/command_checker.rb +32 -20
  39. data/lib/online_migrations/config.rb +12 -4
  40. data/lib/online_migrations/data_migration.rb +127 -0
  41. data/lib/online_migrations/error_messages.rb +16 -0
  42. data/lib/online_migrations/index_definition.rb +1 -1
  43. data/lib/online_migrations/lock_retrier.rb +5 -2
  44. data/lib/online_migrations/migration.rb +8 -1
  45. data/lib/online_migrations/schema_cache.rb +0 -78
  46. data/lib/online_migrations/schema_statements.rb +18 -74
  47. data/lib/online_migrations/shard_aware.rb +44 -0
  48. data/lib/online_migrations/utils.rb +1 -20
  49. data/lib/online_migrations/verbose_sql_logs.rb +1 -7
  50. data/lib/online_migrations/version.rb +1 -1
  51. data/lib/online_migrations.rb +19 -19
  52. metadata +25 -24
  53. data/lib/online_migrations/background_migration.rb +0 -64
  54. data/lib/online_migrations/background_migrations/backfill_column.rb +0 -54
  55. data/lib/online_migrations/background_migrations/background_migration_class_validator.rb +0 -29
  56. data/lib/online_migrations/background_migrations/config.rb +0 -74
  57. data/lib/online_migrations/background_migrations/migration.rb +0 -329
  58. data/lib/online_migrations/background_migrations/migration_job.rb +0 -109
  59. data/lib/online_migrations/background_migrations/migration_job_runner.rb +0 -66
  60. data/lib/online_migrations/background_migrations/migration_job_status_validator.rb +0 -29
  61. data/lib/online_migrations/background_migrations/migration_runner.rb +0 -161
  62. data/lib/online_migrations/background_migrations/migration_status_validator.rb +0 -48
  63. data/lib/online_migrations/background_migrations/scheduler.rb +0 -42
@@ -1,161 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module OnlineMigrations
4
- module BackgroundMigrations
5
- # Runs single background migration.
6
- class MigrationRunner
7
- attr_reader :migration
8
-
9
- def initialize(migration)
10
- @migration = migration
11
- end
12
-
13
- # Runs one background migration job.
14
- def run_migration_job
15
- raise "Should not be called on a composite (with sharding) migration" if migration.composite?
16
- return if migration.cancelled? || migration.succeeded?
17
-
18
- mark_as_running if migration.enqueued?
19
- migration_payload = notifications_payload(migration)
20
-
21
- if !migration.migration_jobs.exists?
22
- ActiveSupport::Notifications.instrument("started.background_migrations", migration_payload)
23
- end
24
-
25
- if should_throttle?
26
- ActiveSupport::Notifications.instrument("throttled.background_migrations", migration_payload)
27
- return
28
- end
29
-
30
- next_migration_job = find_or_create_next_migration_job
31
-
32
- if next_migration_job
33
- job_runner = MigrationJobRunner.new(next_migration_job)
34
- job_runner.run
35
- elsif !migration.migration_jobs.active.exists?
36
- if migration.migration_jobs.failed.exists?
37
- migration.update!(status: :failed, finished_at: Time.current)
38
- else
39
- migration.update!(status: :succeeded, finished_at: Time.current)
40
- end
41
-
42
- ActiveSupport::Notifications.instrument("completed.background_migrations", migration_payload)
43
-
44
- complete_parent_if_needed(migration) if migration.parent.present?
45
- end
46
-
47
- next_migration_job
48
- end
49
-
50
- # Runs the background migration until completion.
51
- #
52
- # @note This method should not be used in production environments
53
- #
54
- def run_all_migration_jobs
55
- run_inline = OnlineMigrations.config.run_background_migrations_inline
56
- if run_inline && !run_inline.call
57
- raise "This method is not intended for use in production environments"
58
- end
59
-
60
- return if migration.completed? || migration.cancelled?
61
-
62
- mark_as_running
63
-
64
- if migration.composite?
65
- migration.children.each do |child_migration|
66
- runner = self.class.new(child_migration)
67
- runner.run_all_migration_jobs
68
- end
69
- else
70
- while migration.running?
71
- run_migration_job
72
- end
73
- end
74
- end
75
-
76
- # Finishes the background migration.
77
- #
78
- # Keep running until the migration is finished.
79
- #
80
- def finish
81
- return if migration.completed? || migration.cancelled?
82
-
83
- if migration.composite?
84
- migration.children.each do |child_migration|
85
- runner = self.class.new(child_migration)
86
- runner.finish
87
- end
88
- else
89
- # Mark is as finishing to avoid being picked up
90
- # by the background migrations scheduler.
91
- migration.finishing!
92
- migration.reset_failed_jobs_attempts
93
-
94
- while migration.finishing?
95
- run_migration_job
96
- end
97
- end
98
- end
99
-
100
- private
101
- def mark_as_running
102
- Migration.transaction do
103
- migration.update!(status: :running, started_at: Time.current, finished_at: nil)
104
-
105
- if (parent = migration.parent)
106
- if parent.started_at
107
- parent.update!(status: :running, finished_at: nil)
108
- else
109
- parent.update!(status: :running, started_at: Time.current, finished_at: nil)
110
- end
111
- end
112
- end
113
- end
114
-
115
- def should_throttle?
116
- ::OnlineMigrations.config.throttler.call
117
- end
118
-
119
- def find_or_create_next_migration_job
120
- min_value, max_value = migration.next_batch_range
121
-
122
- if min_value && max_value
123
- create_migration_job!(min_value, max_value)
124
- else
125
- migration.migration_jobs.enqueued.first || migration.migration_jobs.retriable.first
126
- end
127
- end
128
-
129
- def create_migration_job!(min_value, max_value)
130
- migration.migration_jobs.create!(
131
- min_value: min_value,
132
- max_value: max_value
133
- )
134
- end
135
-
136
- def complete_parent_if_needed(migration)
137
- parent = migration.parent
138
- completed = false
139
-
140
- parent.with_lock do
141
- children = parent.children.select(:status)
142
- if children.all?(&:succeeded?)
143
- parent.update!(status: :succeeded, finished_at: Time.current)
144
- completed = true
145
- elsif children.any?(&:failed?)
146
- parent.update!(status: :failed, finished_at: Time.current)
147
- completed = true
148
- end
149
- end
150
-
151
- if completed
152
- ActiveSupport::Notifications.instrument("completed.background_migrations", notifications_payload(migration))
153
- end
154
- end
155
-
156
- def notifications_payload(migration)
157
- { background_migration: migration }
158
- end
159
- end
160
- end
161
- end
@@ -1,48 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module OnlineMigrations
4
- module BackgroundMigrations
5
- # @private
6
- class MigrationStatusValidator < ActiveModel::Validator
7
- VALID_STATUS_TRANSITIONS = {
8
- # enqueued -> running occurs when the migration starts performing.
9
- # enqueued -> paused occurs when the migration is paused before starting.
10
- "enqueued" => ["running", "paused", "cancelled"],
11
- # running -> paused occurs when a user pauses the migration as
12
- # it's performing.
13
- # running -> finishing occurs when a user manually finishes the migration.
14
- # running -> succeeded occurs when the migration completes successfully.
15
- # running -> failed occurs when the migration raises an exception when running.
16
- "running" => [
17
- "paused",
18
- "finishing",
19
- "succeeded",
20
- "failed",
21
- "cancelled",
22
- ],
23
- # finishing -> succeeded occurs when the migration completes successfully.
24
- # finishing -> failed occurs when the migration raises an exception when running.
25
- "finishing" => ["succeeded", "failed", "cancelled"],
26
- # paused -> running occurs when the migration is resumed after being paused.
27
- "paused" => ["running", "cancelled"],
28
- # failed -> enqueued occurs when the failed migration jobs are retried after being failed.
29
- # failed -> running occurs when the failed migration is retried.
30
- "failed" => ["enqueued", "running", "cancelled"],
31
- }
32
-
33
- def validate(record)
34
- return if !record.status_changed?
35
-
36
- previous_status, new_status = record.status_change
37
- valid_new_statuses = VALID_STATUS_TRANSITIONS.fetch(previous_status, [])
38
-
39
- if !valid_new_statuses.include?(new_status)
40
- record.errors.add(
41
- :status,
42
- "cannot transition background migration from status #{previous_status} to #{new_status}"
43
- )
44
- end
45
- end
46
- end
47
- end
48
- end
@@ -1,42 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module OnlineMigrations
4
- module BackgroundMigrations
5
- # Class responsible for scheduling background migrations.
6
- #
7
- # It selects a single runnable background migration and runs it one step (one batch) at a time.
8
- # A migration is considered runnable if it is not completed and the time interval between
9
- # successive runs has passed.
10
- #
11
- # Scheduler should be configured to run periodically, for example, via cron.
12
- #
13
- # @example Run via whenever
14
- # # add this to schedule.rb
15
- # every 1.minute do
16
- # runner "OnlineMigrations.run_background_data_migrations"
17
- # end
18
- #
19
- # @example Run via whenever (specific shard)
20
- # every 1.minute do
21
- # runner "OnlineMigrations.run_background_data_migrations(shard: :shard_two)"
22
- # end
23
- #
24
- class Scheduler
25
- def self.run(**options)
26
- new.run(**options)
27
- end
28
-
29
- # Runs Scheduler
30
- def run(**options)
31
- active_migrations = Migration.runnable.active.queue_order
32
- active_migrations = active_migrations.where(shard: options[:shard]) if options.key?(:shard)
33
- runnable_migration = active_migrations.select(&:interval_elapsed?).first
34
-
35
- if runnable_migration
36
- runner = MigrationRunner.new(runnable_migration)
37
- runner.run_migration_job
38
- end
39
- end
40
- end
41
- end
42
- end