monarch_migrate 0.5.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (18) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/ci.yml +2 -7
  3. data/.gitignore +1 -0
  4. data/.ruby-version +1 -1
  5. data/Appraisals +9 -2
  6. data/Gemfile.lock +108 -110
  7. data/README.md +158 -62
  8. data/lib/generators/monarch_migrate/install/install_generator.rb +12 -12
  9. data/lib/generators/rails/data_migration/data_migration_generator.rb +10 -2
  10. data/lib/generators/rspec/data_migration_generator.rb +20 -18
  11. data/lib/generators/test_unit/data_migration_generator.rb +20 -17
  12. data/lib/monarch_migrate/migration.rb +26 -8
  13. data/lib/monarch_migrate/migrator.rb +4 -6
  14. data/lib/monarch_migrate/tasks.rake +1 -1
  15. data/lib/monarch_migrate/testing.rb +5 -4
  16. data/lib/monarch_migrate/version.rb +1 -1
  17. data/monarch_migrate.gemspec +3 -3
  18. metadata +8 -7
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 814685cc086ec84d8f1e877c80134db4c702b9d06602f4169f99dc82c1cd1ca8
4
- data.tar.gz: 726adc5a2087bec90a6fa1fc1ec965d633c4de35f69298da2b42e7e043026af8
3
+ metadata.gz: 0f80c87911c7ad7157d9e4a12b913993cf606175dc53e2df5990bb1cf083ca0a
4
+ data.tar.gz: 9c763a32050006716c48172355cf471a33e3a413b0031d8f7be2a226dc274aa9
5
5
  SHA512:
6
- metadata.gz: 665c315826067e28509312d5cf9ef8729803dfb105a123aad44f81df9c0c2f46cd185f9d5381939def677d68553a150c37d2e1d5f040b2b281dc6085b47e31d9
7
- data.tar.gz: ff6cb7e01f9071363eb561f392c80e8b8e97b371f853f79f35f1f4a594b8165f4e063d9225ec1ff3af992874a57ef081d16455a049fe547bfe8c6a354578a8d3
6
+ metadata.gz: 8ad6dd0cd4c17f2cd7dc225de7dfae1269d4b715351eb99173254be7a1dea4b7ac2d1fa6803ae719fc692a116fbe60abfd742cea01dc76237d5bfad6dc19d8f4
7
+ data.tar.gz: d6d2442b63a91bb02dd9d5f2b1eb05dd443038dc3f614a2a057063fd47cac6e31b3571879c2940e0e1d2583a42a9c751559224d0cde3c2b49c0b200d67e6baf0
data/.github/workflows/ci.yml CHANGED
@@ -16,18 +16,13 @@ jobs:
16
16
  fail-fast: false
17
17
  matrix:
18
18
  gemfile:
19
- - "5.2"
19
+ - "6.0"
20
20
  - "6.1"
21
21
  - "7.0"
22
22
  ruby:
23
- - "2.7.3"
24
23
  - "3.0.0"
25
24
  - "3.1.0"
26
- exclude:
27
- - gemfile: "5.2"
28
- ruby: "3.0.0"
29
- - gemfile: "5.2"
30
- ruby: "3.1.0"
25
+ - "3.2.2"
31
26
 
32
27
  env:
33
28
  BUNDLE_GEMFILE: gemfiles/rails_${{ matrix.gemfile }}.gemfile
data/.gitignore CHANGED
@@ -73,6 +73,7 @@ fabric.properties
73
73
 
74
74
  # Migration generated during tests
75
75
  /test/generators/**/tmp
76
+ /test/fixtures/tmp
76
77
 
77
78
  # Ignore gem compile directory
78
79
  /pkg
data/.ruby-version CHANGED
@@ -1 +1 @@
1
- 2.7.3
1
+ 3.2.2
data/Appraisals CHANGED
@@ -1,5 +1,12 @@
1
- appraise "rails_5.2" do
2
- gem "rails", "~> 5.2"
1
+ # We test against only supported Rails versions.
2
+ # https://guides.rubyonrails.org/maintenance_policy.html
3
+
4
+ # Rails 6.0.Z is included in the list of supported series until June 1st 2023.
5
+ appraise "rails_6.0" do
6
+ gem "rails", "~> 6.0"
7
+ gem "net-smtp", require: false
8
+ gem "net-imap", require: false
9
+ gem "net-pop", require: false
3
10
  end
4
11
 
5
12
  appraise "rails_6.1" do
data/Gemfile.lock CHANGED
@@ -1,73 +1,73 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- monarch_migrate (0.4.0)
5
- rails (>= 5.2.0)
4
+ monarch_migrate (0.7.0)
5
+ rails (>= 6.0)
6
6
 
7
7
  GEM
8
8
  remote: https://rubygems.org/
9
9
  specs:
10
- actioncable (7.0.3)
11
- actionpack (= 7.0.3)
12
- activesupport (= 7.0.3)
10
+ actioncable (7.0.5)
11
+ actionpack (= 7.0.5)
12
+ activesupport (= 7.0.5)
13
13
  nio4r (~> 2.0)
14
14
  websocket-driver (>= 0.6.1)
15
- actionmailbox (7.0.3)
16
- actionpack (= 7.0.3)
17
- activejob (= 7.0.3)
18
- activerecord (= 7.0.3)
19
- activestorage (= 7.0.3)
20
- activesupport (= 7.0.3)
15
+ actionmailbox (7.0.5)
16
+ actionpack (= 7.0.5)
17
+ activejob (= 7.0.5)
18
+ activerecord (= 7.0.5)
19
+ activestorage (= 7.0.5)
20
+ activesupport (= 7.0.5)
21
21
  mail (>= 2.7.1)
22
22
  net-imap
23
23
  net-pop
24
24
  net-smtp
25
- actionmailer (7.0.3)
26
- actionpack (= 7.0.3)
27
- actionview (= 7.0.3)
28
- activejob (= 7.0.3)
29
- activesupport (= 7.0.3)
25
+ actionmailer (7.0.5)
26
+ actionpack (= 7.0.5)
27
+ actionview (= 7.0.5)
28
+ activejob (= 7.0.5)
29
+ activesupport (= 7.0.5)
30
30
  mail (~> 2.5, >= 2.5.4)
31
31
  net-imap
32
32
  net-pop
33
33
  net-smtp
34
34
  rails-dom-testing (~> 2.0)
35
- actionpack (7.0.3)
36
- actionview (= 7.0.3)
37
- activesupport (= 7.0.3)
38
- rack (~> 2.0, >= 2.2.0)
35
+ actionpack (7.0.5)
36
+ actionview (= 7.0.5)
37
+ activesupport (= 7.0.5)
38
+ rack (~> 2.0, >= 2.2.4)
39
39
  rack-test (>= 0.6.3)
40
40
  rails-dom-testing (~> 2.0)
41
41
  rails-html-sanitizer (~> 1.0, >= 1.2.0)
42
- actiontext (7.0.3)
43
- actionpack (= 7.0.3)
44
- activerecord (= 7.0.3)
45
- activestorage (= 7.0.3)
46
- activesupport (= 7.0.3)
42
+ actiontext (7.0.5)
43
+ actionpack (= 7.0.5)
44
+ activerecord (= 7.0.5)
45
+ activestorage (= 7.0.5)
46
+ activesupport (= 7.0.5)
47
47
  globalid (>= 0.6.0)
48
48
  nokogiri (>= 1.8.5)
49
- actionview (7.0.3)
50
- activesupport (= 7.0.3)
49
+ actionview (7.0.5)
50
+ activesupport (= 7.0.5)
51
51
  builder (~> 3.1)
52
52
  erubi (~> 1.4)
53
53
  rails-dom-testing (~> 2.0)
54
54
  rails-html-sanitizer (~> 1.1, >= 1.2.0)
55
- activejob (7.0.3)
56
- activesupport (= 7.0.3)
55
+ activejob (7.0.5)
56
+ activesupport (= 7.0.5)
57
57
  globalid (>= 0.3.6)
58
- activemodel (7.0.3)
59
- activesupport (= 7.0.3)
60
- activerecord (7.0.3)
61
- activemodel (= 7.0.3)
62
- activesupport (= 7.0.3)
63
- activestorage (7.0.3)
64
- actionpack (= 7.0.3)
65
- activejob (= 7.0.3)
66
- activerecord (= 7.0.3)
67
- activesupport (= 7.0.3)
58
+ activemodel (7.0.5)
59
+ activesupport (= 7.0.5)
60
+ activerecord (7.0.5)
61
+ activemodel (= 7.0.5)
62
+ activesupport (= 7.0.5)
63
+ activestorage (7.0.5)
64
+ actionpack (= 7.0.5)
65
+ activejob (= 7.0.5)
66
+ activerecord (= 7.0.5)
67
+ activesupport (= 7.0.5)
68
68
  marcel (~> 1.0)
69
69
  mini_mime (>= 1.1.0)
70
- activesupport (7.0.3)
70
+ activesupport (7.0.5)
71
71
  concurrent-ruby (~> 1.0, >= 1.0.2)
72
72
  i18n (>= 1.6, < 2)
73
73
  minitest (>= 5.1)
@@ -80,42 +80,40 @@ GEM
80
80
  break (0.40.0)
81
81
  builder (3.2.4)
82
82
  coderay (1.1.3)
83
- concurrent-ruby (1.1.10)
83
+ concurrent-ruby (1.2.2)
84
84
  crass (1.0.6)
85
- diff-lcs (1.4.4)
86
- digest (3.1.0)
87
- erubi (1.10.0)
88
- globalid (1.0.0)
85
+ date (3.3.3)
86
+ diff-lcs (1.5.0)
87
+ erubi (1.12.0)
88
+ globalid (1.1.0)
89
89
  activesupport (>= 5.0)
90
- i18n (1.10.0)
90
+ i18n (1.14.1)
91
91
  concurrent-ruby (~> 1.0)
92
- loofah (2.18.0)
92
+ loofah (2.21.3)
93
93
  crass (~> 1.0.2)
94
- nokogiri (>= 1.5.9)
95
- mail (2.7.1)
94
+ nokogiri (>= 1.12.0)
95
+ mail (2.8.1)
96
96
  mini_mime (>= 0.1.1)
97
+ net-imap
98
+ net-pop
99
+ net-smtp
97
100
  marcel (1.0.2)
98
101
  method_source (1.0.0)
99
102
  mini_mime (1.1.2)
100
- mini_portile2 (2.8.0)
101
- minitest (5.15.0)
102
- net-imap (0.2.3)
103
- digest
103
+ mini_portile2 (2.8.2)
104
+ minitest (5.18.1)
105
+ net-imap (0.3.6)
106
+ date
104
107
  net-protocol
105
- strscan
106
- net-pop (0.1.1)
107
- digest
108
+ net-pop (0.1.2)
108
109
  net-protocol
110
+ net-protocol (0.2.1)
109
111
  timeout
110
- net-protocol (0.1.3)
111
- timeout
112
- net-smtp (0.3.1)
113
- digest
112
+ net-smtp (0.3.3)
114
113
  net-protocol
115
- timeout
116
- nio4r (2.5.8)
117
- nokogiri (1.13.6)
118
- mini_portile2 (~> 2.8.0)
114
+ nio4r (2.5.9)
115
+ nokogiri (1.15.2)
116
+ mini_portile2 (~> 2.8.2)
119
117
  racc (~> 1.4)
120
118
  parallel (1.22.1)
121
119
  parser (3.1.2.0)
@@ -123,57 +121,58 @@ GEM
123
121
  pry (0.14.1)
124
122
  coderay (~> 1.1)
125
123
  method_source (~> 1.0)
126
- racc (1.6.0)
127
- rack (2.2.3.1)
128
- rack-test (1.1.0)
129
- rack (>= 1.0, < 3)
130
- rails (7.0.3)
131
- actioncable (= 7.0.3)
132
- actionmailbox (= 7.0.3)
133
- actionmailer (= 7.0.3)
134
- actionpack (= 7.0.3)
135
- actiontext (= 7.0.3)
136
- actionview (= 7.0.3)
137
- activejob (= 7.0.3)
138
- activemodel (= 7.0.3)
139
- activerecord (= 7.0.3)
140
- activestorage (= 7.0.3)
141
- activesupport (= 7.0.3)
124
+ racc (1.7.1)
125
+ rack (2.2.7)
126
+ rack-test (2.1.0)
127
+ rack (>= 1.3)
128
+ rails (7.0.5)
129
+ actioncable (= 7.0.5)
130
+ actionmailbox (= 7.0.5)
131
+ actionmailer (= 7.0.5)
132
+ actionpack (= 7.0.5)
133
+ actiontext (= 7.0.5)
134
+ actionview (= 7.0.5)
135
+ activejob (= 7.0.5)
136
+ activemodel (= 7.0.5)
137
+ activerecord (= 7.0.5)
138
+ activestorage (= 7.0.5)
139
+ activesupport (= 7.0.5)
142
140
  bundler (>= 1.15.0)
143
- railties (= 7.0.3)
141
+ railties (= 7.0.5)
144
142
  rails-dom-testing (2.0.3)
145
143
  activesupport (>= 4.2.0)
146
144
  nokogiri (>= 1.6)
147
- rails-html-sanitizer (1.4.2)
148
- loofah (~> 2.3)
149
- railties (7.0.3)
150
- actionpack (= 7.0.3)
151
- activesupport (= 7.0.3)
145
+ rails-html-sanitizer (1.6.0)
146
+ loofah (~> 2.21)
147
+ nokogiri (~> 1.14)
148
+ railties (7.0.5)
149
+ actionpack (= 7.0.5)
150
+ activesupport (= 7.0.5)
152
151
  method_source
153
152
  rake (>= 12.2)
154
153
  thor (~> 1.0)
155
154
  zeitwerk (~> 2.5)
156
155
  rainbow (3.1.1)
157
156
  rake (13.0.6)
158
- regexp_parser (2.4.0)
157
+ regexp_parser (2.5.0)
159
158
  rexml (3.2.5)
160
- rspec-core (3.10.1)
161
- rspec-support (~> 3.10.0)
162
- rspec-expectations (3.10.1)
159
+ rspec-core (3.12.0)
160
+ rspec-support (~> 3.12.0)
161
+ rspec-expectations (3.12.2)
163
162
  diff-lcs (>= 1.2.0, < 2.0)
164
- rspec-support (~> 3.10.0)
165
- rspec-mocks (3.10.2)
163
+ rspec-support (~> 3.12.0)
164
+ rspec-mocks (3.12.3)
166
165
  diff-lcs (>= 1.2.0, < 2.0)
167
- rspec-support (~> 3.10.0)
168
- rspec-rails (5.0.1)
169
- actionpack (>= 5.2)
170
- activesupport (>= 5.2)
171
- railties (>= 5.2)
172
- rspec-core (~> 3.10)
173
- rspec-expectations (~> 3.10)
174
- rspec-mocks (~> 3.10)
175
- rspec-support (~> 3.10)
176
- rspec-support (3.10.2)
166
+ rspec-support (~> 3.12.0)
167
+ rspec-rails (6.0.1)
168
+ actionpack (>= 6.1)
169
+ activesupport (>= 6.1)
170
+ railties (>= 6.1)
171
+ rspec-core (~> 3.11)
172
+ rspec-expectations (~> 3.11)
173
+ rspec-mocks (~> 3.11)
174
+ rspec-support (~> 3.11)
175
+ rspec-support (3.12.0)
177
176
  rubocop (1.29.1)
178
177
  parallel (~> 1.10)
179
178
  parser (>= 3.1.0.0)
@@ -183,26 +182,25 @@ GEM
183
182
  rubocop-ast (>= 1.17.0, < 2.0)
184
183
  ruby-progressbar (~> 1.7)
185
184
  unicode-display_width (>= 1.4.0, < 3.0)
186
- rubocop-ast (1.18.0)
185
+ rubocop-ast (1.19.1)
187
186
  parser (>= 3.1.1.0)
188
187
  rubocop-performance (1.13.3)
189
188
  rubocop (>= 1.7.0, < 2.0)
190
189
  rubocop-ast (>= 0.4.0)
191
190
  ruby-progressbar (1.11.0)
192
- sqlite3 (1.4.2)
191
+ sqlite3 (1.4.4)
193
192
  standard (1.12.1)
194
193
  rubocop (= 1.29.1)
195
194
  rubocop-performance (= 1.13.3)
196
- strscan (3.0.3)
197
- thor (1.2.1)
198
- timeout (0.3.0)
199
- tzinfo (2.0.4)
195
+ thor (1.2.2)
196
+ timeout (0.3.2)
197
+ tzinfo (2.0.6)
200
198
  concurrent-ruby (~> 1.0)
201
- unicode-display_width (2.1.0)
199
+ unicode-display_width (2.2.0)
202
200
  websocket-driver (0.7.5)
203
201
  websocket-extensions (>= 0.1.0)
204
202
  websocket-extensions (0.1.5)
205
- zeitwerk (2.5.4)
203
+ zeitwerk (2.6.8)
206
204
 
207
205
  PLATFORMS
208
206
  ruby
data/README.md CHANGED
@@ -1,6 +1,31 @@
1
- # Sensible Data Migrations for Rails
1
+ # monarch_migrate [![Build Status][ci-image]][ci] [![Gem Version][version-image]][version]
2
2
 
3
- ## Why
3
+ A library for Rails developers who are not willing to leave data migrations to chance.
4
+
5
+ <br />
6
+
7
+ ## Table of Contents
8
+
9
+ - [Rationale](#rationale)
10
+ - [Install](#install)
11
+ - [Usage](#usage)
12
+ - [Create a Data Migration](#create-a-data-migration)
13
+ - [Running Data Migrations](#running-data-migrations)
14
+ - [Display Status of Data Migrations](#display-status-of-data-migrations)
15
+ - [Reverting Data Migrations](#reverting-data-migrations)
16
+ - [Tasks in Data Migrations](#tasks-in-data-migrations)
17
+ - [Testing](#testing)
18
+ - [RSpec](#rspec)
19
+ - [TestUnit](#testunit)
20
+ - [Suggested Workflow](#suggested-workflow)
21
+ - [Known Issues and Limitations](#known-issues-and-limitations)
22
+ - [Trivia](#trivia)
23
+ - [Acknowledgments](#acknowledgments)
24
+ - [License](#license)
25
+
26
+
27
+ ## Rationale
28
+ <sup>[(Back to top)](#table-of-contents)</sup>
4
29
 
5
30
  <blockquote>
6
31
  <p>The main purpose of Rails' migration feature is to issue commands that modify the schema using a consistent process. Migrations can also be used to add or modify data. This is useful in an existing database that can't be destroyed and recreated, such as a production database.</p>
@@ -9,30 +34,35 @@
9
34
  </a>
10
35
  </blockquote>
11
36
 
12
- The motivation behind Rails' migration mechanism is schema modification. Using it
13
- for data changes in the database comes second. Yet, adding or modifying data via
14
- regular migrations can be problematic.
37
+ The motivation behind Rails' migration mechanism is schema modification.
38
+ Using it for data changes in the database comes second. Yet, adding or
39
+ modifying data via regular migrations can be problematic.
15
40
 
16
- The first issue is that application deployment now depends on the data migration
41
+ One issue is that application deployment now depends on the data migration
17
42
  to be completed. This may not be a problem with small databases but large
18
43
  databases with millions of records will respond with hanging or failed migrations.
19
44
 
20
45
  Another issue is that data migration files tend to stay in `db/migrate` for posterity.
21
46
  As a result, they will run whenever a developer sets up their local development environment.
22
- This is unnecessary for a pristine database. Especially when there are [scripts][2] to
47
+ This is unnecessary for a pristine database. Especially with [scripts][seed-scripts] to
23
48
  seed the correct data.
24
49
 
25
- The purpose of `monarch_migrate` is to solve the above issues by separating data from schema migrations.
50
+ The purpose of `monarch_migrate` is to solve the above issues and to:
51
+
52
+ - Provide a [uniform process](#suggested-workflow) for modifying data in the database.
53
+ - Separate data migrations from schema migrations.
54
+ - Allow automated testing of data migrations.
26
55
 
27
- It is assumed that:
56
+ It assumes that:
28
57
 
29
- - You run data migrations *only* on production and rely on seed [scripts][2] i.e. `dev:prime` for local development.
58
+ - You run data migrations only on production and rely on seed [scripts][seed-scripts] for local development.
30
59
  - You run data migrations manually.
31
- - You want to write tests your data migrations.
60
+ - You want to test data migrations in a thorough and automated manner.
32
61
 
33
62
 
34
63
 
35
64
  ## Install
65
+ <sup>[(Back to top)](#table-of-contents)</sup>
36
66
 
37
67
  Add the gem to your Gemfile:
38
68
 
@@ -48,45 +78,45 @@ After you install the gem, you need to run the generator:
48
78
  rails generate monarch_migrate:install
49
79
  ```
50
80
 
51
- The install generator creates a migration file that adds a `data_migration_records`
52
- table. It is where the gem keeps track of data migrations we have already ran.
81
+ The install generator creates a schema migration file that adds a `data_migration_records`
82
+ table. It is where the gem keeps track of data migrations we have ran.
53
83
 
84
+ Run the schema migration to create the table:
54
85
 
86
+ ```shell
87
+ rails db:migrate
88
+ ```
55
89
 
56
- ## Usage
57
90
 
58
- Data migrations have a similar structure to regular migrations in Rails. Files are
59
- put into `db/data_migrate` and follow the same naming pattern.
60
91
 
61
- Let's start with an example.
92
+ ## Usage
93
+ <sup>[(Back to top)](#table-of-contents)</sup>
62
94
 
63
- Suppose we have designed a system where users have first and last names. Time passes and
64
- it becomes clear this is [wrong][3]. Now, we want to put things right and come
65
- up with the following plan:
95
+ Data migrations have a similar structure to regular migrations in Rails.
96
+ Files follow the same naming pattern but reside in `db/data_migrate`.
66
97
 
67
- 1. Add a `name` column to `users` table to hold person's full name.
68
- 2. Adapt the `User` model and use a data migration to update existing records.
69
- 3. Drop `first_name` and `last_name` columns.
70
98
 
71
- To create the data migration to update existing user records, run:
99
+ ### Create a Data Migration
72
100
 
73
101
  ```shell
74
102
  rails generate data_migration backfill_users_name
75
103
  ```
76
104
 
77
- In contrast to regular migrations, there is no need to inherit any classes:
105
+ Edit the newly created file to define the data migration:
78
106
 
79
107
  ```ruby
80
108
  # db/data_migrate/20220605083010_backfill_users_name.rb
81
109
  ActiveRecord::Base.connection.execute(<<-SQL)
82
110
  UPDATE users SET name = concat(first_name, ' ', last_name) WHERE name IS NULL;
83
111
  SQL
84
-
85
- SearchIndex::RebuildJob.perform_later
86
112
  ```
87
113
 
88
- As seen above, it is plain ruby code where you can refer to any object you
89
- need. Each data migration runs in a separate transaction.
114
+ In contrast to regular migrations, there is no need to inherit any classes. It is plain
115
+ ruby code where you can refer to any object you need. If the database adapter supports
116
+ transactions, each data migration will automatically be wrapped in a separate transaction.
117
+
118
+
119
+ ### Running Data Migrations
90
120
 
91
121
  To run pending data migrations:
92
122
 
@@ -94,19 +124,52 @@ To run pending data migrations:
94
124
  rails data:migrate
95
125
  ```
96
126
 
97
- Or a specific version:
127
+ To run a specific version:
98
128
 
99
129
  ```shell
100
130
  rails data:migrate VERSION=20220605083010
101
131
  ```
102
132
 
103
133
 
134
+ ### Display Status of Data Migrations
135
+
136
+ You can see the status of data migrations with:
137
+
138
+ ```shell
139
+ rails data:migrate:status
140
+ ```
141
+
142
+ ### Reverting Data Migrations
143
+
144
+ Rollback functionality is not provided by design. Create another data migration instead.
145
+
146
+
147
+ ### Tasks in Data Migrations
148
+
149
+ After the data manipulation is complete, you may want to trigger a long running task.
150
+ Yet, there are [some pitfalls](#long-running-tasks-in-migrations) to be aware of.
151
+
152
+ To avoid them, use an after-commit callback:
153
+
154
+ ```ruby
155
+ # db/data_migrate/20220605083010_backfill_users_name.rb
156
+ ActiveRecord::Base.connection.execute(<<-SQL)
157
+ UPDATE users SET name = concat(first_name, ' ', last_name) WHERE name IS NULL;
158
+ SQL
159
+
160
+ after_commit do
161
+ SearchIndex::RebuildJob.perform_later
162
+ end
163
+ ```
164
+
165
+
104
166
  ## Testing
167
+ <sup>[(Back to top)](#table-of-contents)</sup>
105
168
 
106
- Testing data migrations can be the difference between simply rerunning
107
- the migration and having to recover from a data loss.
169
+ Testing data migrations can be the difference between rerunning
170
+ the migration and having to recover from a data loss. This is
171
+ why `monarch_migrate` includes test helpers for both RSpec and TestUnit.
108
172
 
109
- This is why `monarch_migrate` includes test helpers for both RSpec and TestUnit.
110
173
 
111
174
  ### RSpec
112
175
 
@@ -128,7 +191,7 @@ describe "20220605083010_backfill_users_name", type: :data_migration do
128
191
  # or if you're using FactoryBot:
129
192
  # user = create(:user, first_name: "Guybrush", last_name: "Threepwood", name: nil)
130
193
 
131
- expect(subject).to change { user.reload.name }.to("Guybrush Threepwood")
194
+ expect { subject }.to change { user.reload.name }.to("Guybrush Threepwood")
132
195
  end
133
196
 
134
197
  it "does not assign name to already migrated users" do
@@ -136,7 +199,7 @@ describe "20220605083010_backfill_users_name", type: :data_migration do
136
199
  # or if you're using FactoryBot:
137
200
  # user = create(:user, first_name: "", last_name: "", name: "Guybrush Threepwood")
138
201
 
139
- expect(subject).not_to change { user.reload.name }
202
+ expect { subject }.not_to change { user.reload.name }
140
203
  end
141
204
 
142
205
  context "when the user has no last name" do
@@ -145,7 +208,7 @@ describe "20220605083010_backfill_users_name", type: :data_migration do
145
208
  # or if you're using FactoryBot:
146
209
  # user = create(:user, first_name: "Guybrush", last_name: nil, name: nil)
147
210
 
148
- expect(subject).to change { user.reload.name }.to("Guybrush")
211
+ expect { subject }.to change { user.reload.name }.to("Guybrush")
149
212
  end
150
213
  end
151
214
 
@@ -200,21 +263,30 @@ class BackfillUsersNameTest < MonarchMigrate::TestCase
200
263
  end
201
264
  ```
202
265
 
203
- Data migrations become obsolete, once the data manipulation successfully completes.
204
- So are the corresponding tests. These will fail after database columns are dropped
205
- e.g. `first_name` and `last_name`.
266
+ In certain cases, it makes sense to also test with manually running the data migration
267
+ against a production clone of the database.
268
+
269
+
270
+
271
+ ## Suggested Workflow
272
+ <sup>[(Back to top)](#table-of-contents)</sup>
273
+
274
+ Data migrations become obsolete, once the data manipulation successfully completes. The same applies for the corresponding tests.
206
275
 
207
- One solution is to use the following development workflow:
276
+ The suggested development workflow is:
208
277
 
209
- 1. Implement the data migration using TDD.
278
+ 1. Implement the data migration. Use TDD, if appropriate.
210
279
  1. Commit, push and wait for CI to pass.
211
280
  1. Request review from peers.
212
281
  1. Once approved, remove the test files in a consecutive commit and push again.
213
282
  1. Merge into trunk.
214
283
 
215
- This will also keep the test files in repo's history for posterity.
284
+ This will keep the test files in repo's history for posterity.
285
+
286
+
216
287
 
217
288
  ## Known Issues and Limitations
289
+ <sup>[(Back to top)](#table-of-contents)</sup>
218
290
 
219
291
  The following issues and limitations are not necessary inherent to `monarch_migrate`.
220
292
  Some are innate to migrations in general.
@@ -233,14 +305,20 @@ Typically, data migrations relate closely to business models. In an ideal Rails
233
305
  data manipulations would depend on model logic to enforce validation, conform to
234
306
  business rules, etc. Hence, it is very tempting to use ActiveRecord models in migrations.
235
307
 
236
- Here is a regular Rails migration for our example:
308
+ Here a regular Rails migration:
237
309
 
238
310
  ```ruby
239
311
  # db/migrate/20220605083010_backfill_users_name.rb
240
- def up
241
- User.all.each do |user|
242
- user.name = "#{user.first_name} #{user.last_name}"
243
- user.save
312
+ class BackfillUsersName < ActiveRecord::Migration[7.0]
313
+ def up
314
+ User.all.each do |user|
315
+ user.name = "#{user.first_name} #{user.last_name}"
316
+ user.save
317
+ end
318
+ end
319
+
320
+ def down
321
+ # ...
244
322
  end
245
323
  end
246
324
  ```
@@ -256,9 +334,15 @@ To avoid issues 1-3, we can rewrite the migration to:
256
334
 
257
335
  ```ruby
258
336
  # db/migrate/20220605083010_backfill_users_name.rb
259
- def up
260
- User.where(name: nil).find_each do |user|
261
- user.update_column(:name, "#{user.first_name} #{user.last_name}")
337
+ class BackfillUsersName < ActiveRecord::Migration[7.0]
338
+ def up
339
+ User.where(name: nil).find_each do |user|
340
+ user.update_column(:name, "#{user.first_name} #{user.last_name}")
341
+ end
342
+ end
343
+
344
+ def down
345
+ # ...
262
346
  end
263
347
  end
264
348
  ```
@@ -266,10 +350,10 @@ end
266
350
  Unfortunately, with regular Rails migrations we will still face issue 4.
267
351
 
268
352
  To avoid it, we need to separate data from schema migrations and not run data
269
- migrations locally. With seed [scripts][2], there is no need to run them anyway.
353
+ migrations locally. With seed [scripts][seed-scripts], there is no need to run them anyway.
270
354
 
271
355
  Keep the above in mind when referencing ActiveRecord models in data migrations. Ideally,
272
- limit their use and do as much processing as possible in Postgres.
356
+ limit their use and do as much processing as possible in the database.
273
357
 
274
358
 
275
359
  ### Long-running Tasks in Migrations
@@ -278,11 +362,16 @@ As mentioned, each data migration runs in a separate transaction.
278
362
  A long-running task within a migration keeps the transaction open for
279
363
  the duration of the task. As a result, the migration may hang or fail.
280
364
 
281
- To avoid this, run such tasks asynchronously.
365
+ You could run such tasks asynchronously (i.e. in a background job) but the task may start
366
+ before the transaction commits. This is a [known issue][sync-issue]. In addition,
367
+ a database under load can suffer from longer commit times.
368
+
369
+ See [Tasks in Data Migrations](#tasks-in-data-migrations).
282
370
 
283
371
 
284
372
 
285
373
  ## Trivia
374
+ <sup>[(Back to top)](#table-of-contents)</sup>
286
375
 
287
376
  One of the most impressive migrations on Earth is the multi-generational
288
377
  round trip of the monarch butterfly.
@@ -302,27 +391,34 @@ Genetically speaking, this is an incredible data migration!
302
391
 
303
392
 
304
393
 
305
- ## See Also
306
-
307
- Alternative gems
308
-
309
- - https://github.com/OffgridElectric/rails-data-migrations
310
- - https://github.com/ilyakatz/data-migrate
311
- - https://github.com/jasonfb/nonschema_migrations
394
+ ## Acknowledgments
395
+ <sup>[(Back to top)](#table-of-contents)</sup>
312
396
 
313
397
  Articles
314
398
 
315
399
  - [Data Migrations in Rails](https://thoughtbot.com/blog/data-migrations-in-rails)
400
+ - [Decoupling database migrations from server startup: why and how](https://pythonspeed.com/articles/schema-migrations-server-startup/)
316
401
  - [Zero downtime migrations: 500 million rows](https://www.honeybadger.io/blog/zero-downtime-migrations-of-large-databases-using-rails-postgres-and-redis/)
317
402
  - [Three Useful Data Migration Patterns for Rails](https://www.ombulabs.com/blog/rails/data-migrations/three-useful-data-migrations-patterns-in-rails.html)
318
403
  - [Ruby on Rails Model Patterns and Anti-patterns](https://blog.appsignal.com/2020/11/18/rails-model-patterns-and-anti-patterns.html)
319
404
  - [Rails Migrations with Zero Downtime](https://www.cloudbees.com/blog/rails-migrations-zero-downtime)
320
405
 
406
+ Alternative gems
407
+
408
+ - https://github.com/OffgridElectric/rails-data-migrations
409
+ - https://github.com/ilyakatz/data-migrate
410
+ - https://github.com/jasonfb/nonschema_migrations
411
+
321
412
 
322
413
 
323
414
  ## License
415
+ <sup>[(Back to top)](#table-of-contents)</sup>
324
416
 
325
417
  The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
326
418
 
327
- [2]: https://thoughtbot.com/blog/priming-the-pump
328
- [3]: https://www.kalzumeus.com/2010/06/17/falsehoods-programmers-believe-about-names/
419
+ [ci-image]: https://github.com/lunohodov/monarch/actions/workflows/ci.yml/badge.svg
420
+ [ci]: https://github.com/lunohodov/monarch/actions/workflows/ci.yml
421
+ [seed-scripts]: https://thoughtbot.com/blog/priming-the-pump
422
+ [sync-issue]: https://github.com/mperham/sidekiq/wiki/FAQ#why-am-i-seeing-a-lot-of-cant-find-modelname-with-id12345-errors-with-sidekiq
423
+ [version-image]: https://badge.fury.io/rb/monarch_migrate.svg
424
+ [version]: https://badge.fury.io/rb/monarch_migrate
data/lib/generators/monarch_migrate/install/install_generator.rb CHANGED
@@ -1,16 +1,14 @@
1
1
  require "rails/generators"
2
- require "rails/generators/active_record"
2
+ require "rails/generators/active_record/migration"
3
3
 
4
4
  module MonarchMigrate
5
5
  module Generators
6
6
  class InstallGenerator < Rails::Generators::Base
7
- include Rails::Generators::Migration
7
+ include ActiveRecord::Generators::Migration
8
8
 
9
- source_root File.expand_path("../templates", __FILE__)
9
+ class_option :database, type: :string, aliases: %i[--db], desc: "The database for your migration. By default, the current environment's primary database is used."
10
10
 
11
- def self.next_migration_number(dir)
12
- ActiveRecord::Generators::Base.next_migration_number(dir)
13
- end
11
+ source_root File.expand_path("templates", __dir__)
14
12
 
15
13
  def create_monarch_migrate_migration
16
14
  return if migration_exists?
@@ -18,17 +16,19 @@ module MonarchMigrate
18
16
 
19
17
  migration_template(
20
18
  "create_data_migration_records.rb.erb",
21
- "db/migrate/create_data_migration_records.rb",
19
+ "#{db_migrate_path}/create_data_migration_records.rb",
22
20
  migration_version: migration_version
23
21
  )
24
22
  end
25
23
 
26
- def migration_version
27
- "[#{ActiveRecord::Migration.current_version}]"
28
- end
24
+ no_commands do
25
+ def migration_version
26
+ "[#{ActiveRecord::Migration.current_version}]"
27
+ end
29
28
 
30
- def migration_table_name
31
- MigrationRecord.table_name
29
+ def migration_table_name
30
+ MigrationRecord.table_name
31
+ end
32
32
  end
33
33
 
34
34
  private
data/lib/generators/rails/data_migration/data_migration_generator.rb CHANGED
@@ -1,3 +1,5 @@
1
+ require "rails/generators"
2
+ require "rails/generators/migration"
1
3
  require "rails/generators/active_record"
2
4
 
3
5
  module Rails
@@ -5,14 +7,20 @@ module Rails
5
7
  class DataMigrationGenerator < Rails::Generators::NamedBase
6
8
  include ActiveRecord::Generators::Migration
7
9
 
8
- source_root File.expand_path("../templates", __FILE__)
10
+ source_root File.expand_path("templates", __dir__)
11
+
12
+ def self.next_migration_number(_)
13
+ ActiveRecord::Generators::Base.next_migration_number(
14
+ MonarchMigrate.data_migrations_path
15
+ )
16
+ end
9
17
 
10
18
  def create_data_migration
11
19
  validate_file_name!
12
20
 
13
21
  migration_template(
14
22
  "data_migration.rb.erb",
15
- File.join(MonarchMigrate.data_migrations_path, "#{file_name}.rb")
23
+ File.join(MonarchMigrate.migrator.path, "#{file_name}.rb")
16
24
  )
17
25
  end
18
26
 
data/lib/generators/rspec/data_migration_generator.rb CHANGED
@@ -1,36 +1,38 @@
1
+ require "rails/generators"
2
+ require "monarch_migrate/migration"
3
+
1
4
  module Rspec
2
5
  module Generators
3
- class DataMigrationGenerator < ::Rails::Generators::NamedBase
6
+ class DataMigrationGenerator < Rails::Generators::NamedBase
7
+ include MonarchMigrate::Migration::Lookup
8
+
4
9
  source_root File.expand_path("templates", __dir__)
5
10
 
6
11
  def create_data_migration_test
7
- unless data_migration
8
- say "Expecting a data migration matching *#{data_migration_pattern} but none found. Aborting..."
9
- return
12
+ if spec_file_name
13
+ template("data_migration_spec.rb.erb", File.join(destination_dir, spec_file_name))
10
14
  end
11
-
12
- template(
13
- "data_migration_spec.rb.erb",
14
- File.join("spec/data_migrations", "#{described_class}_spec.rb")
15
- )
16
15
  end
17
16
 
18
17
  private
19
18
 
20
19
  def described_class
21
- File.basename(data_migration.filename, ".rb")
20
+ File.basename(spec_file_name, ".*")
22
21
  end
23
22
 
24
- def data_migration_pattern
25
- "#{file_name}.rb"
23
+ def destination_dir
24
+ File.join(destination_root, "spec/data_migrations")
26
25
  end
27
26
 
28
- def data_migration
29
- @data_migration ||=
30
- MonarchMigrate.migrator
31
- .migrations
32
- .reverse
33
- .find { |m| m.filename.ends_with?(data_migration_pattern) }
27
+ def spec_file_name
28
+ @spec_file_name ||=
29
+ if behavior == :invoke
30
+ name = migration_exists?(MonarchMigrate.migrator.path, file_name)
31
+ File.basename(name, ".*") << "_spec.rb" if name
32
+ else
33
+ name = migration_exists?(destination_dir, "#{file_name}_spec")
34
+ File.basename(name) if name
35
+ end
34
36
  end
35
37
  end
36
38
  end
data/lib/generators/test_unit/data_migration_generator.rb CHANGED
@@ -1,33 +1,36 @@
1
+ require "rails/generators"
2
+ require "monarch_migrate/migration"
3
+
1
4
  module TestUnit
2
5
  module Generators
3
- class DataMigrationGenerator < ::Rails::Generators::NamedBase
6
+ class DataMigrationGenerator < Rails::Generators::NamedBase
7
+ include MonarchMigrate::Migration::Lookup
8
+
4
9
  source_root File.expand_path("templates", __dir__)
5
10
 
6
11
  check_class_collision suffix: "Test"
7
12
 
8
- def create_data_migration_test
9
- unless data_migration
10
- say "Expecting a data migration matching *#{data_migration_pattern} but none found. Aborting..."
11
- return
13
+ def create_data_migration_test_file
14
+ if test_file_name
15
+ template "unit_test.rb.erb", File.join(destination_dir, test_file_name)
12
16
  end
13
-
14
- prefix = File.basename(data_migration.filename, ".rb")
15
-
16
- template "unit_test.rb.erb", File.join("test/data_migrations", "#{prefix}_test.rb")
17
17
  end
18
18
 
19
19
  private
20
20
 
21
- def data_migration_pattern
22
- "#{file_name}.rb"
21
+ def test_file_name
22
+ @test_file_name ||=
23
+ if behavior == :invoke
24
+ name = migration_exists?(MonarchMigrate.migrator.path, file_name)
25
+ File.basename(name, ".*") << "_test.rb" if name
26
+ else
27
+ name = migration_exists?(destination_dir, "#{file_name}_test")
28
+ File.basename(name) if name
29
+ end
23
30
  end
24
31
 
25
- def data_migration
26
- @data_migration ||=
27
- MonarchMigrate.migrator
28
- .migrations
29
- .reverse
30
- .find { |m| m.filename.ends_with?(data_migration_pattern) }
32
+ def destination_dir
33
+ File.join(destination_root, "test/data_migrations")
31
34
  end
32
35
  end
33
36
  end
data/lib/monarch_migrate/migration.rb CHANGED
@@ -2,8 +2,19 @@
2
2
 
3
3
  module MonarchMigrate
4
4
  class Migration
5
+ module Lookup
6
+ def migration_lookup_at(dirname)
7
+ Dir.glob("#{dirname}/[0-9]*_*.rb")
8
+ end
9
+
10
+ def migration_exists?(dirname, file_name)
11
+ migration_lookup_at(dirname).grep(/\d+_#{file_name}.rb$/).first
12
+ end
13
+ end
14
+
5
15
  def initialize(path)
6
16
  @path = path.to_s
17
+ @after_commit_callback = nil
7
18
  end
8
19
 
9
20
  def filename
@@ -11,7 +22,7 @@ module MonarchMigrate
11
22
  end
12
23
 
13
24
  def name
14
- File.basename(path, ".rb").match(/^[0-9]+_(.*)$/)[1].humanize
25
+ File.basename(path, ".rb").delete_prefix("#{version}_").humanize
15
26
  end
16
27
 
17
28
  def version
@@ -22,27 +33,34 @@ module MonarchMigrate
22
33
  !MigrationRecord.exists?(version: version)
23
34
  end
24
35
 
25
- def run(io = nil)
26
- io ||= File.open(File::NULL, "w")
36
+ def after_commit(&block)
37
+ @after_commit_callback = block
38
+ end
27
39
 
40
+ def run
28
41
  ActiveRecord::Base.connection.transaction do
29
- io.puts "Running data migration #{version}: #{name}"
42
+ puts "Running data migration #{version}: #{name}"
30
43
 
31
44
  begin
32
45
  instance_eval File.read(path), path
33
46
  MigrationRecord.create!(version: version)
34
- io.puts "Migration complete"
47
+ puts "Migration complete"
35
48
  rescue => e
36
- io.puts "Migration failed due to #{e}"
37
- raise ActiveRecord::Rollback
49
+ puts "Migration failed due to #{e}"
50
+ # Deliberately raising ActiveRecord::Rollback does not
51
+ # pass on the exception and the callback will be triggered
52
+ raise
38
53
  end
39
54
 
40
- io.puts
55
+ puts
41
56
  end
57
+
58
+ after_commit_callback&.call
42
59
  end
43
60
 
44
61
  private
45
62
 
46
63
  attr_reader :path
64
+ attr_reader :after_commit_callback
47
65
  end
48
66
  end
data/lib/monarch_migrate/migrator.rb CHANGED
@@ -20,14 +20,12 @@ module MonarchMigrate
20
20
  migrations.select(&:pending?)
21
21
  end
22
22
 
23
- def run(io = nil)
24
- io ||= File.open(File::NULL, "w")
25
-
23
+ def run
26
24
  if pending_migrations.any?
27
- io.puts "Running #{pending_migrations.size} data migrations"
28
- pending_migrations.sort_by(&:version).each { |m| m.run(io) }
25
+ puts "Running #{pending_migrations.size} data migrations"
26
+ pending_migrations.sort_by(&:version).each(&:run)
29
27
  else
30
- io.puts "No data migrations pending"
28
+ puts "No data migrations pending"
31
29
  []
32
30
  end
33
31
  end
data/lib/monarch_migrate/tasks.rake CHANGED
@@ -3,7 +3,7 @@ require "monarch_migrate"
3
3
  namespace :data do
4
4
  desc "Run pending data migrations, or a single version specified by environment variable VERSION"
5
5
  task migrate: :environment do
6
- MonarchMigrate.migrator.run($stdout)
6
+ MonarchMigrate.migrator.run
7
7
  end
8
8
 
9
9
  namespace :migrate do
data/lib/monarch_migrate/testing.rb CHANGED
@@ -1,11 +1,13 @@
1
1
  require "active_support/core_ext/string"
2
- require "stringio"
2
+ require "active_support/testing/stream"
3
3
 
4
4
  module MonarchMigrate
5
5
  module Testing
6
6
  MigrationRunError = Class.new(RuntimeError)
7
7
 
8
8
  module Helpers
9
+ include ::ActiveSupport::Testing::Stream
10
+
9
11
  def data_migration
10
12
  @data_migration ||=
11
13
  begin
@@ -17,9 +19,8 @@ module MonarchMigrate
17
19
  end
18
20
 
19
21
  def run_data_migration
20
- output = StringIO.new
21
- data_migration.run(output)
22
- output.string.tap { ensure_no_error(_1) }
22
+ output = capture(:stdout) { data_migration.run }
23
+ ensure_no_error(output)
23
24
  end
24
25
 
25
26
  protected
data/lib/monarch_migrate/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module MonarchMigrate
2
- VERSION = "0.5.0"
2
+ VERSION = "0.7.0"
3
3
  end
data/monarch_migrate.gemspec CHANGED
@@ -6,10 +6,10 @@ Gem::Specification.new do |spec|
6
6
  spec.authors = ["Yanko Ivanov"]
7
7
  spec.email = ["yanko.ivanov@gmail.com"]
8
8
 
9
- spec.summary = "Sensible data migrations for Rails"
9
+ spec.summary = "A library for Rails developers who are not willing to leave data migrations to chance."
10
10
  spec.homepage = "https://github.com/lunohodov/monarch"
11
11
  spec.license = "MIT"
12
- spec.required_ruby_version = Gem::Requirement.new(">= 2.7.3")
12
+ spec.required_ruby_version = Gem::Requirement.new(">= 3.0")
13
13
 
14
14
  spec.metadata["homepage_uri"] = spec.homepage
15
15
  spec.metadata["source_code_uri"] = "https://github.com/lunohodov/monarch"
@@ -20,5 +20,5 @@ Gem::Specification.new do |spec|
20
20
  `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features|gemfiles|tmp)/}) }
21
21
  end
22
22
 
23
- spec.add_dependency("rails", ">= 5.2.0")
23
+ spec.add_dependency("rails", ">= 6.0")
24
24
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: monarch_migrate
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.0
4
+ version: 0.7.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Yanko Ivanov
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-06-18 00:00:00.000000000 Z
11
+ date: 2023-06-29 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rails
@@ -16,14 +16,14 @@ dependencies:
16
16
  requirements:
17
17
  - - ">="
18
18
  - !ruby/object:Gem::Version
19
- version: 5.2.0
19
+ version: '6.0'
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - ">="
25
25
  - !ruby/object:Gem::Version
26
- version: 5.2.0
26
+ version: '6.0'
27
27
  description:
28
28
  email:
29
29
  - yanko.ivanov@gmail.com
@@ -79,15 +79,16 @@ required_ruby_version: !ruby/object:Gem::Requirement
79
79
  requirements:
80
80
  - - ">="
81
81
  - !ruby/object:Gem::Version
82
- version: 2.7.3
82
+ version: '3.0'
83
83
  required_rubygems_version: !ruby/object:Gem::Requirement
84
84
  requirements:
85
85
  - - ">="
86
86
  - !ruby/object:Gem::Version
87
87
  version: '0'
88
88
  requirements: []
89
- rubygems_version: 3.1.6
89
+ rubygems_version: 3.4.10
90
90
  signing_key:
91
91
  specification_version: 4
92
- summary: Sensible data migrations for Rails
92
+ summary: A library for Rails developers who are not willing to leave data migrations
93
+ to chance.
93
94
  test_files: []