monarch_migrate 0.4.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6b59ad47f996564cc02e0577133692329e9a3b7052e2b0e0268c713cbf44b563
-  data.tar.gz: 98ab757d6e2fcdba24451c5f1816555e7ba6d65ba5c47d1dc0ab4cc94d562a7d
+  metadata.gz: d70afb71a32546dd38ae87d4e39956c97bd095c56cc957266a159edb4cbfc3cb
+  data.tar.gz: 2b95f318d045045dba0cb712bc0aa647d95717688e2cb258702e9bea9c398d6f
 SHA512:
-  metadata.gz: 296572f26ef2ac623e3409c4048f18ada537588100ada9b42caf08d73d9bd251c5ff9111b1079de512ff95892c3263177393268b860a6624617849f4c15ca49a
-  data.tar.gz: bb9a83dfb41ce4649ae45b58d4aaea1faeb9e47057f10b0161926a472164f15a2485faab8c4448058391f3daf5a1cd19c5c26cad8dd8a4377a7204f93bb4dd18
+  metadata.gz: f97b549b57586af15371dfadba856b9d506989d210a76ae21cc85e20d0eff097ca58fd26a1901760c8755d0b45be6583bfabc2f9f71e288cd0949d1ab98df150
+  data.tar.gz: 6f773f40eb0028229a75aa58055b3765580261f4c50d21bd01ab64ab5b5f8f001129626f442d022825d9038729a6bdf2519dcff425b31febeed681230257347c

data/.github/workflows/ci.yml

@@ -45,5 +45,16 @@ jobs:
       - name: "Reset app database"
         run: bundle exec rake fake:db:reset
 
-      - name: "Run test"
-        run: bundle exec rake
+      - name: "Run tests without acceptance"
+        run: |
+          bundle exec rake test
+
+      - name: "Run acceptance tests for RSpec"
+        run: |
+          bundle add rspec-rails
+          bundle exec rake TEST=test/acceptance/testing_test.rb TESTOPTS="--name='test_rspec_tests'"
+
+      - name: "Run acceptance tests for TestUnit"
+        run: |
+          bundle remove rspec-rails
+          bundle exec rake TEST=test/acceptance/testing_test.rb TESTOPTS="--name='test_test_unit_tests'"

data/.gitignore

@@ -72,11 +72,13 @@ fabric.properties
 .idea/caches/build_file_checksums.ser
 
 # Migration generated during tests
-/test/generators/monarch_migrate/tmp
+/test/generators/**/tmp
 
 # Ignore gem compile directory
 /pkg
 
+/gemfiles/*.lock
+
 # Ignore bundler config.
 /.bundle
 

data/Appraisals

@@ -1,11 +1,14 @@
 appraise "rails_5.2" do
-  gem "railties", "~> 5.2"
+  gem "rails", "~> 5.2"
 end
 
 appraise "rails_6.1" do
-  gem "railties", "~> 6.1"
+  gem "rails", "~> 6.1"
+  gem "net-smtp", require: false
+  gem "net-imap", require: false
+  gem "net-pop", require: false
 end
 
 appraise "rails_7.0" do
-  gem "railties", "~> 7.0"
+  gem "rails", "~> 7.0"
 end

data/Gemfile

@@ -2,3 +2,11 @@ source "https://rubygems.org"
 
 # Specify your gem's dependencies in *.gemspec
 gemspec name: "monarch_migrate"
+
+gem "appraisal"
+gem "break"
+gem "pry", require: false
+gem "rake"
+gem "rspec-rails"
+gem "sqlite3"
+gem "standard"

data/Gemfile.lock

@@ -1,32 +1,73 @@
 PATH
   remote: .
   specs:
-    monarch_migrate (0.4.0)
-      activerecord (>= 5.2.0)
-      railties (>= 5.2.0)
+    monarch_migrate (0.6.0)
+      rails (>= 5.2.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    actionpack (7.0.3)
-      actionview (= 7.0.3)
-      activesupport (= 7.0.3)
+    actioncable (7.0.3.1)
+      actionpack (= 7.0.3.1)
+      activesupport (= 7.0.3.1)
+      nio4r (~> 2.0)
+      websocket-driver (>= 0.6.1)
+    actionmailbox (7.0.3.1)
+      actionpack (= 7.0.3.1)
+      activejob (= 7.0.3.1)
+      activerecord (= 7.0.3.1)
+      activestorage (= 7.0.3.1)
+      activesupport (= 7.0.3.1)
+      mail (>= 2.7.1)
+      net-imap
+      net-pop
+      net-smtp
+    actionmailer (7.0.3.1)
+      actionpack (= 7.0.3.1)
+      actionview (= 7.0.3.1)
+      activejob (= 7.0.3.1)
+      activesupport (= 7.0.3.1)
+      mail (~> 2.5, >= 2.5.4)
+      net-imap
+      net-pop
+      net-smtp
+      rails-dom-testing (~> 2.0)
+    actionpack (7.0.3.1)
+      actionview (= 7.0.3.1)
+      activesupport (= 7.0.3.1)
       rack (~> 2.0, >= 2.2.0)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actionview (7.0.3)
-      activesupport (= 7.0.3)
+    actiontext (7.0.3.1)
+      actionpack (= 7.0.3.1)
+      activerecord (= 7.0.3.1)
+      activestorage (= 7.0.3.1)
+      activesupport (= 7.0.3.1)
+      globalid (>= 0.6.0)
+      nokogiri (>= 1.8.5)
+    actionview (7.0.3.1)
+      activesupport (= 7.0.3.1)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.1, >= 1.2.0)
-    activemodel (7.0.3)
-      activesupport (= 7.0.3)
-    activerecord (7.0.3)
-      activemodel (= 7.0.3)
-      activesupport (= 7.0.3)
-    activesupport (7.0.3)
+    activejob (7.0.3.1)
+      activesupport (= 7.0.3.1)
+      globalid (>= 0.3.6)
+    activemodel (7.0.3.1)
+      activesupport (= 7.0.3.1)
+    activerecord (7.0.3.1)
+      activemodel (= 7.0.3.1)
+      activesupport (= 7.0.3.1)
+    activestorage (7.0.3.1)
+      actionpack (= 7.0.3.1)
+      activejob (= 7.0.3.1)
+      activerecord (= 7.0.3.1)
+      activesupport (= 7.0.3.1)
+      marcel (~> 1.0)
+      mini_mime (>= 1.1.0)
+    activesupport (7.0.3.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -36,42 +77,103 @@ GEM
       rake
       thor (>= 0.14.0)
     ast (2.4.2)
+    break (0.40.0)
     builder (3.2.4)
+    coderay (1.1.3)
     concurrent-ruby (1.1.10)
     crass (1.0.6)
+    diff-lcs (1.5.0)
+    digest (3.1.0)
     erubi (1.10.0)
-    i18n (1.10.0)
+    globalid (1.0.0)
+      activesupport (>= 5.0)
+    i18n (1.12.0)
       concurrent-ruby (~> 1.0)
     loofah (2.18.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
+    mail (2.7.1)
+      mini_mime (>= 0.1.1)
+    marcel (1.0.2)
     method_source (1.0.0)
-    minitest (5.15.0)
-    nokogiri (1.13.6-x86_64-darwin)
+    mini_mime (1.1.2)
+    mini_portile2 (2.8.0)
+    minitest (5.16.2)
+    net-imap (0.2.3)
+      digest
+      net-protocol
+      strscan
+    net-pop (0.1.1)
+      digest
+      net-protocol
+      timeout
+    net-protocol (0.1.3)
+      timeout
+    net-smtp (0.3.1)
+      digest
+      net-protocol
+      timeout
+    nio4r (2.5.8)
+    nokogiri (1.13.7)
+      mini_portile2 (~> 2.8.0)
       racc (~> 1.4)
     parallel (1.22.1)
     parser (3.1.2.0)
       ast (~> 2.4.1)
+    pry (0.14.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
     racc (1.6.0)
-    rack (2.2.3.1)
-    rack-test (1.1.0)
-      rack (>= 1.0, < 3)
+    rack (2.2.4)
+    rack-test (2.0.2)
+      rack (>= 1.3)
+    rails (7.0.3.1)
+      actioncable (= 7.0.3.1)
+      actionmailbox (= 7.0.3.1)
+      actionmailer (= 7.0.3.1)
+      actionpack (= 7.0.3.1)
+      actiontext (= 7.0.3.1)
+      actionview (= 7.0.3.1)
+      activejob (= 7.0.3.1)
+      activemodel (= 7.0.3.1)
+      activerecord (= 7.0.3.1)
+      activestorage (= 7.0.3.1)
+      activesupport (= 7.0.3.1)
+      bundler (>= 1.15.0)
+      railties (= 7.0.3.1)
     rails-dom-testing (2.0.3)
       activesupport (>= 4.2.0)
       nokogiri (>= 1.6)
-    rails-html-sanitizer (1.4.2)
+    rails-html-sanitizer (1.4.3)
       loofah (~> 2.3)
-    railties (7.0.3)
-      actionpack (= 7.0.3)
-      activesupport (= 7.0.3)
+    railties (7.0.3.1)
+      actionpack (= 7.0.3.1)
+      activesupport (= 7.0.3.1)
       method_source
       rake (>= 12.2)
       thor (~> 1.0)
       zeitwerk (~> 2.5)
     rainbow (3.1.1)
     rake (13.0.6)
-    regexp_parser (2.4.0)
+    regexp_parser (2.5.0)
     rexml (3.2.5)
+    rspec-core (3.11.0)
+      rspec-support (~> 3.11.0)
+    rspec-expectations (3.11.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.11.0)
+    rspec-mocks (3.11.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.11.0)
+    rspec-rails (5.1.2)
+      actionpack (>= 5.2)
+      activesupport (>= 5.2)
+      railties (>= 5.2)
+      rspec-core (~> 3.10)
+      rspec-expectations (~> 3.10)
+      rspec-mocks (~> 3.10)
+      rspec-support (~> 3.10)
+    rspec-support (3.11.0)
     rubocop (1.29.1)
       parallel (~> 1.10)
       parser (>= 3.1.0.0)
@@ -81,30 +183,37 @@ GEM
       rubocop-ast (>= 1.17.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.18.0)
+    rubocop-ast (1.19.1)
       parser (>= 3.1.1.0)
     rubocop-performance (1.13.3)
       rubocop (>= 1.7.0, < 2.0)
       rubocop-ast (>= 0.4.0)
     ruby-progressbar (1.11.0)
-    sqlite3 (1.4.2)
+    sqlite3 (1.4.4)
     standard (1.12.1)
       rubocop (= 1.29.1)
       rubocop-performance (= 1.13.3)
+    strscan (3.0.3)
     thor (1.2.1)
+    timeout (0.3.0)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
-    unicode-display_width (2.1.0)
-    zeitwerk (2.5.4)
+    unicode-display_width (2.2.0)
+    websocket-driver (0.7.5)
+      websocket-extensions (>= 0.1.0)
+    websocket-extensions (0.1.5)
+    zeitwerk (2.6.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   appraisal
-  minitest
+  break
   monarch_migrate!
+  pry
   rake
+  rspec-rails
   sqlite3
   standard
 

data/README.md

@@ -1,4 +1,31 @@
-# Sensible Data Migrations for Rails
+# monarch_migrate [![Build Status][ci-image]][ci] [![Gem Version][version-image]][version]
+
+A library for Rails developers who are not willing to leave data migrations to chance.
+
+<br />
+
+## Table of Contents
+
+- [Rationale](#rationale)
+- [Install](#install)
+- [Usage](#usage)
+  - [Create a Data Migration](#create-a-data-migration)
+  - [Running Data Migrations](#running-data-migrations)
+  - [Display Status of Data Migrations](#display-status-of-data-migrations)
+  - [Reverting Data Migrations](#reverting-data-migrations)
+  - [Background Tasks in Data Migrations](#background-tasks-in-data-migrations)
+- [Testing](#testing)
+  - [RSpec](#rspec)
+  - [TestUnit](#testunit)
+- [Suggested Workflow](#suggested-workflow)
+- [Known Issues and Limitations](#known-issues-and-limitations)
+- [Trivia](#trivia)
+- [Acknowledgments](#acknowledgments)
+- [License](#license)
+
+
+## Rationale
+<sup>[(Back to top)](#table-of-contents)</sup>
 
 <blockquote>
   <p>The main purpose of Rails' migration feature is to issue commands that modify the schema using a consistent process. Migrations can also be used to add or modify data. This is useful in an existing database that can't be destroyed and recreated, such as a production database.</p>
@@ -7,31 +34,35 @@
   </a>
 </blockquote>
 
-The motivation behind Rails' migration mechanism is schema modification. Using it
-for data changes in the database comes second.
-
-Yet, adding of modifying data via regular migrations can be problematic.
+The motivation behind Rails' migration mechanism is schema modification.
+Using it for data changes in the database comes second. Yet, adding or
+modifying data via regular migrations can be problematic.
 
-The first issue is that application deployment now depends on the data migration
+One issue is that application deployment now depends on the data migration
 to be completed. This may not be a problem with small databases but large
 databases with millions of records will respond with hanging or failed migrations.
 
-Another issue is that data migration files usually stay in `db/migrate` for posterity.
+Another issue is that data migration files tend to stay in `db/migrate` for posterity.
 As a result, they will run whenever a developer sets up their local development environment.
-This is unnecessary for a pristine database. Especially so when there are [scripts][2] to
+This is unnecessary for a pristine database. Especially with [scripts][seed-scripts] to
 seed the correct data.
 
-In addition, using ActiveRecord models in migrations has its [quirks](#using-activerecord-models-in-migrations).
+The purpose of `monarch_migrate` is to solve the above issues and to:
 
-The purpose of `monarch_migrate` is to solve the above issues with separating data
-from schema migrations.
+- Provide a [uniform process](#suggested-workflow) for modifying data in the database.
+- Separate data migrations from schema migrations.
+- Allow automated testing of data migrations.
 
-This library assumes that:
+It assumes that:
 
-- You run data migrations *only* on production and rely on seed [scripts][2] i.e. `dev:prime` for local development.
+- You run data migrations only on production and rely on seed [scripts][seed-scripts] for local development.
 - You run data migrations manually.
+- You want to test data migrations in a thorough and automated manner.
+
+
 
 ## Install
+<sup>[(Back to top)](#table-of-contents)</sup>
 
 Add the gem to your Gemfile:
 
@@ -41,40 +72,51 @@ gem "monarch_migrate"
 
 Run the bundle command to install it.
 
-After you install MonarchMigrate, you need to run the generator:
+After you install the gem, you need to run the generator:
 
 ```shell
 rails generate monarch_migrate:install
 ```
 
-The above generates a schema migration which adds a table for keeping track
-of run data migrations.
+The install generator creates a schema migration file that adds a `data_migration_records`
+table. It is where the gem keeps track of data migrations we have ran.
+
+Run the schema migration to create the table:
+
+```shell
+rails db:migrate
+```
+
 
 
 ## Usage
+<sup>[(Back to top)](#table-of-contents)</sup>
 
-Data migrations in MonarchMigrate have a similar structure to regular migrations
-in Rails. Files are put into `db/data_migrate` and follow the same naming pattern.
+Data migrations have a similar structure to regular migrations in Rails.
+Files follow the same naming pattern but reside in `db/data_migrate`.
 
-To create a new data migration, run:
+
+### Create a Data Migration
 
 ```shell
-rails generate monarch_migrate:data_migration downcase_usernames
+rails generate data_migration backfill_users_name
 ```
 
-In contrast to regular migrations, there is no need to inherit any classes:
+Edit the newly created file to define the data migration:
 
 ```ruby
-# db/data_migrate/20220605083010_downcase_usernames.rb
+# db/data_migrate/20220605083010_backfill_users_name.rb
 ActiveRecord::Base.connection.execute(<<-SQL)
-  UPDATE users SET username = lower(username);
+  UPDATE users SET name = concat(first_name, ' ', last_name) WHERE name IS NULL;
 SQL
-
-SearchIndex.rebuild
 ```
 
-As seen above, it is plain ruby code where you can refer to any object you
-need. MonarchMigrate will run each migration in a separate transaction.
+In contrast to regular migrations, there is no need to inherit any classes. It is plain
+ruby code where you can refer to any object you need. If the database adapter supports
+transactions, each data migration will automatically be wrapped in a separate transaction.
+
+
+### Running Data Migrations
 
 To run pending data migrations:
 
@@ -82,18 +124,175 @@ To run pending data migrations:
 rails data:migrate
 ```
 
-Or a specific version:
+To run a specific version:
 
 ```shell
 rails data:migrate VERSION=20220605083010
 ```
 
+
+### Display Status of Data Migrations
+
+You can see the status of data migrations with:
+
+```shell
+rails data:migrate:status
+```
+
+### Reverting Data Migrations
+
+Rollback functionality is not provided by design. Create another data migration instead.
+
+
+### Background Tasks in Data Migrations
+
+After the data manipulation is complete, you may want to trigger a long running task.
+Yet, there are [some pitfalls](#long-running-tasks-in-migrations) to be aware of.
+
+To avoid them, use an after-commit callback:
+
+```ruby
+# db/data_migrate/20220605083010_backfill_users_name.rb
+ActiveRecord::Base.connection.execute(<<-SQL)
+  UPDATE users SET name = concat(first_name, ' ', last_name) WHERE name IS NULL;
+SQL
+
+after_commit do
+  SearchIndex::RebuildJob.perform_later
+end
+```
+
+
+## Testing
+<sup>[(Back to top)](#table-of-contents)</sup>
+
+Testing data migrations can be the difference between rerunning
+the migration and having to recover from a data loss. This is
+why `monarch_migrate` includes test helpers for both RSpec and TestUnit.
+
+
+### RSpec
+
+For `rspec`, add the following line to your `spec/rails_helper.rb`:
+
+```ruby
+require "monarch_migrate/rspec"
+```
+
+Then:
+
+```ruby
+# spec/data_migrations/20220605083010_backfill_users_name_spec.rb
+describe "20220605083010_backfill_users_name", type: :data_migration do
+  subject { run_data_migration }
+
+  it "assigns user's name" do
+    user = users(:without_name_migrated)
+    # or if you're using FactoryBot:
+    # user = create(:user, first_name: "Guybrush", last_name: "Threepwood", name: nil)
+
+    expect { subject }.to change { user.reload.name }.to("Guybrush Threepwood")
+  end
+
+  it "does not assign name to already migrated users" do
+    user = users(:with_name_migrated)
+    # or if you're using FactoryBot:
+    # user = create(:user, first_name: "", last_name: "", name: "Guybrush Threepwood")
+
+    expect { subject }.not_to change { user.reload.name }
+  end
+
+  context "when the user has no last name" do
+    it "does not leave a trailing space" do
+      user = users(:without_name_migrated)
+      # or if you're using FactoryBot:
+      # user = create(:user, first_name: "Guybrush", last_name: nil, name: nil)
+
+      expect { subject }.to change { user.reload.name }.to("Guybrush")
+    end
+  end
+
+  # And so on ...
+end
+```
+
+### TestUnit
+
+For `test_unit`, add this line to your `test/test_helper.rb`:
+
+```ruby
+require "monarch_migrate/test_unit"
+```
+
+Then:
+
+```ruby
+# test/data_migrations/20220605083010_backfill_users_name_test.rb
+class BackfillUsersNameTest < MonarchMigrate::TestCase
+  def test_assigns_users_name
+    user = users(:without_name_migrated)
+    # or if you're using FactoryBot:
+    # user = create(:user, first_name: "Guybrush", last_name: "Threepwood", name: nil)
+
+    run_data_migration
+
+    assert_equal "Guybrush Threepwood", user.reload.name
+  end
+
+  def test_does_not_assign_name_to_alredy_migrated_users
+    user = users(:with_name_migrated)
+    # or if you're using FactoryBot:
+    # user = create(:user, first_name: "", last_name: "", name: "Guybrush Threepwood")
+
+    run_data_migration
+
+    assert_equal "Guybrush Threepwood", user.reload.name
+  end
+
+  def test_does_not_leave_trailing_space_when_user_has_no_last_name
+    user = users(:without_name_migrated)
+    # or if you're using FactoryBot:
+    # user = create(:user, first_name: "Guybrush", last_name: nil, name: nil)
+
+    run_data_migration
+
+    assert_equal "Guybrush", user.reload.name
+  end
+
+  # And so on ...
+end
+```
+
+In certain cases, it makes sense to also test with manually running the data migration
+against a production clone of the database.
+
+
+
+## Suggested Workflow
+<sup>[(Back to top)](#table-of-contents)</sup>
+
+Data migrations become obsolete, once the data manipulation successfully completes. The same applies for the corresponding tests.
+
+The suggested development workflow is:
+
+1. Implement the data migration. Use TDD, if appropriate.
+1. Commit, push and wait for CI to pass.
+1. Request review from peers.
+1. Once approved, remove the test files in a consecutive commit and push again.
+1. Merge into trunk.
+
+This will keep the test files in repo's history for posterity.
+
+
+
 ## Known Issues and Limitations
+<sup>[(Back to top)](#table-of-contents)</sup>
 
-The following issues and limitations are not necessary inherent to MonarchMigrate.
+The following issues and limitations are not necessary inherent to `monarch_migrate`.
 Some are innate to migrations in general.
 
-### Using ActiveRecord Models in Migrations
+
+### Using Models in Migrations
 
 <blockquote>
   <p>The Active Record way claims that intelligence belongs in your models, not in the database.</p>
@@ -106,15 +305,7 @@ Typically, data migrations relate closely to business models. In an ideal Rails
 data manipulations would depend on model logic to enforce validation, conform to
 business rules, etc. Hence, it is very tempting to use ActiveRecord models in migrations.
 
-Suppose we have designed a system where users have first and last names. Time passes and
-it becomes clear this is [wrong][3]. Now, we want to put things right and come
-up with the following plan:
-
-1. Add a `name` column to `users` table to hold the entire name of a person.
-2. Change the `User` model and use a data migration to update existing records.
-3. Drop `first_name` and `last_name` columns.
-
-A regular Rails migration for updating existing records may look something like this:
+Here a regular Rails migration:
 
 ```ruby
 # db/migrate/20220605083010_backfill_users_name.rb
@@ -147,33 +338,28 @@ end
 Unfortunately, with regular Rails migrations we will still face issue 4.
 
 To avoid it, we need to separate data from schema migrations and not run data
-migrations locally. With seed [scripts][2], there is no need to run them anyway.
+migrations locally. With seed [scripts][seed-scripts], there is no need to run them anyway.
 
 Keep the above in mind when referencing ActiveRecord models in data migrations. Ideally,
-limit their use and do as much processing as possible in Postgres.
+limit their use and do as much processing as possible in the database.
 
 
 ### Long-running Tasks in Migrations
 
-As mentioned above, MonarchMigrate (similar to Rails) runs each migration in a separate
-transaction. A long-running task within a migration will keep the transaction open for
+As mentioned, each data migration runs in a separate transaction.
+A long-running task within a migration keeps the transaction open for
 the duration of the task. As a result, the migration may hang or fail.
 
-To avoid this, we can run such tasks asynchronously.
+You could run such tasks asynchronously (i.e. in a background job) but the task may start
+before the transaction commits. This is a [known issue][sync-issue]. In addition,
+a database under load can suffer from longer commit times.
 
-Back to the previous example:
+See [Background Tasks in Data Migrations](#background-tasks-in-data-migrations).
 
-```ruby
-# db/data_migrate/20220605083010_downcase_usernames.rb
-ActiveRecord::Base.connection.execute(<<-SQL)
-  UPDATE users SET username = lower(username);
-SQL
-
-SearchIndex::RebuildJob.perform_later
-```
 
 
 ## Trivia
+<sup>[(Back to top)](#table-of-contents)</sup>
 
 One of the most impressive migrations on Earth is the multi-generational
 round trip of the monarch butterfly.
@@ -185,19 +371,16 @@ The population cycles through three to five generations to reach their
 destination. In the end, a new generation of butterflies complete the
 journey their great-great-great-grandparents started.
 
-It is still a mystery to scientists how the new generations know where to go,
+It is a mystery to scientists how the new generations know where to go,
 but they appear to navigate using a combination of the Earth's magnetic field
 and the position of the sun.
 
-Genetically speaking, this is a truly incredible data migration!
+Genetically speaking, this is an incredible data migration!
 
 
-## See Also
 
-Alternative gems
-
-- [nonschema_migrations](https://github.com/jasonfb/nonschema_migrations) - Exactly like schema migrations but for data.
-- [data-migrate](https://github.com/ilyakatz/data-migrate) - A gem to run data migrations alongside schema migrations.
+## Acknowledgments
+<sup>[(Back to top)](#table-of-contents)</sup>
 
 Articles
 
@@ -207,10 +390,22 @@ Articles
 - [Ruby on Rails Model Patterns and Anti-patterns](https://blog.appsignal.com/2020/11/18/rails-model-patterns-and-anti-patterns.html)
 - [Rails Migrations with Zero Downtime](https://www.cloudbees.com/blog/rails-migrations-zero-downtime)
 
+Alternative gems
+
+- https://github.com/OffgridElectric/rails-data-migrations
+- https://github.com/ilyakatz/data-migrate
+- https://github.com/jasonfb/nonschema_migrations
+
+
 
 ## License
+<sup>[(Back to top)](#table-of-contents)</sup>
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
 
-[2]: https://thoughtbot.com/blog/priming-the-pump
-[3]: https://www.kalzumeus.com/2010/06/17/falsehoods-programmers-believe-about-names/
+[ci-image]: https://github.com/lunohodov/monarch/actions/workflows/ci.yml/badge.svg
+[ci]: https://github.com/lunohodov/monarch/actions/workflows/ci.yml
+[seed-scripts]: https://thoughtbot.com/blog/priming-the-pump
+[sync-issue]: https://github.com/mperham/sidekiq/wiki/FAQ#why-am-i-seeing-a-lot-of-cant-find-modelname-with-id12345-errors-with-sidekiq
+[version-image]: https://badge.fury.io/rb/monarch_migrate.svg
+[version]: https://badge.fury.io/rb/monarch_migrate

data/Rakefile

@@ -1,15 +1,24 @@
 require "bundler/gem_tasks"
 require "rake/testtask"
+require "standard/rake"
 
 namespace :fake do
   require_relative "test/fake/application"
   Fake::Application.load_tasks
 end
 
-Rake::TestTask.new(:test) do |t|
+Rake::TestTask.new("test") do |t|
   t.libs << "test"
   t.libs << "lib"
   t.test_files = FileList["test/**/*_test.rb"]
+    .exclude("test/acceptance/**/*_test.rb")
+    .exclude("test/fixtures/**/*_test.rb")
 end
 
-task default: :test
+Rake::TestTask.new("test:acceptance") do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/acceptance/*_test.rb"]
+end
+
+task default: %w[test test:acceptance]

data/lib/generators/monarch_migrate/install/USAGE

@@ -2,4 +2,4 @@ Description:
   Generate a schema migration to create the table used for data migrations
 
 Examples:
-    rails generate monarch_migrate:install
+  rails generate monarch_migrate:install

data/lib/generators/monarch_migrate/install/install_generator.rb

@@ -28,7 +28,7 @@ module MonarchMigrate
       end
 
       def migration_table_name
-        MonarchMigrate.data_migrations_table_name
+        MigrationRecord.table_name
       end
 
       private

data/lib/generators/rails/data_migration/USAGE

@@ -0,0 +1,5 @@
+Description:
+  Generate a new data migration
+
+Examples:
+  rails generate data_migration assign_name_to_users

data/lib/generators/{monarch_migrate → rails}/data_migration/data_migration_generator.rb

@@ -1,6 +1,6 @@
 require "rails/generators/active_record"
 
-module MonarchMigrate
+module Rails
   module Generators
     class DataMigrationGenerator < Rails::Generators::NamedBase
       include ActiveRecord::Generators::Migration
@@ -16,6 +16,8 @@ module MonarchMigrate
         )
       end
 
+      hook_for :test_framework, as: :data_migration
+
       private
 
       def validate_file_name!

data/lib/generators/rspec/USAGE

@@ -0,0 +1,12 @@
+Description:
+  Generate a spec for existing data migration.
+
+Examples:
+  Given a data migration `db/data_migrate/200010101011_downcase_users_name.rb`:
+    rails generate rspec:data_migration 200010101011_downcase_users_name
+
+  Alternatively, you can skip the version:
+    rails generate rspec:data_migration downcase_users_name
+
+  This will create:
+    spec/data_migrations/200010101011_downcase_users_name_spec.rb

data/lib/generators/rspec/data_migration_generator.rb

@@ -0,0 +1,37 @@
+module Rspec
+  module Generators
+    class DataMigrationGenerator < ::Rails::Generators::NamedBase
+      source_root File.expand_path("templates", __dir__)
+
+      def create_data_migration_test
+        unless data_migration
+          say "Expecting a data migration matching *#{data_migration_pattern} but none found. Aborting..."
+          return
+        end
+
+        template(
+          "data_migration_spec.rb.erb",
+          File.join("spec/data_migrations", "#{described_class}_spec.rb")
+        )
+      end
+
+      private
+
+      def described_class
+        File.basename(data_migration.filename, ".rb")
+      end
+
+      def data_migration_pattern
+        "#{file_name}.rb"
+      end
+
+      def data_migration
+        @data_migration ||=
+          MonarchMigrate.migrator
+            .migrations
+            .reverse
+            .find { |m| m.filename.ends_with?(data_migration_pattern) }
+      end
+    end
+  end
+end

data/lib/generators/rspec/templates/data_migration_spec.rb.erb

@@ -0,0 +1,5 @@
+require "rails_helper"
+
+RSpec.describe "<%= described_class %>", type: :data_migration do
+  pending "add some examples to (or delete) #{__FILE__}"
+end

data/lib/generators/test_unit/USAGE

@@ -0,0 +1,12 @@
+Description:
+  Generate a spec for existing data migration.
+
+Examples:
+  Given a data migration `db/data_migrate/200010101011_downcase_users_name.rb`:
+    rails generate test_unit:data_migration 200010101011_downcase_users_name
+
+  Alternatively, you can skip the version:
+    rails generate test_unit:data_migration downcase_users_name
+
+  This will create:
+    test/data_migrations/200010101011_downcase_users_name_test.rb

data/lib/generators/test_unit/data_migration_generator.rb

@@ -0,0 +1,34 @@
+module TestUnit
+  module Generators
+    class DataMigrationGenerator < ::Rails::Generators::NamedBase
+      source_root File.expand_path("templates", __dir__)
+
+      check_class_collision suffix: "Test"
+
+      def create_data_migration_test
+        unless data_migration
+          say "Expecting a data migration matching *#{data_migration_pattern} but none found. Aborting..."
+          return
+        end
+
+        prefix = File.basename(data_migration.filename, ".rb")
+
+        template "unit_test.rb.erb", File.join("test/data_migrations", "#{prefix}_test.rb")
+      end
+
+      private
+
+      def data_migration_pattern
+        "#{file_name}.rb"
+      end
+
+      def data_migration
+        @data_migration ||=
+          MonarchMigrate.migrator
+            .migrations
+            .reverse
+            .find { |m| m.filename.ends_with?(data_migration_pattern) }
+      end
+    end
+  end
+end

data/lib/generators/test_unit/templates/unit_test.rb.erb

@@ -0,0 +1,7 @@
+require "test_helper"
+
+class <%= class_name %>Test < MonarchMigrate::TestCase
+  # test "the truth" do
+  #   assert true
+  # end
+end

data/lib/monarch_migrate/migration.rb

@@ -4,6 +4,7 @@ module MonarchMigrate
   class Migration
     def initialize(path)
       @path = path.to_s
+      @after_commit_callback = nil
     end
 
     def filename
@@ -22,27 +23,34 @@ module MonarchMigrate
       !MigrationRecord.exists?(version: version)
     end
 
-    def run(io = nil)
-      io ||= File.open(File::NULL, "w")
+    def after_commit(&block)
+      @after_commit_callback = block
+    end
 
+    def run
       ActiveRecord::Base.connection.transaction do
-        io.puts "Running data migration #{version}: #{name}"
+        puts "Running data migration #{version}: #{name}"
 
         begin
           instance_eval File.read(path), path
           MigrationRecord.create!(version: version)
-          io.puts "Migration complete"
+          puts "Migration complete"
         rescue => e
-          io.puts "Migration failed due to #{e}"
-          raise ActiveRecord::Rollback
+          puts "Migration failed due to #{e}"
+          # Deliberately raising ActiveRecord::Rollback does not
+          # pass on the exception and the callback will be triggered
+          raise
         end
 
-        io.puts
+        puts
       end
+
+      after_commit_callback&.call
     end
 
     private
 
     attr_reader :path
+    attr_reader :after_commit_callback
   end
 end

data/lib/monarch_migrate/migrator.rb

@@ -20,14 +20,12 @@ module MonarchMigrate
       migrations.select(&:pending?)
     end
 
-    def run(io = nil)
-      io ||= File.open(File::NULL, "w")
-
+    def run
       if pending_migrations.any?
-        io.puts "Running #{pending_migrations.size} data migrations"
-        pending_migrations.sort_by(&:version).each { |m| m.run(io) }
+        puts "Running #{pending_migrations.size} data migrations"
+        pending_migrations.sort_by(&:version).each(&:run)
       else
-        io.puts "No data migrations pending"
+        puts "No data migrations pending"
         []
       end
     end

data/lib/monarch_migrate/railtie.rb

@@ -1,3 +1,5 @@
+require "rails"
+
 module MonarchMigrate
   class Railtie < Rails::Railtie
     railtie_name :monarch_migrate

data/lib/monarch_migrate/rspec.rb

@@ -0,0 +1,5 @@
+require "monarch_migrate/testing"
+
+RSpec.configure do |config|
+  config.include MonarchMigrate::Testing::RSpec, type: :data_migration
+end

data/lib/monarch_migrate/tasks.rake

@@ -3,7 +3,7 @@ require "monarch_migrate"
 namespace :data do
   desc "Run pending data migrations, or a single version specified by environment variable VERSION"
   task migrate: :environment do
-    MonarchMigrate.migrator.run($stdout)
+    MonarchMigrate.migrator.run
   end
 
   namespace :migrate do

data/lib/monarch_migrate/test_unit.rb

@@ -0,0 +1,5 @@
+require "monarch_migrate/testing"
+
+class MonarchMigrate::TestCase < ActiveSupport::TestCase
+  include MonarchMigrate::Testing::TestUnit
+end

data/lib/monarch_migrate/testing.rb

@@ -0,0 +1,88 @@
+require "active_support/core_ext/string"
+require "active_support/testing/stream"
+
+module MonarchMigrate
+  module Testing
+    MigrationRunError = Class.new(RuntimeError)
+
+    module Helpers
+      include ::ActiveSupport::Testing::Stream
+
+      def data_migration
+        @data_migration ||=
+          begin
+            filename = data_migration_basename
+            migrator
+              .migrations
+              .find(method(:not_found)) { |m| m.filename.ends_with?(filename) }
+          end
+      end
+
+      def run_data_migration
+        output = capture(:stdout) { data_migration.run }
+        ensure_no_error(output)
+      end
+
+      protected
+
+      def data_migration_basename
+        raise NotImplementedError
+      end
+
+      def data_migration_failed(message)
+        raise NotImplementedError
+      end
+
+      def migrator
+        ::MonarchMigrate.migrator
+      end
+
+      def not_found
+        path = File.join(migrator.path, "*_#{data_migration_basename}")
+        raise <<~MSG
+          \n\r
+          Can not find data migration at path: #{path}
+          \n\r
+        MSG
+      end
+
+      def ensure_no_error(str)
+        if %r{Migration failed}.match?(str)
+          data_migration_failed(<<-CMD)
+            Failed running data migration: #{data_migration.filename}
+            \n\r
+            Output:
+            \n\r
+            \n\r
+            #{str}
+          CMD
+        end
+      end
+    end
+
+    module RSpec
+      include Helpers
+
+      def data_migration_basename
+        spec_path = ::RSpec.current_example.metadata[:file_path]
+        File.basename(spec_path).sub(/_spec\.rb$/, ".rb")
+      end
+
+      def data_migration_failed(message)
+        fail message
+      end
+    end
+
+    module TestUnit
+      include Helpers
+
+      def data_migration_basename
+        File.basename(self.class.name.underscore).sub(/_test$/, ".rb")
+      end
+
+      def data_migration_failed(message)
+        flunk message
+      end
+    end
+  end
+end

data/lib/monarch_migrate/version.rb

@@ -1,3 +1,3 @@
 module MonarchMigrate
-  VERSION = "0.4.0"
+  VERSION = "0.6.0"
 end

data/lib/monarch_migrate.rb

@@ -15,6 +15,7 @@ module MonarchMigrate
   end
 end
 
+require "rails"
 require "monarch_migrate/migration"
 require "monarch_migrate/migration_record"
 require "monarch_migrate/migrator"

data/monarch_migrate.gemspec

@@ -20,12 +20,5 @@ Gem::Specification.new do |spec|
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features|gemfiles|tmp)/}) }
   end
 
-  spec.add_dependency("activerecord", ">= 5.2.0")
-  spec.add_dependency("railties", ">= 5.2.0")
-
-  spec.add_development_dependency("appraisal")
-  spec.add_development_dependency("minitest")
-  spec.add_development_dependency("rake")
-  spec.add_development_dependency("sqlite3")
-  spec.add_development_dependency("standard")
+  spec.add_dependency("rails", ">= 5.2.0")
 end

metadata

@@ -1,17 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: monarch_migrate
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Yanko Ivanov
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-06-06 00:00:00.000000000 Z
+date: 2022-10-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: activerecord
+  name: rails
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -24,90 +24,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 5.2.0
-- !ruby/object:Gem::Dependency
-  name: railties
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 5.2.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 5.2.0
-- !ruby/object:Gem::Dependency
-  name: appraisal
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: minitest
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: sqlite3
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: standard
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description:
 email:
 - yanko.ivanov@gmail.com
@@ -126,18 +42,27 @@ files:
 - Rakefile
 - bin/setup
 - db/schema.rb
-- lib/generators/monarch_migrate/data_migration/USAGE
-- lib/generators/monarch_migrate/data_migration/data_migration_generator.rb
-- lib/generators/monarch_migrate/data_migration/templates/data_migration.rb.erb
 - lib/generators/monarch_migrate/install/USAGE
 - lib/generators/monarch_migrate/install/install_generator.rb
 - lib/generators/monarch_migrate/install/templates/create_data_migration_records.rb.erb
+- lib/generators/rails/data_migration/USAGE
+- lib/generators/rails/data_migration/data_migration_generator.rb
+- lib/generators/rails/data_migration/templates/data_migration.rb.erb
+- lib/generators/rspec/USAGE
+- lib/generators/rspec/data_migration_generator.rb
+- lib/generators/rspec/templates/data_migration_spec.rb.erb
+- lib/generators/test_unit/USAGE
+- lib/generators/test_unit/data_migration_generator.rb
+- lib/generators/test_unit/templates/unit_test.rb.erb
 - lib/monarch_migrate.rb
 - lib/monarch_migrate/migration.rb
 - lib/monarch_migrate/migration_record.rb
 - lib/monarch_migrate/migrator.rb
 - lib/monarch_migrate/railtie.rb
+- lib/monarch_migrate/rspec.rb
 - lib/monarch_migrate/tasks.rake
+- lib/monarch_migrate/test_unit.rb
+- lib/monarch_migrate/testing.rb
 - lib/monarch_migrate/version.rb
 - monarch_migrate.gemspec
 homepage: https://github.com/lunohodov/monarch

data/lib/generators/monarch_migrate/data_migration/USAGE

@@ -1,5 +0,0 @@
-Description:
-  Generate a new data migration
-
-Examples:
-    rails generate monarch_migrate:data_migration assign_name_to_users