monarch_migrate 0.4.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 6b59ad47f996564cc02e0577133692329e9a3b7052e2b0e0268c713cbf44b563
+  data.tar.gz: 98ab757d6e2fcdba24451c5f1816555e7ba6d65ba5c47d1dc0ab4cc94d562a7d
+SHA512:
+  metadata.gz: 296572f26ef2ac623e3409c4048f18ada537588100ada9b42caf08d73d9bd251c5ff9111b1079de512ff95892c3263177393268b860a6624617849f4c15ca49a
+  data.tar.gz: bb9a83dfb41ce4649ae45b58d4aaea1faeb9e47057f10b0161926a472164f15a2485faab8c4448058391f3daf5a1cd19c5c26cad8dd8a4377a7204f93bb4dd18

data/.github/workflows/ci.yml

@@ -0,0 +1,49 @@
+name: "CI Tests"
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  test:
+    name: "Ruby ${{ matrix.ruby }}, Rails ${{ matrix.gemfile }}"
+
+    runs-on: 'ubuntu-latest'
+
+    strategy:
+      fail-fast: false
+      matrix:
+        gemfile:
+          - "5.2"
+          - "6.1"
+          - "7.0"
+        ruby:
+          - "2.7.3"
+          - "3.0.0"
+          - "3.1.0"
+        exclude:
+          - gemfile: "5.2"
+            ruby: "3.0.0"
+          - gemfile: "5.2"
+            ruby: "3.1.0"
+
+    env:
+      BUNDLE_GEMFILE: gemfiles/rails_${{ matrix.gemfile }}.gemfile
+      RAILS_ENV: test
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: "Install Ruby ${{ matrix.ruby  }}"
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+
+      - name: "Reset app database"
+        run: bundle exec rake fake:db:reset
+
+      - name: "Run test"
+        run: bundle exec rake

data/.gitignore

@@ -0,0 +1,113 @@
+# See https://help.github.com/articles/ignoring-files for more about ignoring files.
+#
+# If you find yourself ignoring temporary files generated by your text editor
+# or operating system, you probably want to add a global ignore instead:
+#   git config --global core.excludesfile '~/.gitignore_global'
+
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# Android studio 3.1+ serialized cache file
+.idea/caches/build_file_checksums.ser
+
+# Migration generated during tests
+/test/generators/monarch_migrate/tmp
+
+# Ignore gem compile directory
+/pkg
+
+# Ignore bundler config.
+/.bundle
+
+# Ignore the default SQLite database.
+/db/*.sqlite3
+/db/*.sqlite3-journal
+
+# Ignore all logfiles and tempfiles.
+/log/*
+/tmp/*
+!/log/.keep
+!/tmp/.keep
+
+# Ignore uploaded files in development
+/storage/*
+!/storage/.keep
+
+/node_modules
+/yarn-error.log
+
+/public/assets
+.byebug_history
+
+# Ignore master key for decrypting credentials and more.
+/config/master.key
+
+.foreman
+
+/public/packs
+/public/packs-test
+/node_modules
+/yarn-error.log
+yarn-debug.log*
+.yarn-integrity

data/.ruby-version

@@ -0,0 +1 @@
+2.7.3

data/Appraisals

@@ -0,0 +1,11 @@
+appraise "rails_5.2" do
+  gem "railties", "~> 5.2"
+end
+
+appraise "rails_6.1" do
+  gem "railties", "~> 6.1"
+end
+
+appraise "rails_7.0" do
+  gem "railties", "~> 7.0"
+end

data/Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in *.gemspec
+gemspec name: "monarch_migrate"

data/Gemfile.lock

@@ -0,0 +1,112 @@
+PATH
+  remote: .
+  specs:
+    monarch_migrate (0.4.0)
+      activerecord (>= 5.2.0)
+      railties (>= 5.2.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    actionpack (7.0.3)
+      actionview (= 7.0.3)
+      activesupport (= 7.0.3)
+      rack (~> 2.0, >= 2.2.0)
+      rack-test (>= 0.6.3)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.2.0)
+    actionview (7.0.3)
+      activesupport (= 7.0.3)
+      builder (~> 3.1)
+      erubi (~> 1.4)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.1, >= 1.2.0)
+    activemodel (7.0.3)
+      activesupport (= 7.0.3)
+    activerecord (7.0.3)
+      activemodel (= 7.0.3)
+      activesupport (= 7.0.3)
+    activesupport (7.0.3)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+    appraisal (2.4.1)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    ast (2.4.2)
+    builder (3.2.4)
+    concurrent-ruby (1.1.10)
+    crass (1.0.6)
+    erubi (1.10.0)
+    i18n (1.10.0)
+      concurrent-ruby (~> 1.0)
+    loofah (2.18.0)
+      crass (~> 1.0.2)
+      nokogiri (>= 1.5.9)
+    method_source (1.0.0)
+    minitest (5.15.0)
+    nokogiri (1.13.6-x86_64-darwin)
+      racc (~> 1.4)
+    parallel (1.22.1)
+    parser (3.1.2.0)
+      ast (~> 2.4.1)
+    racc (1.6.0)
+    rack (2.2.3.1)
+    rack-test (1.1.0)
+      rack (>= 1.0, < 3)
+    rails-dom-testing (2.0.3)
+      activesupport (>= 4.2.0)
+      nokogiri (>= 1.6)
+    rails-html-sanitizer (1.4.2)
+      loofah (~> 2.3)
+    railties (7.0.3)
+      actionpack (= 7.0.3)
+      activesupport (= 7.0.3)
+      method_source
+      rake (>= 12.2)
+      thor (~> 1.0)
+      zeitwerk (~> 2.5)
+    rainbow (3.1.1)
+    rake (13.0.6)
+    regexp_parser (2.4.0)
+    rexml (3.2.5)
+    rubocop (1.29.1)
+      parallel (~> 1.10)
+      parser (>= 3.1.0.0)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.17.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.18.0)
+      parser (>= 3.1.1.0)
+    rubocop-performance (1.13.3)
+      rubocop (>= 1.7.0, < 2.0)
+      rubocop-ast (>= 0.4.0)
+    ruby-progressbar (1.11.0)
+    sqlite3 (1.4.2)
+    standard (1.12.1)
+      rubocop (= 1.29.1)
+      rubocop-performance (= 1.13.3)
+    thor (1.2.1)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    unicode-display_width (2.1.0)
+    zeitwerk (2.5.4)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  appraisal
+  minitest
+  monarch_migrate!
+  rake
+  sqlite3
+  standard
+
+BUNDLED WITH
+   2.1.4

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Y.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,216 @@
+# Sensible Data Migrations for Rails
+
+<blockquote>
+  <p>The main purpose of Rails' migration feature is to issue commands that modify the schema using a consistent process. Migrations can also be used to add or modify data. This is useful in an existing database that can't be destroyed and recreated, such as a production database.</p>
+  <a href="https://guides.rubyonrails.org/active_record_migrations.html#migrations-and-seed-data">
+    <sup>–Migrations and Seed Data, Rails Guide for Active Record Migrations</sup>
+  </a>
+</blockquote>
+
+The motivation behind Rails' migration mechanism is schema modification. Using it
+for data changes in the database comes second.
+
+Yet, adding of modifying data via regular migrations can be problematic.
+
+The first issue is that application deployment now depends on the data migration
+to be completed. This may not be a problem with small databases but large
+databases with millions of records will respond with hanging or failed migrations.
+
+Another issue is that data migration files usually stay in `db/migrate` for posterity.
+As a result, they will run whenever a developer sets up their local development environment.
+This is unnecessary for a pristine database. Especially so when there are [scripts][2] to
+seed the correct data.
+
+In addition, using ActiveRecord models in migrations has its [quirks](#using-activerecord-models-in-migrations).
+
+The purpose of `monarch_migrate` is to solve the above issues with separating data
+from schema migrations.
+
+This library assumes that:
+
+- You run data migrations *only* on production and rely on seed [scripts][2] i.e. `dev:prime` for local development.
+- You run data migrations manually.
+
+## Install
+
+Add the gem to your Gemfile:
+
+```ruby
+gem "monarch_migrate"
+```
+
+Run the bundle command to install it.
+
+After you install MonarchMigrate, you need to run the generator:
+
+```shell
+rails generate monarch_migrate:install
+```
+
+The above generates a schema migration which adds a table for keeping track
+of run data migrations.
+
+
+## Usage
+
+Data migrations in MonarchMigrate have a similar structure to regular migrations
+in Rails. Files are put into `db/data_migrate` and follow the same naming pattern.
+
+To create a new data migration, run:
+
+```shell
+rails generate monarch_migrate:data_migration downcase_usernames
+```
+
+In contrast to regular migrations, there is no need to inherit any classes:
+
+```ruby
+# db/data_migrate/20220605083010_downcase_usernames.rb
+ActiveRecord::Base.connection.execute(<<-SQL)
+  UPDATE users SET username = lower(username);
+SQL
+
+SearchIndex.rebuild
+```
+
+As seen above, it is plain ruby code where you can refer to any object you
+need. MonarchMigrate will run each migration in a separate transaction.
+
+To run pending data migrations:
+
+```shell
+rails data:migrate
+```
+
+Or a specific version:
+
+```shell
+rails data:migrate VERSION=20220605083010
+```
+
+## Known Issues and Limitations
+
+The following issues and limitations are not necessary inherent to MonarchMigrate.
+Some are innate to migrations in general.
+
+### Using ActiveRecord Models in Migrations
+
+<blockquote>
+  <p>The Active Record way claims that intelligence belongs in your models, not in the database.</p>
+  <a href="https://guides.rubyonrails.org/active_record_migrations.html#active-record-and-referential-integrity">
+    <sup>–Active Record and Referential Integrity, Rails Guide for Active Record Migrations</sup>
+  </a>
+</blockquote>
+
+Typically, data migrations relate closely to business models. In an ideal Rails world,
+data manipulations would depend on model logic to enforce validation, conform to
+business rules, etc. Hence, it is very tempting to use ActiveRecord models in migrations.
+
+Suppose we have designed a system where users have first and last names. Time passes and
+it becomes clear this is [wrong][3]. Now, we want to put things right and come
+up with the following plan:
+
+1. Add a `name` column to `users` table to hold the entire name of a person.
+2. Change the `User` model and use a data migration to update existing records.
+3. Drop `first_name` and `last_name` columns.
+
+A regular Rails migration for updating existing records may look something like this:
+
+```ruby
+# db/migrate/20220605083010_backfill_users_name.rb
+def up
+  User.all.each do |user|
+    user.name = "#{user.first_name} #{user.last_name}"
+    user.save
+  end
+end
+```
+
+The code above is problematic because:
+
+1. It iterates through every user.
+2. It invokes validations and callbacks, which may have unintended consequences.
+3. It does not check if the user has already been migrated.
+4. It will fail when a future developer runs the migration during local development setup after `first_name` and `last_name` columns are gone.
+
+To avoid issues 1-3, we can rewrite the migration to:
+
+```ruby
+# db/migrate/20220605083010_backfill_users_name.rb
+def up
+  User.where(name: nil).find_each do |user|
+    user.update_column(:name, "#{user.first_name} #{user.last_name}")
+  end
+end
+```
+
+Unfortunately, with regular Rails migrations we will still face issue 4.
+
+To avoid it, we need to separate data from schema migrations and not run data
+migrations locally. With seed [scripts][2], there is no need to run them anyway.
+
+Keep the above in mind when referencing ActiveRecord models in data migrations. Ideally,
+limit their use and do as much processing as possible in Postgres.
+
+
+### Long-running Tasks in Migrations
+
+As mentioned above, MonarchMigrate (similar to Rails) runs each migration in a separate
+transaction. A long-running task within a migration will keep the transaction open for
+the duration of the task. As a result, the migration may hang or fail.
+
+To avoid this, we can run such tasks asynchronously.
+
+Back to the previous example:
+
+```ruby
+# db/data_migrate/20220605083010_downcase_usernames.rb
+ActiveRecord::Base.connection.execute(<<-SQL)
+  UPDATE users SET username = lower(username);
+SQL
+
+SearchIndex::RebuildJob.perform_later
+```
+
+
+## Trivia
+
+One of the most impressive migrations on Earth is the multi-generational
+round trip of the monarch butterfly.
+
+Each year, millions of monarch butterflies leave their northern ranges
+and fly south, where they gather in huge roosts to survive the winter.
+When spring arrives, the monarchs start their return journey north.
+The population cycles through three to five generations to reach their
+destination. In the end, a new generation of butterflies complete the
+journey their great-great-great-grandparents started.
+
+It is still a mystery to scientists how the new generations know where to go,
+but they appear to navigate using a combination of the Earth's magnetic field
+and the position of the sun.
+
+Genetically speaking, this is a truly incredible data migration!
+
+
+## See Also
+
+Alternative gems
+
+- [nonschema_migrations](https://github.com/jasonfb/nonschema_migrations) - Exactly like schema migrations but for data.
+- [data-migrate](https://github.com/ilyakatz/data-migrate) - A gem to run data migrations alongside schema migrations.
+
+Articles
+
+- [Data Migrations in Rails](https://thoughtbot.com/blog/data-migrations-in-rails)
+- [Zero downtime migrations: 500 million rows](https://www.honeybadger.io/blog/zero-downtime-migrations-of-large-databases-using-rails-postgres-and-redis/)
+- [Three Useful Data Migration Patterns for Rails](https://www.ombulabs.com/blog/rails/data-migrations/three-useful-data-migrations-patterns-in-rails.html)
+- [Ruby on Rails Model Patterns and Anti-patterns](https://blog.appsignal.com/2020/11/18/rails-model-patterns-and-anti-patterns.html)
+- [Rails Migrations with Zero Downtime](https://www.cloudbees.com/blog/rails-migrations-zero-downtime)
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+[2]: https://thoughtbot.com/blog/priming-the-pump
+[3]: https://www.kalzumeus.com/2010/06/17/falsehoods-programmers-believe-about-names/

data/Rakefile

@@ -0,0 +1,15 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+namespace :fake do
+  require_relative "test/fake/application"
+  Fake::Application.load_tasks
+end
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/*_test.rb"]
+end
+
+task default: :test

data/bin/setup

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+set -e
+
+# Install required gems, including Appraisal, which helps us test against
+# multiple Rails versions
+gem install bundler --conservative
+bundle check || bundle install
+
+if [ -z "$CI" ]; then
+  bundle exec appraisal install
+fi
+
+# Set up database for the application that we test against
+RAILS_ENV=test bundle exec rake fake:db:reset

data/db/schema.rb

@@ -0,0 +1,20 @@
+# This file is auto-generated from the current state of the database. Instead
+# of editing this file, please use the migrations feature of Active Record to
+# incrementally modify your database, and then regenerate this schema definition.
+#
+# This file is the source Rails uses to define your schema when running `bin/rails
+# db:schema:load`. When creating a new database, `bin/rails db:schema:load` tends to
+# be faster and is potentially less error prone than running all of your
+# migrations from scratch. Old migrations may fail to apply correctly if those
+# migrations use external dependencies or application code.
+#
+# It's strongly recommended that you check this file into your version control system.
+
+ActiveRecord::Schema.define(version: 2022_05_25_052032) do
+  create_table "data_migration_records", force: :cascade do |t|
+    t.string "version", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["version"], name: "index_data_migration_records_on_version", unique: true
+  end
+end

data/lib/generators/monarch_migrate/data_migration/USAGE

@@ -0,0 +1,5 @@
+Description:
+  Generate a new data migration
+
+Examples:
+    rails generate monarch_migrate:data_migration assign_name_to_users

data/lib/generators/monarch_migrate/data_migration/data_migration_generator.rb

@@ -0,0 +1,28 @@
+require "rails/generators/active_record"
+
+module MonarchMigrate
+  module Generators
+    class DataMigrationGenerator < Rails::Generators::NamedBase
+      include ActiveRecord::Generators::Migration
+
+      source_root File.expand_path("../templates", __FILE__)
+
+      def create_data_migration
+        validate_file_name!
+
+        migration_template(
+          "data_migration.rb.erb",
+          File.join(MonarchMigrate.data_migrations_path, "#{file_name}.rb")
+        )
+      end
+
+      private
+
+      def validate_file_name!
+        unless /^[_a-z0-9]+$/.match?(file_name)
+          raise ActiveRecord::IllegalMigrationNameError.new(file_name)
+        end
+      end
+    end
+  end
+end

data/lib/generators/monarch_migrate/install/USAGE

@@ -0,0 +1,5 @@
+Description:
+  Generate a schema migration to create the table used for data migrations
+
+Examples:
+    rails generate monarch_migrate:install

data/lib/generators/monarch_migrate/install/install_generator.rb

@@ -0,0 +1,45 @@
+require "rails/generators"
+require "rails/generators/active_record"
+
+module MonarchMigrate
+  module Generators
+    class InstallGenerator < Rails::Generators::Base
+      include Rails::Generators::Migration
+
+      source_root File.expand_path("../templates", __FILE__)
+
+      def self.next_migration_number(dir)
+        ActiveRecord::Generators::Base.next_migration_number(dir)
+      end
+
+      def create_monarch_migrate_migration
+        return if migration_exists?
+        return if migration_table_exists?
+
+        migration_template(
+          "create_data_migration_records.rb.erb",
+          "db/migrate/create_data_migration_records.rb",
+          migration_version: migration_version
+        )
+      end
+
+      def migration_version
+        "[#{ActiveRecord::Migration.current_version}]"
+      end
+
+      def migration_table_name
+        MonarchMigrate.data_migrations_table_name
+      end
+
+      private
+
+      def migration_table_exists?
+        ActiveRecord::Base.connection.data_source_exists?(migration_table_name)
+      end
+
+      def migration_exists?
+        Dir.glob("db/migrate/*.rb").any? { |f| f.end_with?("create_data_migration_records.rb") }
+      end
+    end
+  end
+end

data/lib/generators/monarch_migrate/install/templates/create_data_migration_records.rb.erb

@@ -0,0 +1,10 @@
+class CreateDataMigrationRecords < ActiveRecord::Migration<%= migration_version %>
+  def change
+    create_table :<%= migration_table_name %> do |t|
+      t.string :version, null: false
+      t.timestamps
+    end
+
+    add_index :<%= migration_table_name %>, :version, unique: true
+  end
+end

data/lib/monarch_migrate/migration.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module MonarchMigrate
+  class Migration
+    def initialize(path)
+      @path = path.to_s
+    end
+
+    def filename
+      File.basename(path)
+    end
+
+    def name
+      File.basename(path, ".rb").match(/^[0-9]+_(.*)$/)[1].humanize
+    end
+
+    def version
+      filename.match(/^([0-9]+)_/)[1]
+    end
+
+    def pending?
+      !MigrationRecord.exists?(version: version)
+    end
+
+    def run(io = nil)
+      io ||= File.open(File::NULL, "w")
+
+      ActiveRecord::Base.connection.transaction do
+        io.puts "Running data migration #{version}: #{name}"
+
+        begin
+          instance_eval File.read(path), path
+          MigrationRecord.create!(version: version)
+          io.puts "Migration complete"
+        rescue => e
+          io.puts "Migration failed due to #{e}"
+          raise ActiveRecord::Rollback
+        end
+
+        io.puts
+      end
+    end
+
+    private
+
+    attr_reader :path
+  end
+end

data/lib/monarch_migrate/migration_record.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module MonarchMigrate
+  class MigrationRecord < ActiveRecord::Base
+    class << self
+      def table_name
+        "#{table_name_prefix}#{MonarchMigrate.data_migrations_table_name}#{table_name_suffix}"
+      end
+
+      def normalized_versions
+        all_versions.map { |v| normalize_version(v) }
+      end
+
+      private
+
+      def all_versions
+        order(version: :asc).pluck(:version)
+      end
+
+      def normalize_version(version)
+        "%.3d" % version.to_i
+      end
+    end
+  end
+end

data/lib/monarch_migrate/migrator.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module MonarchMigrate
+  class Migrator
+    attr_reader :path
+    attr_reader :version
+
+    def initialize(path, version: nil)
+      @path = path.to_s
+      @version = version
+    end
+
+    def migrations
+      migration_files.sort.map do |f|
+        Migration.new(f)
+      end
+    end
+
+    def pending_migrations
+      migrations.select(&:pending?)
+    end
+
+    def run(io = nil)
+      io ||= File.open(File::NULL, "w")
+
+      if pending_migrations.any?
+        io.puts "Running #{pending_migrations.size} data migrations"
+        pending_migrations.sort_by(&:version).each { |m| m.run(io) }
+      else
+        io.puts "No data migrations pending"
+        []
+      end
+    end
+
+    def migrations_status
+      db_list = MigrationRecord.normalized_versions
+
+      file_list = migrations.filter_map do |migration|
+        version = migration.version
+        status = db_list.delete(version) ? "up" : "down"
+        [status, version, migration.name]
+      end
+
+      db_list.map! do |version|
+        ["up", version, "***** NO FILE *****"]
+      end
+
+      (db_list + file_list).sort_by { |_, version, _| version.to_i }
+    end
+
+    private
+
+    def migration_files
+      if version
+        Dir["#{path}/#{version}_*.rb"].tap do |entries|
+          raise ActiveRecord::UnknownMigrationVersionError.new(version) if entries.empty?
+        end
+      else
+        Dir["#{path}/*_*.rb"]
+      end
+    end
+  end
+end

data/lib/monarch_migrate/railtie.rb

@@ -0,0 +1,9 @@
+module MonarchMigrate
+  class Railtie < Rails::Railtie
+    railtie_name :monarch_migrate
+
+    rake_tasks do
+      load File.expand_path("tasks.rake", __dir__)
+    end
+  end
+end

data/lib/monarch_migrate/tasks.rake

@@ -0,0 +1,34 @@
+require "monarch_migrate"
+
+namespace :data do
+  desc "Run pending data migrations, or a single version specified by environment variable VERSION"
+  task migrate: :environment do
+    MonarchMigrate.migrator.run($stdout)
+  end
+
+  namespace :migrate do
+    desc "Display status of data migrations"
+    task status: :environment do
+      unless ActiveRecord::Base.connection.data_source_exists?(MonarchMigrate.data_migrations_table_name)
+        Kernel.abort "Data migrations table does not exist yet."
+      end
+
+      database =
+        if Rails::VERSION::MAJOR < 6
+          ActiveRecord::Base.connection_config[:database]
+        else
+          ActiveRecord::Base.connection_db_config.database
+        end
+
+      puts "\ndatabase: #{database}\n\n"
+      puts "#{"Status".center(8)}  #{"Data Migration ID".ljust(19)}  Data Migration Name"
+      puts "-" * 50
+
+      MonarchMigrate.migrator.migrations_status.each do |status, version, name|
+        puts "#{status.center(8)}  #{version.ljust(19)}  #{name}"
+      end
+
+      puts
+    end
+  end
+end

data/lib/monarch_migrate/version.rb

@@ -0,0 +1,3 @@
+module MonarchMigrate
+  VERSION = "0.4.0"
+end

data/lib/monarch_migrate.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+# :nodoc
+module MonarchMigrate
+  def self.data_migrations_table_name
+    "data_migration_records"
+  end
+
+  def self.data_migrations_path
+    "db/data_migrate"
+  end
+
+  def self.migrator
+    Migrator.new(data_migrations_path, version: ENV.fetch("VERSION", nil))
+  end
+end
+
+require "monarch_migrate/migration"
+require "monarch_migrate/migration_record"
+require "monarch_migrate/migrator"
+require "monarch_migrate/railtie"
+require "monarch_migrate/version"

data/monarch_migrate.gemspec

@@ -0,0 +1,31 @@
+require_relative "lib/monarch_migrate/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "monarch_migrate"
+  spec.version = MonarchMigrate::VERSION
+  spec.authors = ["Yanko Ivanov"]
+  spec.email = ["yanko.ivanov@gmail.com"]
+
+  spec.summary = "Sensible data migrations for Rails"
+  spec.homepage = "https://github.com/lunohodov/monarch"
+  spec.license = "MIT"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.7.3")
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/lunohodov/monarch"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path("..", __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features|gemfiles|tmp)/}) }
+  end
+
+  spec.add_dependency("activerecord", ">= 5.2.0")
+  spec.add_dependency("railties", ">= 5.2.0")
+
+  spec.add_development_dependency("appraisal")
+  spec.add_development_dependency("minitest")
+  spec.add_development_dependency("rake")
+  spec.add_development_dependency("sqlite3")
+  spec.add_development_dependency("standard")
+end

metadata

@@ -0,0 +1,168 @@
+--- !ruby/object:Gem::Specification
+name: monarch_migrate
+version: !ruby/object:Gem::Version
+  version: 0.4.0
+platform: ruby
+authors:
+- Yanko Ivanov
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2022-06-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.2.0
+- !ruby/object:Gem::Dependency
+  name: railties
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.2.0
+- !ruby/object:Gem::Dependency
+  name: appraisal
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: standard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description:
+email:
+- yanko.ivanov@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".github/workflows/ci.yml"
+- ".gitignore"
+- ".ruby-version"
+- Appraisals
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- bin/setup
+- db/schema.rb
+- lib/generators/monarch_migrate/data_migration/USAGE
+- lib/generators/monarch_migrate/data_migration/data_migration_generator.rb
+- lib/generators/monarch_migrate/data_migration/templates/data_migration.rb.erb
+- lib/generators/monarch_migrate/install/USAGE
+- lib/generators/monarch_migrate/install/install_generator.rb
+- lib/generators/monarch_migrate/install/templates/create_data_migration_records.rb.erb
+- lib/monarch_migrate.rb
+- lib/monarch_migrate/migration.rb
+- lib/monarch_migrate/migration_record.rb
+- lib/monarch_migrate/migrator.rb
+- lib/monarch_migrate/railtie.rb
+- lib/monarch_migrate/tasks.rake
+- lib/monarch_migrate/version.rb
+- monarch_migrate.gemspec
+homepage: https://github.com/lunohodov/monarch
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/lunohodov/monarch
+  source_code_uri: https://github.com/lunohodov/monarch
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.7.3
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.6
+signing_key:
+specification_version: 4
+summary: Sensible data migrations for Rails
+test_files: []