data-migration 1.0.0

data/data-migration.gemspec

@@ -0,0 +1,44 @@
+Gem::Specification.new do |gem|
+  gem.name = "data-migration"
+  gem.version = File.read(File.expand_path("../lib/data_migration.rb", __FILE__)).match(/VERSION\s*=\s*"(.*?)"/)[1]
+
+  repository_url = "https://github.com/amkisko/data-migration.rb"
+  root_files = %w[CHANGELOG.md LICENSE.md README.md]
+  root_files << "#{gem.name}.gemspec"
+
+  gem.license = "MIT"
+
+  gem.platform = Gem::Platform::RUBY
+
+  gem.authors = ["Andrei Makarov"]
+  gem.email = ["andrei@kiskolabs.com"]
+  gem.homepage = repository_url
+  gem.summary = "Data migrations kit for ActiveRecord and ActiveJob"
+  gem.description = gem.summary
+  gem.metadata = {
+    "homepage" => repository_url,
+    "source_code_uri" => repository_url,
+    "bug_tracker_uri" => "#{repository_url}/issues",
+    "changelog_uri" => "#{repository_url}/blob/main/CHANGELOG.md",
+    "rubygems_mfa_required" => "true"
+  }
+
+  gem.files = `git ls-files`.split("\n")
+  gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
+
+  gem.required_ruby_version = ">= 3"
+  gem.require_paths = ["lib"]
+
+  gem.add_dependency "rails", "> 5"
+  gem.add_dependency "activejob", "> 5"
+  gem.add_dependency "activerecord", "> 5"
+  gem.add_dependency "activesupport", "> 5"
+
+  gem.add_development_dependency "bundler", "~> 2"
+  gem.add_development_dependency "rspec", "~> 3"
+  gem.add_development_dependency "rspec_junit_formatter", "~> 0.6"
+  gem.add_development_dependency "simplecov", "~> 0.21"
+  gem.add_development_dependency "simplecov-cobertura", "~> 2"
+  gem.add_development_dependency "sqlite3", "~> 2.4"
+end

data/lib/data_migration/config.rb

@@ -0,0 +1,84 @@
+module DataMigration
+  class Config
+    def schema_migrations_path
+      @schema_migrations_path ||= "db/migrate"
+    end
+
+    attr_writer :data_migrations_path
+    def data_migrations_path
+      @data_migrations_path ||= "db/data_migrations"
+    end
+
+    attr_writer :data_migrations_full_path
+    def data_migrations_full_path
+      @data_migrations_full_path ||= Rails.root.join(data_migrations_path)
+    end
+
+    def data_migrations_path_glob
+      "#{data_migrations_full_path}/*.rb"
+    end
+
+    attr_writer :generate_spec
+    def generate_spec?
+      @generate_spec.nil? ? true : @generate_spec
+    end
+
+    attr_writer :job_class
+    def job_class
+      @job_class ||= DataMigration::Job
+    end
+
+    attr_writer :task_class
+    def task_class
+      @task_class ||= DataMigration::Task
+    end
+
+    attr_writer :operator_resolver
+    def operator_resolver(&block)
+      if block_given?
+        @operator_resolver = block
+      else
+        @operator_resolver ||= -> do
+          if Object.const_defined?(:ActionReporter)
+            ActionReporter.current_user
+          elsif Object.const_defined?(:Audited)
+            Audited.store[:audited_user]
+          end
+        end
+      end
+    end
+
+    attr_writer :monitoring_context
+    def monitoring_context(&block)
+      if block_given?
+        @monitoring_context = block
+      else
+        @monitoring_context ||= ->(migration) do
+          context = {
+            data_migration_name: migration.name,
+            data_migration_id: migration.try(:to_global_id) || migration.id,
+            data_migration_operator_id: migration.operator&.try(:to_global_id) || migration.operator&.id
+          }
+          if Object.const_defined?(:ActionReporter)
+            ActionReporter.current_user ||= migration.operator
+            ActionReporter.context(**context)
+          elsif Object.const_defined?(:Audited)
+            Audited.store[:audited_user] ||= migration.operator
+          else
+            Rails.logger.info("Data migration context: #{context.inspect}")
+          end
+        end
+      end
+    end
+
+    attr_writer :job_queue_name
+    def job_queue_name
+      @job_queue_name ||= :default
+    end
+
+    attr_writer :default_jobs_limit
+    def default_jobs_limit
+      @default_jobs_limit ||= 10
+    end
+  end
+end

data/lib/data_migration/job.rb

@@ -0,0 +1,63 @@
+require "active_job"
+
+module DataMigration
+  class Job < ActiveJob::Base
+    queue_as { DataMigration.config.job_queue_name }
+
+    discard_on StandardError
+
+    def perform(task_id, *job_args, **job_kwargs)
+      task = DataMigration::Task.find(task_id)
+      DataMigration.config.monitoring_context.call(task)
+
+      migration_name = task.name
+      migration_path = task.file_path
+
+      unless task.file_exists?
+        DataMigration.notify("#{migration_name} not found")
+        return
+      end
+
+      task.job_check_in!(job_id, job_args:, job_kwargs:)
+
+      require migration_path
+      klass_name = migration_name.gsub(/^[0-9_]+/, "").camelize
+      klass = klass_name.safe_constantize
+      raise "Data migration class #{klass_name} not found" unless klass.is_a?(Class)
+      raise "Data migration class #{klass_name} must implement `perform` method" unless klass.method_defined?(:perform)
+
+      if task.started_at.blank?
+        task.update!(started_at: Time.current, status: :started)
+      end
+
+      if task.requires_pause?
+        DataMigration::Job.set(wait: task.pause_minutes.minutes).perform_later(task_id, *job_args, **job_kwargs)
+        task.update!(status: :paused)
+        return
+      end
+
+      Thread.current[:data_migration_enqueue_called] ||= {}
+      Thread.current[:data_migration_enqueue_kwargs] ||= {}
+      klass.define_method(:enqueue) do |**enqueue_kwargs|
+        Thread.current[:data_migration_enqueue_called][klass.name] = true
+        Thread.current[:data_migration_enqueue_kwargs][klass.name] = enqueue_kwargs
+      end
+
+      task.update!(status: :performing, pause_minutes: 0)
+      klass.new.perform(**job_kwargs)
+      task.job_check_out!(job_id)
+
+      enqueue_called = Thread.current[:data_migration_enqueue_called].delete(klass.name)
+      enqueue_kwargs = Thread.current[:data_migration_enqueue_kwargs].delete(klass.name)
+      if enqueue_called
+        if enqueue_kwargs[:background] == false
+          self.class.new.perform(task_id, *job_args, **enqueue_kwargs)
+        else
+          DataMigration::Job.perform_later(task_id, *job_args, **enqueue_kwargs)
+        end
+      else
+        task.update!(completed_at: Time.current, status: :completed)
+      end
+    end
+  end
+end

data/lib/data_migration/task.rb

@@ -0,0 +1,127 @@
+require "active_record"
+
+module DataMigration
+  class JobConcurrencyLimitError < StandardError; end
+  class JobConflictError < StandardError; end
+
+  def self.tasks_table_name
+    "data_migration_tasks"
+  end
+
+  class Task < ActiveRecord::Base
+    self.table_name = DataMigration.tasks_table_name
+
+    belongs_to :operator, polymorphic: true, optional: true
+
+    before_validation do
+      self.operator ||= DataMigration.config.operator_resolver.call
+      self.jobs_limit ||= DataMigration.config.default_jobs_limit
+    end
+
+    enum :status, {
+      started: "started",
+      performing: "performing",
+      paused: "paused",
+      completed: "completed"
+    }
+
+    validates :name, presence: true
+    validates :pause_minutes, numericality: { greater_than_or_equal_to: 0, only_integer: true }, if: -> { pause_minutes.present? }
+    validates :jobs_limit, numericality: { greater_than_or_equal_to: 0, only_integer: true }, if: -> { jobs_limit.present? }
+    validate :file_should_exist
+
+    after_save do
+      if saved_change_to_started_at? && started_at.present?
+        DataMigration.notify("#{user_title} started")
+      end
+      if saved_change_to_completed_at? && completed_at.present?
+        DataMigration.notify("#{user_title} finished")
+      end
+    end
+
+    scope :not_started, -> { where(status: nil, started_at: nil) }
+    scope :started, -> { where(status: :started) }
+    scope :paused, -> { where(status: :paused) }
+    scope :performing, -> { where(status: :performing) }
+    scope :completed, -> { where(status: :completed) }
+
+    def self.job_class
+      DataMigration.config.job_class
+    end
+
+    def self.perform_now(name, **kwargs)
+      create!(name:).perform_now(**kwargs)
+    end
+
+    def perform_now(**perform_args)
+      update!(kwargs: perform_args)
+      self.class.job_class.perform_now(id, **perform_args)
+    end
+
+    def self.perform_later(name, **kwargs)
+      create!(name:).perform_later(**kwargs)
+    end
+
+    def perform_later(**perform_args)
+      update!(kwargs: perform_args)
+      self.class.job_class.perform_later(id, **perform_args)
+    end
+
+    def self.prepare(name, pause_minutes: nil, jobs_limit: nil)
+      create!(name:, pause_minutes:, jobs_limit:)
+    end
+
+    def self.root_path
+      DataMigration.config.data_migrations_full_path
+    end
+
+    def self.list
+      Dir[DataMigration.config.data_migrations_path_glob].map { |f| File.basename(f, ".*") }
+    end
+
+    def file_path
+      "#{self.class.root_path}/#{name}.rb"
+    end
+
+    def file_exists?
+      self.class.list.include?(name)
+    end
+
+    def not_started?
+      status.nil? && started_at.nil?
+    end
+
+    def job_check_in!(job_id, job_args: [], job_kwargs: {})
+      self.current_jobs ||= {}
+
+      raise DataMigration::JobConflictError, "#{user_title} already has job ##{job_id}" if current_jobs.key?(job_id)
+      raise DataMigration::JobConcurrencyLimitError, "#{user_title} reached limit of #{jobs_limit} jobs" if jobs_limit.present? && current_jobs.size >= jobs_limit
+
+      self.current_jobs[job_id] = {
+        ts: Time.current,
+        args: job_args,
+        kwargs: job_kwargs
+      }
+      save!
+    end
+
+    def job_check_out!(job_id)
+      self.current_jobs.delete(job_id)
+      save!
+    end
+
+    def user_title
+      "Data migration ##{id} #{name}"
+    end
+
+    def requires_pause?
+      pause_minutes.positive? && !paused?
+    end
+
+    private
+
+    def file_should_exist
+      errors.add(:name, "is not found") unless file_exists?
+    end
+  end
+end

data/lib/data_migration.rb

@@ -0,0 +1,37 @@
+require "data_migration/config"
+require "data_migration/job"
+require "data_migration/task"
+
+module DataMigration
+  VERSION = "1.0.0".freeze
+
+  module_function
+
+  def config
+    @@config ||= DataMigration::Config.new
+  end
+
+  def configure
+    yield config
+  end
+
+  def notify(message, context: {})
+    if Object.const_defined?(:ActionReporter)
+      ActionReporter.notify(message, context:)
+    elsif Object.const_defined?(:Rails)
+      Rails.logger.info("#{message} #{context.inspect}")
+    end
+  end
+
+  def perform_now(...)
+    DataMigration::Task.perform_now(...)
+  end
+
+  def perform_later(...)
+    DataMigration::Task.perform_later(...)
+  end
+
+  def prepare(...)
+    DataMigration::Task.prepare(...)
+  end
+end

data/lib/generators/data_migration_generator.rb

@@ -0,0 +1,13 @@
+require "rails/generators"
+require "rails/generators/active_record"
+require "rails/generators/active_record/migration/migration_generator"
+
+class DataMigrationGenerator < ActiveRecord::Generators::MigrationGenerator
+  source_root File.expand_path("../templates", __FILE__)
+
+  def create_migration_file
+    set_local_assigns!
+    validate_file_name!
+    migration_template "data_migration.rb", "#{DataMigration.config.data_migrations_path}/#{file_name}.rb"
+  end
+end

data/lib/generators/install_generator.rb

@@ -0,0 +1,17 @@
+require "rails/generators"
+require "rails/generators/active_record"
+require "rails/generators/active_record/migration/migration_generator"
+
+class InstallGenerator < ActiveRecord::Generators::MigrationGenerator
+  source_root File.expand_path("../templates", __FILE__)
+
+  def create_migration_file
+    set_local_assigns!
+    validate_file_name!
+    migration_template "install_#{name}.rb", "#{DataMigration.config.schema_migrations_path}/#{file_name}.rb"
+  end
+
+  def migration_parent
+    "ActiveRecord::Migration[#{ActiveRecord::Migration.current_version}]"
+  end
+end

data/lib/generators/templates/data_migration.rb.tt

@@ -0,0 +1,26 @@
+# USAGE: within remote console run one of the following commands
+# - rails db:migrate:data <%= migration_number %>_<%= migration_file_name %>
+# - DataMigration.perform_now("<%= migration_number %>_<%= migration_file_name %>")
+# - DataMigration.perform_later("<%= migration_number %>_<%= migration_file_name %>")
+# - DataMigration.prepare("<%= migration_number %>_<%= migration_file_name %>", pause: 1.minute, jobs_limit: 2).perform_later
+
+class <%= migration_class_name %>
+  def perform(**kwargs)
+
+  end
+end
+<%- if DataMigration.config.generate_spec? %>
+# TESTING:
+# - bin/rspec <%= DataMigration.config.data_migrations_path %>/<%= migration_number %>_<%= migration_file_name %>.rb
+if Rails.env.test? && Object.const_defined?(:RSpec)
+  require "rails_helper"
+
+  RSpec.describe <%= migration_class_name %>, type: :data_migration do
+    subject(:perform) { described_class.new.perform }
+
+    it do
+      expect { perform }.not_to raise_error
+    end
+  end
+end
+<%- end %>

data/lib/generators/templates/install_data_migration_tasks.rb.tt

@@ -0,0 +1,27 @@
+class <%= migration_class_name %> < <%= migration_parent %>
+  def self.up
+    create_table :<%= DataMigration.tasks_table_name %>, force: true do |t|
+      t.string "name", null: false
+
+      t.jsonb "kwargs", default: {}, null: false
+      t.jsonb "current_jobs", default: {}, null: false
+
+      t.string "status"
+      t.datetime "started_at"
+      t.datetime "completed_at"
+
+      t.bigint "operator_id"
+      t.string "operator_type"
+
+      t.integer "pause_minutes", default: 0, null: false
+      t.integer "jobs_limit"
+
+      t.datetime "created_at", null: false
+      t.datetime "updated_at", null: false
+    end
+  end
+
+  def self.down
+    drop_table :<%= DataMigration.tasks_table_name %>
+  end
+end

data/spec/data_migration/config_spec.rb

@@ -0,0 +1,116 @@
+require "spec_helper"
+
+describe DataMigration::Config do
+  subject(:config) { described_class.new }
+
+  let(:current_user) do
+    User.new(id: 1)
+  end
+  let(:action_reporter) do
+    Class.new do
+      def current_user
+        User.new(id: 3)
+      end
+
+      def context(**kwargs)
+        kwargs
+      end
+    end.new
+  end
+  let(:audited) do
+    Class.new do
+      def store
+        { audited_user: User.new(id: 4) }
+      end
+    end.new
+  end
+  let(:logger) do
+    Class.new do
+      def info(message)
+        puts message
+      end
+    end.new
+  end
+  let(:other_user) do
+    User.new(id: 2)
+  end
+  let(:migration) do
+    DataMigration::Task.new(id: 1, name: "test", operator: other_user)
+  end
+
+  before do
+    allow(Rails).to receive(:root).and_return(rails_root)
+    allow(Rails).to receive(:logger).and_return(logger)
+  end
+
+  it "has default schema migrations path" do
+    expect(config.schema_migrations_path).to eq "db/migrate"
+  end
+
+  it "has default data migrations path" do
+    expect(config.data_migrations_path).to eq "db/data_migrations"
+  end
+
+  it "has default data migrations full path" do
+    expect(config.data_migrations_full_path).to eq Rails.root.join("db/data_migrations")
+  end
+
+  it "has default data migrations path glob" do
+    expect(config.data_migrations_path_glob).to eq Rails.root.join("db/data_migrations/*.rb").to_s
+  end
+
+  it "has default generate spec" do
+    expect(config.generate_spec?).to be true
+  end
+
+  it "has default task class" do
+    expect(config.task_class).to eq DataMigration::Task
+  end
+
+  it "has default job class" do
+    expect(config.job_class).to eq DataMigration::Job
+  end
+
+  it "has default job queue name" do
+    expect(config.job_queue_name).to eq :default
+  end
+
+  it "has default default jobs limit" do
+    expect(config.default_jobs_limit).to eq 10
+  end
+
+  it "has default monitoring context" do
+    expect(config.monitoring_context).to be_a Proc
+
+    expect(Rails.logger).to receive(:info).with("Data migration context: {:data_migration_name=>\"test\", :data_migration_id=>1, :data_migration_operator_id=>2}")
+    expect { config.monitoring_context.call(migration) }.not_to raise_error
+  end
+
+  context "when ActionReporter is defined" do
+    before do
+      stub_const("ActionReporter", action_reporter)
+    end
+
+    it "sets monitoring context" do
+      expect(ActionReporter).to receive(:context).with(
+        data_migration_name: "test",
+        data_migration_id: 1,
+        data_migration_operator_id: 2
+      )
+      expect(ActionReporter).to receive(:current_user).and_return(nil)
+      expect(ActionReporter).to receive(:current_user=).with(other_user)
+      expect { config.monitoring_context.call(migration) }.not_to raise_error
+    end
+  end
+
+  context "when Audited is defined" do
+    before do
+      stub_const("Audited", audited)
+    end
+
+    it "sets monitoring context" do
+      expect(Audited).to receive(:store).and_return(audited_user: User.new(id: 5))
+      expect { config.monitoring_context.call(migration) }.not_to raise_error
+    end
+  end
+end

data/spec/data_migration/job_spec.rb

@@ -0,0 +1,96 @@
+require "spec_helper"
+
+describe DataMigration::Job do
+  subject(:job) { DataMigration::Job.new }
+
+  let(:operator) { User.create!(email: "test@example.com") }
+  let(:task) { DataMigration::Task.create!(name: migration_name, operator: operator) }
+
+  let(:migration_name) { "20241206200111_create_users" }
+  let(:data_migrations_full_path) { File.expand_path("../fixtures/data_migrations", __dir__) }
+
+  before do
+    allow(Rails).to receive(:env).and_return(rails_env)
+    allow(Rails).to receive(:logger).and_return(rails_logger)
+    allow(DataMigration.config).to receive(:data_migrations_full_path).and_return(data_migrations_full_path)
+  end
+
+  it "runs rspec spec/fixtures/data_migrations/20241206200111_create_users.rb" do
+    output = `RUN_MIGRATION_TESTS=1 rspec #{data_migrations_full_path}/#{migration_name}.rb`
+    expect(output).to include("1 example, 0 failures")
+  end
+
+  describe "#perform" do
+    subject(:perform) { job.perform(task.id, **job_kwargs) }
+
+    let(:job_kwargs) { {} }
+
+    it "updates the task status to completed" do
+      expect { perform }.to change { task.reload.status }.to("completed")
+    end
+
+    context "when migration file is not found" do
+      before do
+        task
+        allow_any_instance_of(DataMigration::Task).to receive(:file_exists?).and_return(false)
+      end
+
+      it "updates the task status to failed" do
+        expect(DataMigration).to receive(:notify).with("#{migration_name} not found")
+        expect { perform }.not_to raise_error
+      end
+    end
+
+    context "when migration class is not found" do
+      let(:migration_name) { "20241206200112_create_bad_users" }
+
+      it "raises an error" do
+        expect { perform }.to raise_error("Data migration class #{migration_name.gsub(/^[0-9_]+/, "").camelize} not found")
+      end
+    end
+
+    context "when migration class does not implement perform method" do
+      let(:migration_name) { "20241206200113_change_users" }
+
+      it "raises an error" do
+        expect { perform }.to raise_error("Data migration class #{migration_name.gsub(/^[0-9_]+/, "").camelize} must implement `perform` method")
+      end
+    end
+
+    context "when there is an enqueue call" do
+      let(:migration_name) { "20241206200114_create_batch_users" }
+
+      it "runs the migration in foreground" do
+        expect { perform }.to change { User.count }.by(3)
+        expect(task.reload.current_jobs.count).to eq(0)
+        expect(task.status).to eq("completed")
+        expect(User.pluck(:email)).to match_array(["test@example.com", "test_1@example.com", "test_2@example.com"])
+      end
+
+      context "when background is true" do
+        let(:job_kwargs) { { background: true } }
+
+        before do
+          operator
+        end
+
+        it "runs the migration in background" do
+          expect { perform }.to change(User, :count).by(1)
+          expect(task.reload.status).to eq("performing")
+          expect(task.current_jobs.count).to eq(0)
+          expect(task.kwargs).to eq({})
+
+          expect { job.perform(task.id, index: 2, background: true) }.to change(User, :count).by(1)
+          expect(task.reload.status).to eq("performing")
+          expect(task.reload.current_jobs.count).to eq(0)
+          expect(task.kwargs).to eq({})
+
+          expect { job.perform(task.id, index: 3, background: true) }.to change(User, :count).by(0)
+          expect(task.reload.status).to eq("completed")
+          expect(task.reload.current_jobs.count).to eq(0)
+          expect(task.kwargs).to eq({})
+        end
+      end
+    end
+  end
+end