data-migration 1.2.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9a8f110139404aa9a93001f7d7d62df44dfa39ed17c6467c3042b71071f0f5a9
-  data.tar.gz: 17ea84c1d597d6ba5907f0841bee35cf231225f9ac6febfa0dfa4b6c697105d3
+  metadata.gz: f317e1c1d38d56179e7236be0d37be8105859b653792c8eb4bdae60c0010affa
+  data.tar.gz: df14fb4e41b46c7fffebf89d9367b810128216f285b15da18d8c5b3297f653d4
 SHA512:
-  metadata.gz: 2cb6a04608ff5fa007f3bc3e5e6fd819413d73b9041ee8ec542de029719d6c089706528e3ec72e9d0442d75596709cb2646f004c00bce6a9100a063be2e91eab
-  data.tar.gz: 53b024625cc45959ac7cc94c5ccb5bf22f1ff463a04cfea4f847a43292b688092aaf9f3e757193fd15c17c47f2a9539b208be3153e043cca313f3b78682ff36c
+  metadata.gz: 11da0da9e0b6593df4bd94a728d66f2ecbd5672e3a9819ec90bf58d32f811053b7e8f7890631faa7ea99253f9652e14f5383ece6af4f367ba16971a8cd7b1051
+  data.tar.gz: e93699e6817f780e7d12457742bd4d053ad029b6478f1a86cce911a75bfb49b6567bfb8822030dfdd8a1474fc4ffc47d6ec92519d041449250643b6b7c2c784f

data/CHANGELOG.md

@@ -1,5 +1,18 @@
 # CHANGELOG
 
+## 2.0.0 (2026-09-04)
+
+- BREAKING: Require Ruby 3.4 or newer.
+- Add `rfcs/` starter pack: process (RFC 0001), positioning (RFC 0002), and Standards Track design RFCs 0003–0005 for task status, operator run path, and jobs.
+- Report stopped data migrations as failed and release their job slots (RFC 0006).
+- Serialize task job-slot updates across worker instances.
+- Limit the gem package to runtime code and release documentation.
+
+## 1.3.0
+
+- Fix data migration tasks table generator
+- Update README with better examples
+
 ## 1.2.0
 
 - Wrap Task.status enum with ActiveRecord version check

data/README.md

@@ -1,10 +1,12 @@
 # data-migration.rb
 
-[![Gem Version](https://badge.fury.io/rb/data-migration.svg)](https://badge.fury.io/rb/data-migration) [![Test Status](https://github.com/amkisko/data-migration.rb/actions/workflows/test.yml/badge.svg)](https://github.com/amkisko/data-migration.rb/actions/workflows/test.yml) [![codecov](https://codecov.io/gh/amkisko/data-migration.rb/graph/badge.svg?token=881AFPL643)](https://codecov.io/gh/amkisko/data-migration.rb)
+[![Gem Version](https://badge.fury.io/rb/data-migration.svg)](https://badge.fury.io/rb/data-migration) [![Test Status](https://github.com/amkisko/data-migration.rb/actions/workflows/test.yml/badge.svg)](https://github.com/amkisko/data-migration.rb/actions/workflows/test.yml) [![codecov](https://codecov.io/gh/amkisko/data-migration.rb/graph/badge.svg?token=881AFPL643)](https://app.codecov.io/github/amkisko/data-migration.rb)
 
 Data migrations kit for ActiveRecord and ActiveJob.
 
-Sponsored by [Kisko Labs](https://www.kiskolabs.com).
+Requires Ruby 3.4 or newer.
+
+Design changes go through `rfcs/` (RFC 0001).
 
 ## Data migrations concept
 
@@ -39,26 +41,32 @@ Using RubyGems:
 gem install data-migration
 ```
 
-## Gemfile
+### Gemfile
 
 ```ruby
 gem "data-migration"
 ```
 
-## Usage
-
-### Run data migrations
+### Data migration tasks table
 
 ```sh
-bin/rails db:migrate:data 20241207120000_create_users
+bin/rails g data_migration:install data_migration_tasks
 ```
 
+## Usage
+
 ### Generate data migration job
 
 ```sh
 bin/rails g data_migration create_users
 ```
 
+### Run data migrations
+
+```sh
+bin/rails db:migrate:data 20241207120000_create_users
+```
+
 ## Configuration
 
 ### Set data migrations directory
@@ -110,6 +118,7 @@ end
 - ActiveRecord migrations generator is used to generate data migration files
 - Data migrations are not reversible, it is operator's responsibility to ensure that data migration has correct effect
 - Keep migrations logic stable and predictable, e.g. by checking uniqueness of created/updated records
+- Task rows persist migration keyword arguments and active job metadata; pass record references instead of secrets or unnecessary personal data
 
 ## Contributing
 
@@ -126,7 +135,7 @@ Contribution policy:
 
 ## Publishing
 
-Prefer using script `usr/bin/release.sh`, it will ensure that repository is synced and after publishing gem will create a tag.
+Prefer using script `usr/bin/release.rb`, it will ensure that repository is synced and after publishing gem will create a tag.
 
 ```sh
 GEM_VERSION=$(grep -Eo "VERSION\s*=\s*\".+\"" lib/data-migration.rb  | grep -Eo "[0-9.]{5,}")
@@ -136,6 +145,27 @@ gem push data-migration-$GEM_VERSION.gem
 git tag $GEM_VERSION && git push --tags && gh release create $GEM_VERSION --generate-notes
 ```
 
+## Links
+
+- [GitHub](https://github.com/amkisko/data-migration.rb)
+- [GitLab](https://gitlab.com/amkisko/data-migration.rb)
+- [RubyGems](https://rubygems.org/gems/data-migration)
+- [Versions Atom](https://rubygems.org/gems/data-migration/versions.atom) (feed id `8154436273761`)
+- [libraries.io](https://libraries.io/rubygems/data-migration)
+- [Deps.dev](https://deps.dev/rubygems/data-migration)
+- [SonarCloud](https://sonarcloud.io/project/overview?id=amkisko_data-migration.rb)
+- [Snyk](https://snyk.io/test/github/amkisko/data-migration.rb)
+- [Codecov](https://app.codecov.io/github/amkisko/data-migration.rb)
+- [OpenSSF Scorecard](https://scorecard.dev/viewer/?uri=github.com/amkisko/data-migration.rb)
+
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Sponsors
+
+Sponsored by [Kisko Labs](https://www.kiskolabs.com).
+
+<a href="https://www.kiskolabs.com">
+  <img src="kisko.svg" width="200" alt="Sponsored by Kisko Labs" />
+</a>

data/data-migration.gemspec

@@ -2,7 +2,6 @@ Gem::Specification.new do |gem|
   gem.name = "data-migration"
   gem.version = File.read(File.expand_path("../lib/data-migration.rb", __FILE__)).match(/VERSION\s*=\s*"(.*?)"/)[1]
 
-  repository_url = "https://github.com/amkisko/data-migration.rb"
   root_files = %w[CHANGELOG.md LICENSE.md README.md]
   root_files << "#{gem.name}.gemspec"
 
@@ -11,23 +10,24 @@ Gem::Specification.new do |gem|
   gem.platform = Gem::Platform::RUBY
 
   gem.authors = ["Andrei Makarov"]
-  gem.email = ["andrei@kiskolabs.com"]
-  gem.homepage = repository_url
+  gem.email = ["contact@kiskolabs.com"]
+  gem.homepage = "https://github.com/amkisko/data-migration.rb"
   gem.summary = "Data migrations kit for ActiveRecord and ActiveJob"
-  gem.description = gem.summary
+  gem.description = "Run and track data migrations through ActiveRecord tasks and ActiveJob workers."
   gem.metadata = {
-    "homepage" => repository_url,
-    "source_code_uri" => repository_url,
-    "bug_tracker_uri" => "#{repository_url}/issues",
-    "changelog_uri" => "#{repository_url}/blob/main/CHANGELOG.md",
+    "homepage" => "https://github.com/amkisko/data-migration.rb",
+    "source_code_uri" => "https://github.com/amkisko/data-migration.rb",
+    "bug_tracker_uri" => "https://github.com/amkisko/data-migration.rb/issues",
+    "changelog_uri" => "https://github.com/amkisko/data-migration.rb/blob/main/CHANGELOG.md",
     "rubygems_mfa_required" => "true"
   }
 
-  gem.files = `git ls-files`.split("\n")
-  gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.files = `git ls-files`.split("\n").select do |file|
+    root_files.include?(file) || file.start_with?("bin/", "lib/")
+  end
   gem.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
 
-  gem.required_ruby_version = ">= 3"
+  gem.required_ruby_version = ">= 3.4"
   gem.require_paths = ["lib"]
 
   gem.add_dependency "rails", "> 5"
@@ -35,10 +35,22 @@ Gem::Specification.new do |gem|
   gem.add_dependency "activerecord", "> 5"
   gem.add_dependency "activesupport", "> 5"
 
-  gem.add_development_dependency "bundler", "~> 2"
+  gem.add_development_dependency "bundler", ">= 2"
   gem.add_development_dependency "rspec", "~> 3"
+  gem.add_development_dependency "polyrun", ">= 2.2.0"
+  gem.add_development_dependency "prosopite", "~> 2.0"
+  gem.add_development_dependency "webmock", "~> 3"
   gem.add_development_dependency "rspec_junit_formatter", "~> 0.6"
-  gem.add_development_dependency "simplecov", "~> 0.21"
-  gem.add_development_dependency "simplecov-cobertura", "~> 2"
-  gem.add_development_dependency "sqlite3", "~> 2.4"
+  gem.add_development_dependency "simplecov", "~> 0.22"
+  gem.add_development_dependency "simplecov-cobertura", "~> 3"
+  gem.add_development_dependency "rbs", "~> 3"
+  gem.add_development_dependency "standard", "~> 1.52"
+  gem.add_development_dependency "standard-custom", "~> 1.0"
+  gem.add_development_dependency "standard-performance", "~> 1.8"
+  gem.add_development_dependency "standard-rails", "~> 1.5"
+  gem.add_development_dependency "standard-rspec", "~> 0.3"
+  gem.add_development_dependency "rubocop-rails", "~> 2.33"
+  gem.add_development_dependency "rubocop-rspec", "~> 3.8"
+  gem.add_development_dependency "rubocop-thread_safety", "~> 0.7"
+  gem.add_development_dependency "appraisal", "~> 2"
 end

data/lib/data-migration.rb

@@ -3,7 +3,7 @@ require "data_migration/job"
 require "data_migration/task"
 
 module DataMigration
-  VERSION = "1.2.0".freeze
+  VERSION = "2.0.0".freeze
 
   module_function
 

data/lib/data_migration/config.rb

@@ -20,7 +20,7 @@ module DataMigration
 
     attr_writer :generate_spec
     def generate_spec?
-      @generate_spec.nil? ? true : @generate_spec
+      @generate_spec.nil? || @generate_spec
     end
 
     attr_writer :job_class

data/lib/data_migration/job.rb

@@ -7,6 +7,10 @@ module DataMigration
     discard_on StandardError
 
     def perform(task_id, *job_args, **job_kwargs)
+      checked_in = false
+      failed = false
+      migration_started = false
+      migration_class = nil
       task = DataMigration::Task.find(task_id)
       DataMigration.config.monitoring_context.call(task)
 
@@ -15,16 +19,19 @@ module DataMigration
 
       unless task.file_exists?
         DataMigration.notify("#{migration_name} not found")
+        task.update_columns(status: task.class.statuses.fetch("failed"), updated_at: Time.current)
         return
       end
 
       task.job_check_in!(job_id, job_args: job_args, job_kwargs: job_kwargs)
+      checked_in = true
+      migration_started = true
 
       require migration_path
       klass_name = migration_name.gsub(/^[0-9_]+/, "").camelize
-      klass = klass_name.safe_constantize
-      raise "Data migration class #{klass_name} not found" unless klass.is_a?(Class)
-      raise "Data migration class #{klass_name} must implement `perform` method" unless klass.method_defined?(:perform)
+      migration_class = klass_name.safe_constantize
+      raise "Data migration class #{klass_name} not found" unless migration_class.is_a?(Class)
+      raise "Data migration class #{klass_name} must implement `perform` method" unless migration_class.method_defined?(:perform)
 
       if task.started_at.blank?
         task.update!(started_at: Time.current, status: :started)
@@ -38,17 +45,18 @@ module DataMigration
 
       Thread.current[:data_migration_enqueue_called] ||= {}
       Thread.current[:data_migration_enqueue_kwargs] ||= {}
-      klass.define_method(:enqueue) do |**enqueue_kwargs|
-        Thread.current[:data_migration_enqueue_called][klass.name] = true
-        Thread.current[:data_migration_enqueue_kwargs][klass.name] = enqueue_kwargs
+      migration_class.define_method(:enqueue) do |**enqueue_kwargs|
+        Thread.current[:data_migration_enqueue_called][migration_class.name] = true
+        Thread.current[:data_migration_enqueue_kwargs][migration_class.name] = enqueue_kwargs
       end
 
       task.update!(status: :performing, pause_minutes: 0)
-      klass.new.perform(**job_kwargs)
+      migration_class.new.perform(**job_kwargs)
       task.job_check_out!(job_id)
+      checked_in = false
 
-      enqueue_called = Thread.current[:data_migration_enqueue_called].delete(klass.name)
-      enqueue_kwargs = Thread.current[:data_migration_enqueue_kwargs].delete(klass.name)
+      enqueue_called = Thread.current[:data_migration_enqueue_called].delete(migration_class.name)
+      enqueue_kwargs = Thread.current[:data_migration_enqueue_kwargs].delete(migration_class.name)
       if enqueue_called
         if enqueue_kwargs[:background] == false
           self.class.new.perform(task_id, *job_args, **enqueue_kwargs)
@@ -58,6 +66,16 @@ module DataMigration
       else
         task.update!(completed_at: Time.current, status: :completed)
       end
+    rescue
+      failed = migration_started
+      raise
+    ensure
+      task.job_check_out!(job_id, status: (failed ? :failed : nil)) if checked_in
+      task.update_columns(status: task.class.statuses.fetch("failed"), updated_at: Time.current) if failed && !checked_in
+      if migration_class
+        Thread.current[:data_migration_enqueue_called]&.delete(migration_class.name)
+        Thread.current[:data_migration_enqueue_kwargs]&.delete(migration_class.name)
+      end
     end
   end
 end

data/lib/data_migration/task.rb

@@ -23,13 +23,10 @@ module DataMigration
       started: "started",
       performing: "performing",
       paused: "paused",
+      failed: "failed",
       completed: "completed"
     }
-    if ActiveRecord::VERSION::MAJOR >= 7
-      enum :status, STATUS_OPTIONS
-    else
-      enum status: STATUS_OPTIONS
-    end
+    enum :status, STATUS_OPTIONS
 
     validates :name, presence: true
     validates :pause_minutes, numericality: {greater_than_or_equal_to: 0, only_integer: true}, if: -> { pause_minutes.present? }
@@ -49,6 +46,7 @@ module DataMigration
     scope :started, -> { where(status: :started) }
     scope :paused, -> { where(status: :paused) }
     scope :performing, -> { where(status: :performing) }
+    scope :failed, -> { where(status: :failed) }
     scope :completed, -> { where(status: :completed) }
 
     def self.job_class
@@ -98,22 +96,27 @@ module DataMigration
     end
 
     def job_check_in!(job_id, job_args: [], job_kwargs: {})
-      self.current_jobs ||= {}
-
-      raise DataMigration::JobConflictError, "#{user_title} already has job ##{job_id}" if current_jobs.key?(job_id)
-      raise DataMigration::JobConcurrencyLimitError, "#{user_title} reached limit of #{jobs_limit} jobs" if jobs_limit.present? && current_jobs.size >= jobs_limit
-
-      self.current_jobs[job_id] = {
-        ts: Time.current,
-        args: job_args,
-        kwargs: job_kwargs
-      }
-      save!
+      with_lock do
+        self.current_jobs ||= {}
+
+        raise DataMigration::JobConflictError, "#{user_title} already has job ##{job_id}" if current_jobs.key?(job_id)
+        raise DataMigration::JobConcurrencyLimitError, "#{user_title} reached limit of #{jobs_limit} jobs" if jobs_limit.present? && current_jobs.size >= jobs_limit
+
+        current_jobs[job_id] = {
+          ts: Time.current,
+          args: job_args,
+          kwargs: job_kwargs
+        }
+        save!
+      end
     end
 
-    def job_check_out!(job_id)
-      self.current_jobs.delete(job_id)
-      save!
+    def job_check_out!(job_id, status: nil)
+      with_lock do
+        current_jobs.delete(job_id)
+        self.status = status if status
+        save!(validate: false)
+      end
     end
 
     def user_title

data/lib/generators/data_migration/install_generator.rb

@@ -0,0 +1,28 @@
+require "rails/generators"
+require "rails/generators/active_record"
+require "rails/generators/active_record/migration/migration_generator"
+
+module DataMigration
+  module Generators
+    class InstallGenerator < ActiveRecord::Generators::MigrationGenerator
+      source_root File.expand_path("../templates", __FILE__)
+
+      attr_reader :table_exists, :table_columns
+      def create_migration_file
+        if ActiveRecord::Base.connection.table_exists?(name)
+          puts "\e[31mWARNING: Table `#{name}` already exists\e[0m"
+          @table_exists = true
+          @table_columns = ActiveRecord::Base.connection.columns(name)
+        end
+
+        set_local_assigns!
+        validate_file_name!
+        migration_template "install_#{name}.rb", "#{DataMigration.config.schema_migrations_path}/install_#{file_name}.rb"
+      end
+
+      def migration_parent
+        "ActiveRecord::Migration[#{ActiveRecord::Migration.current_version}]"
+      end
+    end
+  end
+end

data/lib/generators/{templates → data_migration/templates}/install_data_migration_tasks.rb.tt

@@ -1,4 +1,8 @@
 class <%= migration_class_name %> < <%= migration_parent %>
+  <% if table_exists %>
+    # Current columns: "<%= table_columns.map(&:name).join("\", \"") %>"
+  <% end %>
+
   def self.up
     create_table :<%= DataMigration.tasks_table_name %>, force: true do |t|
       t.string "name", null: false