data-migration 1.0.0

data/spec/data_migration/task_spec.rb

@@ -0,0 +1,152 @@
+require "spec_helper"
+
+describe DataMigration::Task do
+  subject(:task) { DataMigration::Task.create!(name: migration_name) }
+
+  let(:migration_name) { "20241206200111_create_users" }
+  let(:data_migrations_full_path) { File.expand_path("../fixtures/data_migrations", __dir__) }
+
+  before do
+    # allow(Rails).to receive(:root).and_return(rails_root)
+    # allow(Rails).to receive(:logger).and_return(rails_logger)
+    allow(DataMigration.config).to receive(:data_migrations_full_path).and_return(data_migrations_full_path)
+  end
+
+  describe "#file_path" do
+    it "returns the full path to the migration file" do
+      expect(task.file_path).to eq("#{data_migrations_full_path}/#{migration_name}.rb")
+    end
+  end
+
+  describe "#file_exists?" do
+    it "returns true if the file exists" do
+      expect(task.file_exists?).to be(true)
+      expect(task.valid?).to be(true)
+    end
+
+    context "when the file does not exist" do
+      let(:migration_name) { "20241206200111_create_something_else" }
+      let(:task) { DataMigration::Task.new(name: migration_name) }
+
+      it "returns false" do
+        expect(task.file_exists?).to be(false)
+        expect(task.valid?).to be(false)
+      end
+    end
+  end
+
+  describe "#requires_pause?" do
+    before do
+      task.status = :started
+      task.pause_minutes = 10
+    end
+
+    it "returns true if pause_minutes is positive and status is not paused" do
+      expect(task.requires_pause?).to be(true)
+    end
+
+    context "when pause_minutes is 0" do
+      before do
+        task.pause_minutes = 0
+      end
+
+      it "returns false" do
+        expect(task.requires_pause?).to be(false)
+      end
+    end
+
+    context "when status is paused" do
+      before do
+        task.status = :paused
+      end
+
+      it "returns false" do
+        expect(task.requires_pause?).to be(false)
+      end
+    end
+  end
+
+  describe "#perform_now" do
+    it "calls the job class with the task and arguments" do
+      kwargs = { foo: "bar" }
+      expect(DataMigration::Job).to receive(:perform_now).with(task.id, **kwargs)
+      task.perform_now(**kwargs)
+    end
+  end
+
+  describe "#perform_later" do
+    it "calls the job class with the task and arguments" do
+      kwargs = { foo: "bar" }
+      expect(DataMigration::Job).to receive(:perform_later).with(task.id, **kwargs)
+      task.perform_later(**kwargs)
+    end
+  end
+
+  describe "#prepare" do
+    subject(:task) { DataMigration::Task.prepare(migration_name, pause_minutes: 10, jobs_limit: 5) }
+
+    it "creates a new task with the given name, pause, and jobs_limit" do
+      expect { task }.to change(DataMigration::Task, :count).by(1)
+      expect(task.name).to eq(migration_name)
+      expect(task.status).to be_nil
+      expect(task.pause_minutes).to eq(10)
+      expect(task.jobs_limit).to eq(5)
+    end
+
+    context "when chained with perform_now" do
+      it "calls the job class with the task and arguments" do
+        perform_args = { "foo" => "bar" }
+        expect(DataMigration::Job).to receive(:perform_now).with(task.id, **perform_args)
+        task.perform_now(**perform_args)
+        expect(task.kwargs).to eq(perform_args)
+      end
+    end
+  end
+
+  describe "#job_check_in!" do
+    subject(:job_check_in!) { task.job_check_in!(job_id, job_args: ["foo"], job_kwargs: { bar: "baz" }) }
+
+    let(:job_id) { "123" }
+
+    it "adds the job to the current_jobs hash" do
+      expect { job_check_in! }.to change { task.current_jobs.size }.by(1)
+    end
+
+    context "when there is a job with the same id" do
+      before do
+        task.job_check_in!(job_id, job_args: ["foo"], job_kwargs: { bar: "baz" })
+      end
+
+      it "raises a JobConflictError" do
+        expect { job_check_in! }.to raise_error(DataMigration::JobConflictError)
+      end
+    end
+
+    context "when default_jobs_limit is 1 and there are jobs" do
+      before do
+        DataMigration.config.default_jobs_limit = 1
+        task.job_check_in!("321", job_args: ["foo"], job_kwargs: { bar: "baz" })
+      end
+
+      it "raises a JobConcurrencyLimitError" do
+        expect { job_check_in! }.to raise_error(DataMigration::JobConcurrencyLimitError)
+      end
+    end
+  end
+
+  describe "#job_check_out!" do
+    subject(:job_check_out!) { task.job_check_out!(job_id) }
+
+    let(:job_id) { "123" }
+
+    before do
+      task.save!
+      task.job_check_in!(job_id, job_args: ["foo"], job_kwargs: { bar: "baz" })
+    end
+
+    it "removes the job from the current_jobs hash" do
+      expect { job_check_out! }.to change { task.current_jobs.size }.by(-1)
+      expect(task.current_jobs).not_to have_key(job_id)
+    end
+  end
+end

data/spec/data_migration_spec.rb

@@ -0,0 +1,65 @@
+require "spec_helper"
+
+describe DataMigration do
+  let(:gem_specification) { Gem::Specification.load(File.expand_path("../../data-migration.gemspec", __FILE__)) }
+
+  it "has a version number" do
+    expect(described_class::VERSION).to eq gem_specification.version.to_s
+  end
+
+  let(:changelog_file) { File.expand_path("../../CHANGELOG.md", __FILE__) }
+  it "has changelog for the version" do
+    expect(File.exist?(changelog_file)).to be true
+    expect(File.read(changelog_file)).to include("# #{gem_specification.version}")
+  end
+
+  let(:license_file) { File.expand_path("../../LICENSE.md", __FILE__) }
+  it "has license" do
+    expect(File.exist?(license_file)).to be true
+  end
+
+  let(:readme_file) { File.expand_path("../../README.md", __FILE__) }
+  it "has readme" do
+    expect(File.exist?(readme_file)).to be true
+  end
+
+  describe ".notify" do
+    subject(:notify) { DataMigration.notify("test") }
+    let(:action_reporter) { Class.new }
+    before do
+      stub_const("ActionReporter", action_reporter)
+    end
+
+    it "sends message to ActionReporter" do
+      expect(action_reporter).to receive(:notify).with("test", context: {})
+      expect { notify }.not_to raise_error
+    end
+  end
+
+  describe ".perform_now" do
+    subject(:perform_now) { DataMigration.perform_now("test_perform_now") }
+
+    it "delegates to DataMigration::Task.perform_now" do
+      expect(DataMigration::Task).to receive(:perform_now).with("test_perform_now")
+      expect { perform_now }.not_to raise_error
+    end
+  end
+
+  describe ".perform_later" do
+    subject(:perform_later) { DataMigration.perform_later("test_perform_later") }
+
+    it "delegates to DataMigration::Task.perform_later" do
+      expect(DataMigration::Task).to receive(:perform_later).with("test_perform_later")
+      expect { perform_later }.not_to raise_error
+    end
+  end
+
+  describe ".prepare" do
+    subject(:prepare) { DataMigration.prepare("test_prepare") }
+
+    it "delegates to DataMigration::Task.prepare" do
+      expect(DataMigration::Task).to receive(:prepare).with("test_prepare")
+      expect { prepare }.not_to raise_error
+    end
+  end
+end

data/spec/fixtures/data_migrations/20241206200111_create_users.rb

@@ -0,0 +1,17 @@
+class CreateUsers
+  def perform(**kwargs)
+    User.find_or_create_by(email: "test@example.com")
+  end
+end
+
+if ENV["RUN_MIGRATION_TESTS"] && Object.const_defined?(:RSpec)
+  require "rails_helper"
+
+  RSpec.describe CreateUsers, type: :data_migration do
+    subject(:perform) { described_class.new.perform }
+
+    it do
+      expect { perform }.not_to raise_error
+    end
+  end
+end

data/spec/fixtures/data_migrations/20241206200112_create_bad_users.rb

@@ -0,0 +1,5 @@
+class CreateGoodUsers
+  def perform(**kwargs)
+    User.find_or_create_by(email: "test@example.com")
+  end
+end

data/spec/fixtures/data_migrations/20241206200113_change_users.rb

@@ -0,0 +1,5 @@
+class ChangeUsers
+  def change(**kwargs)
+    User.find_or_create_by(email: "test@example.com")
+  end
+end

data/spec/fixtures/data_migrations/20241206200114_create_batch_users.rb

@@ -0,0 +1,9 @@
+class CreateBatchUsers
+  def perform(index: 1, background: false)
+    return if index > 2
+
+    User.find_or_create_by(email: "test_#{index}@example.com")
+
+    enqueue(index: index + 1, background:)
+  end
+end

data/spec/fixtures/schema.rb

@@ -0,0 +1,26 @@
+ActiveRecord::Schema.define(version: 2024_12_06_200411) do
+  create_table :data_migration_tasks, force: true do |t|
+    t.string "name", null: false
+
+    t.json "kwargs", default: {}, null: false
+    t.json "current_jobs", default: {}, null: false
+
+    t.string "status"
+    t.datetime "started_at"
+    t.datetime "completed_at"
+
+    t.bigint "operator_id"
+    t.string "operator_type"
+
+    t.integer "pause_minutes", default: 0, null: false
+    t.integer "jobs_limit"
+
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+  end
+
+  create_table :users, force: true do |t|
+    t.string :email
+    t.string :version
+  end
+end

data/spec/generators/install_generator_spec.rb

@@ -0,0 +1,48 @@
+require "spec_helper"
+
+require "generators/install_generator"
+
+describe InstallGenerator, type: :generator do
+  include FileUtils
+
+  subject(:generator) { described_class.start params }
+
+  let(:root_path) { rails_root(File.expand_path("../../../tmp/rspec", __FILE__)) }
+
+  let(:migration_name) { "data_migration_tasks" }
+  let(:params) { [migration_name] }
+  let(:created_files) { Dir["#{root_path}/db/migrate/*_#{migration_name}.rb"] }
+  let(:migration_content) { File.readlines(created_files.first).reject(&:blank?).map(&:strip) }
+
+  before do
+    mkdir_p root_path.to_s
+    allow(DataMigration.config).to receive(:schema_migrations_path).and_return("#{root_path}/db/migrate")
+    allow(Rails).to receive(:root).and_return(root_path)
+  end
+
+  after do
+    rm_rf root_path.to_s
+  end
+
+  it "creates a migration file" do
+    generator
+
+    expect(created_files).not_to be_empty
+
+    expect(migration_content).to include("create_table :data_migration_tasks, force: true do |t|")
+  end
+
+  context "when tasks_table_name is not default" do
+    before do
+      allow(DataMigration).to receive(:tasks_table_name).and_return("other_data_migration_tasks")
+    end
+
+    it "creates a migration file with the correct table name" do
+      generator
+
+      expect(created_files).not_to be_empty
+
+      expect(migration_content).to include("create_table :other_data_migration_tasks, force: true do |t|")
+    end
+  end
+end

data/spec/generators/migration_generator_spec.rb

@@ -0,0 +1,50 @@
+require "spec_helper"
+
+require "generators/data_migration_generator"
+
+describe DataMigrationGenerator, type: :generator do
+  include FileUtils
+
+  subject(:generator) { described_class.start params }
+
+  let(:root_path) { rails_root(File.expand_path("../../../tmp/rspec", __FILE__)) }
+
+  let(:migration_name) { "create_users" }
+  let(:params) { [migration_name] }
+  let(:created_files) { Dir["#{root_path}/db/data_migrations/*_#{migration_name}.rb"] }
+  let(:migration_content) { File.readlines(created_files.first).reject(&:blank?).map(&:strip) }
+
+  before do
+    mkdir_p root_path.to_s
+    allow(DataMigration.config).to receive(:data_migrations_path).and_return("#{root_path}/db/data_migrations")
+    allow(Rails).to receive(:root).and_return(root_path)
+  end
+
+  after do
+    rm_rf root_path.to_s
+  end
+
+  it "creates a migration file" do
+    generator
+
+    expect(created_files).not_to be_empty
+
+    expect(migration_content).to include("class CreateUsers")
+    expect(migration_content).to include("def perform(**kwargs)")
+    expect(migration_content).to include("RSpec.describe CreateUsers, type: :data_migration do")
+  end
+
+  context "when generate_spec is false" do
+    before do
+      DataMigration.config.generate_spec = false
+    end
+
+    it "does not add RSpec describe block" do
+      generator
+
+      expect(created_files).not_to be_empty
+
+      expect(migration_content).not_to include("RSpec.describe CreateUsers, type: :data_migration do")
+    end
+  end
+end

data/spec/rails_helper.rb

@@ -0,0 +1,21 @@
+require "active_record"
+
+require "data_migration"
+
+require "support/rails_helpers"
+
+RSpec.configure do |config|
+  include RailsHelpers
+
+  config.before(:suite) do
+    ActiveRecord::Base.establish_connection(adapter: "sqlite3", database: ":memory:")
+    load File.expand_path("../fixtures/schema.rb", __FILE__)
+  end
+
+  config.before(:each) do
+    tables = ActiveRecord::Base.connection.tables
+    tables.each do |table|
+      ActiveRecord::Base.connection.execute("DELETE FROM #{table}")
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,30 @@
+require "simplecov"
+SimpleCov.start do
+  add_filter "/spec/"
+  add_filter { |source_file| source_file.lines.count < 5 }
+end
+
+require "simplecov-cobertura"
+SimpleCov.formatter = SimpleCov::Formatter::CoberturaFormatter
+
+require "active_record"
+
+require "data_migration"
+
+Dir[File.expand_path("support/**/*.rb", __dir__)].each { |f| require_relative f }
+
+RSpec.configure do |config|
+  include RailsHelpers
+
+  config.before(:suite) do
+    ActiveRecord::Base.establish_connection(adapter: "sqlite3", database: ":memory:")
+    load File.expand_path("../fixtures/schema.rb", __FILE__)
+  end
+
+  config.before(:each) do
+    tables = ActiveRecord::Base.connection.tables
+    tables.each do |table|
+      ActiveRecord::Base.connection.execute("DELETE FROM #{table}")
+    end
+  end
+end

data/spec/support/junit_formatter.rb

@@ -0,0 +1,6 @@
+if ENV["CI"]
+  require "rspec_junit_formatter"
+  RSpec.configure do |config|
+    config.add_formatter RspecJunitFormatter, "coverage/junit-coverage.xml"
+  end
+end

data/spec/support/rails_helpers.rb

@@ -0,0 +1,53 @@
+module RailsHelpers
+  class User < ActiveRecord::Base
+    self.table_name = "users"
+  end
+
+  def rails_root(destination_root = File.expand_path("../../tmp/rspec", __dir__))
+    Class.new do
+      def initialize(destination_root)
+        @destination_root = destination_root
+      end
+
+      def to_s
+        @destination_root
+      end
+
+      def join(*args)
+        File.join(@destination_root, *args)
+      end
+    end.new(destination_root)
+  end
+
+  def rails_env(env = "development")
+    Class.new do
+      def initialize(env)
+        @env = env.to_s
+      end
+
+      def development?
+        @env == "development"
+      end
+
+      def test?
+        @env == "test"
+      end
+
+      def production?
+        @env == "production"
+      end
+
+      def to_s
+        @env
+      end
+    end.new(env)
+  end
+
+  def rails_logger
+    Class.new do
+      def info(message)
+        puts message
+      end
+    end.new
+  end
+end

data/usr/bin/release.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -e
+
+e() {
+  GREEN='\033[0;32m'
+  NC='\033[0m'
+  echo -e "${GREEN}$1${NC}"
+  eval "$1"
+}
+
+e "bundle"
+e "bundle exec rspec"
+
+if [[ $(git diff --shortstat 2>/dev/null | tail -n1) != "" ]]; then
+  echo -e "\033[1;31mgit working directory not clean, please commit your changes first \033[0m"
+  exit 1
+fi
+
+GEM_NAME="data-migration"
+VERSION=$(grep -Eo "VERSION\s*=\s*\".+\"" lib/data_migration.rb | grep -Eo "[0-9.]{5,}")
+GEM_FILE="$GEM_NAME-$VERSION.gem"
+
+e "gem build $GEM_NAME.gemspec"
+
+echo "Ready to release $GEM_FILE $VERSION"
+read -p "Continue? [Y/n] " answer
+if [[ "$answer" != "Y" ]]; then
+  echo "Exiting"
+  exit 1
+fi
+
+e "gem push $GEM_FILE"
+e "git tag $VERSION && git push --tags"
+e "gh release create $VERSION --generate-notes"