data_migrate 8.1.1 → 11.3.0

data/lib/data_migrate/database_tasks.rb

@@ -5,57 +5,150 @@ require "data_migrate/config"
 module DataMigrate
   ##
   # This class extends DatabaseTasks to add a schema_file method.
-  class DatabaseTasks
+  module DatabaseTasks
     extend ActiveRecord::Tasks::DatabaseTasks
+    extend self
 
-    class << self
-      def schema_file_type(_format = nil)
-        "data_schema.rb"
-      end
+    if respond_to?(:register_task)
+      register_task(/mysql/,        "ActiveRecord::Tasks::MySQLDatabaseTasks")
+      register_task(/trilogy/,      "ActiveRecord::Tasks::MySQLDatabaseTasks")
+      register_task(/postgresql/,   "ActiveRecord::Tasks::PostgreSQLDatabaseTasks")
+      register_task(/sqlite/,       "ActiveRecord::Tasks::SQLiteDatabaseTasks")
+    end
 
-      def dump_filename(namespace, format = ActiveRecord::Base.schema_format)
-        filename = if namespace == "primary"
-          schema_file_type(format)
+    # These method are only introduced in Rails 7.1
+    unless respond_to?(:with_temporary_pool_for_each)
+      def with_temporary_pool_for_each(env: ActiveRecord::Tasks::DatabaseTasks.env, name: nil, &block) # :nodoc:
+        if name
+          db_config = ActiveRecord::Base.configurations.configs_for(env_name: env, name: name)
+           with_temporary_pool(db_config, &block)
         else
-          "#{namespace}_#{schema_file_type(format)}"
+          ActiveRecord::Base.configurations.configs_for(env_name: env, name: name).each do |db_config|
+             with_temporary_pool(db_config, &block)
+          end
         end
-
-        ENV["DATA_SCHEMA"] || File.join(schema_location, filename)
       end
 
-      def schema_dump_path(db_config, format = ActiveRecord.schema_format)
-        return ENV["DATA_SCHEMA"] if ENV["DATA_SCHEMA"]
-
-        filename = if db_config.primary?
-          schema_file_type(format)
-        else
-          [db_config.name, schema_file_type(format)].join("_")
+      def with_temporary_connection(db_config, &block) # :nodoc:
+        with_temporary_pool(db_config) do |pool|
+          pool.with_connection(&block)
         end
+      end
 
-        return unless filename
+      def migration_class # :nodoc:
+        ActiveRecord::Base
+      end
 
-        File.dirname(filename) == schema_location ? filename : File.join(schema_location, filename)
+      def migration_connection # :nodoc:
+        migration_class.connection
       end
 
-      def schema_location
-        db_dir
+      private def with_temporary_pool(db_config)
+        original_db_config = migration_class.connection_db_config
+        pool = migration_class.connection_handler.establish_connection(db_config)
+
+        yield pool
+      ensure
+        migration_class.connection_handler.establish_connection(original_db_config)
       end
+    end
 
-      def check_schema_file(filename)
-        unless File.exist?(filename)
-          message = +%{#{filename} doesn't exist yet. Run `rake data:migrate` to create it, then try again.}
-          Kernel.abort message
+    def db_configs_with_versions
+      db_configs_with_versions = Hash.new { |h, k| h[k] = [] }
+
+      with_temporary_pool_for_each do |pool|
+        db_config = pool.db_config
+        if db_config.primary?
+          versions_to_run = DataMigrate::DatabaseTasks.pending_data_migrations.map { |m| m[:version] }
+          target_version = ActiveRecord::Tasks::DatabaseTasks.target_version
+
+          versions_to_run.each do |version|
+            next if target_version && target_version != version
+            db_configs_with_versions[version] << DatabaseConfigurationWrapper.new(db_config)
+          end
         end
       end
+
+      db_configs_with_versions
     end
 
-    # This overrides ActiveRecord::Tasks::DatabaseTasks
-    def self.schema_file(_format = nil)
+    def schema_file(_format = nil)
       File.join(db_dir, "data_schema.rb")
     end
 
-    def self.forward(step = 1)
-      DataMigrate::DataMigrator.assure_data_schema_table
+    def schema_file_type(_format = nil)
+      "data_schema.rb"
+    end
+
+    # This method is removed in Rails 7.0
+    def dump_filename(spec_name, format = ActiveRecord::Base.schema_format)
+      filename = if spec_name == "primary"
+        schema_file_type(format)
+      else
+        "#{spec_name}_#{schema_file_type(format)}"
+      end
+
+      ENV["DATA_SCHEMA"] || File.join(db_dir, filename)
+    end
+
+    def check_schema_file(filename)
+      unless File.exist?(filename)
+        message = +%{#{filename} doesn't exist yet. Run `rake data:migrate` to create it, then try again.}
+        Kernel.abort message
+      end
+    end
+
+    def pending_migrations
+      sort_migrations(
+        pending_schema_migrations,
+        pending_data_migrations
+      )
+    end
+
+    def sort_migrations(*migrations)
+      migrations.flatten.sort { |a, b| sort_string(a) <=> sort_string(b) }
+    end
+
+    def sort_string migration
+      "#{migration[:version]}_#{migration[:kind] == :data ? 1 : 0}"
+    end
+
+    def data_migrations_path
+      ::DataMigrate.config.data_migrations_path
+    end
+
+    def run_migration(migration, direction)
+      if migration[:kind] == :data
+        ::ActiveRecord::Migration.write("== %s %s" % ['Data', "=" * 71])
+        ::DataMigrate::DataMigrator.run(direction, data_migrations_path, migration[:version])
+      else
+        ::ActiveRecord::Migration.write("== %s %s" % ['Schema', "=" * 69])
+        ::DataMigrate::SchemaMigration.run(
+          direction,
+          ::DataMigrate::SchemaMigration.migrations_paths,
+          migration[:version]
+        )
+      end
+    end
+
+    def schema_dump_path(db_config, format = ActiveRecord.schema_format)
+      return ENV["DATA_SCHEMA"] if ENV["DATA_SCHEMA"]
+
+      # We only require a schema.rb file for the primary database
+      return unless db_config.primary?
+
+      File.join(File.dirname(ActiveRecord::Tasks::DatabaseTasks.schema_dump_path(db_config, format)), schema_file_type)
+    end
+
+    # Override this method from `ActiveRecord::Tasks::DatabaseTasks`
+    # to ensure that the sha saved in ar_internal_metadata table
+    # is from the original schema.rb file
+    def schema_sha1(file)
+      ActiveRecord::Tasks::DatabaseTasks.schema_dump_path(ActiveRecord::Base.configurations.configs_for(env_name: ActiveRecord::Tasks::DatabaseTasks.env, name: "primary"))
+    end
+
+    def forward(step = 1)
+      DataMigrate::DataMigrator.create_data_schema_table
       migrations = pending_migrations.reverse.pop(step).reverse
       migrations.each do | pending_migration |
         if pending_migration[:kind] == :data
@@ -63,64 +156,135 @@ module DataMigrate
           DataMigrate::DataMigrator.run(:up, data_migrations_path, pending_migration[:version])
         elsif pending_migration[:kind] == :schema
           ActiveRecord::Migration.write("== %s %s" % ["Schema", "=" * 69])
-          DataMigrate::SchemaMigration.run(:up, schema_migrations_path, pending_migration[:version])
+          DataMigrate::SchemaMigration.run(:up, DataMigrate::SchemaMigration.migrations_paths, pending_migration[:version])
         end
       end
     end
 
-    def self.data_migrations_path
-      DataMigrate.config.data_migrations_path
+    def pending_data_migrations
+      data_migrations = DataMigrate::DataMigrator.migrations(data_migrations_path)
+      data_migrator = DataMigrate::RailsHelper.data_migrator(:up, data_migrations)
+      sort_migrations(
+        data_migrator.pending_migrations.map { |m| { version: m.version, name: m.name, kind: :data } }
+      )
     end
 
-    def self.schema_migrations_path
-      "db/migrate/"
+    def pending_schema_migrations
+      ::DataMigrate::SchemaMigration.pending_schema_migrations
     end
 
-    def self.pending_migrations
-      sort_migrations(pending_schema_migrations, pending_data_migrations)
-    end
+    def past_migrations(sort = nil)
+      data_versions = DataMigrate::RailsHelper.data_schema_migration.table_exists? ? DataMigrate::RailsHelper.data_schema_migration.normalized_versions : []
+      schema_versions = DataMigrate::RailsHelper.schema_migration.normalized_versions
+      migrations = data_versions.map { |v| { version: v.to_i, kind: :data } } + schema_versions.map { |v| { version: v.to_i, kind: :schema } }
 
-    def self.pending_data_migrations
-      data_migrations = DataMigrate::DataMigrator.migrations(data_migrations_path)
-      sort_migrations(DataMigrate::DataMigrator.new(:up, data_migrations ).
-        pending_migrations.map {|m| { version: m.version, name: m.name, kind: :data }})
+      sort&.downcase == "asc" ? sort_migrations(migrations) : sort_migrations(migrations).reverse
     end
 
-    def self.pending_schema_migrations
-      ::DataMigrate::SchemaMigration.pending_schema_migrations
+    def self.migrate_with_data
+      DataMigrate::DataMigrator.create_data_schema_table
+
+      ActiveRecord::Migration.verbose = ENV["VERBOSE"] ? ENV["VERBOSE"] == "true" : true
+
+      # 7.2 removes the param for db_configs_with_versions in https://github.com/rails/rails/commit/9572fcb4a0bd5396436689a6a42613886871cd81
+      # 7.1 stable backported the change in https://github.com/rails/rails/commit/c53ec4b60980036b43528829d4b0b7457f759224
+      schema_mapped_versions = if Gem::Dependency.new("railties", ">= 7.1.4").match?("railties", Gem.loaded_specs["railties"].version, true)
+        ActiveRecord::Tasks::DatabaseTasks.db_configs_with_versions
+      else
+        db_configs = ActiveRecord::Base.configurations.configs_for(env_name: ActiveRecord::Tasks::DatabaseTasks.env)
+
+        ActiveRecord::Tasks::DatabaseTasks.db_configs_with_versions(db_configs)
+      end
+
+      data_mapped_versions = DataMigrate::DatabaseTasks.db_configs_with_versions
+
+      mapped_versions = schema_mapped_versions.merge(data_mapped_versions) do |_key, schema_db_configs, data_db_configs|
+        schema_db_configs + data_db_configs
+      end
+
+      mapped_versions.sort.each do |version, db_configs|
+        db_configs.each do |db_config|
+          if is_data_migration = db_config.is_a?(DataMigrate::DatabaseConfigurationWrapper)
+            db_config = db_config.db_config
+          end
+
+          DataMigrate::DatabaseTasks.with_temporary_connection(db_config) do
+            if is_data_migration
+              DataMigrate::DataMigrator.run(:up, DataMigrate::DatabaseTasks.data_migrations_path, version)
+            else
+              ActiveRecord::Tasks::DatabaseTasks.migrate(version)
+            end
+          end
+        end
+      end
     end
 
-    def self.sort_migrations(set1, set2 = nil)
-      migrations = set1 + (set2 || [])
-      migrations.sort {|a, b|  sort_string(a) <=> sort_string(b)}
+    def self.prepare_all_with_data
+      seed = false
+
+      each_current_configuration(env) do |db_config|
+        next unless primary?(db_config)
+
+        with_temporary_pool(db_config) do |pool|
+          connection = pool.respond_to?(:lease_connection) ? pool.lease_connection : pool.connection
+          unless database_exists?(connection)
+            create(db_config)
+            if File.exist?(schema_dump_path(db_config))
+              load_schema(db_config, schema_format, nil)
+              load_schema_current(
+                :ruby,
+                ENV["DATA_SCHEMA"]
+              )
+            end
+
+            seed = true
+          end
+
+          migrate_with_data
+          if dump_schema_after_migration?
+            dump_schema(db_config)
+            DataMigrate::Tasks::DataMigrateTasks.dump
+          end
+        end
+      end
+
+      load_seed if seed
     end
 
-    def self.sort_string(migration)
-      "#{migration[:version]}_#{migration[:kind] == :data ? 1 : 0}"
+    private
+
+    def database_exists?(connection)
+      if connection.respond_to?(:database_exists?)  # Rails 7.1+
+        connection.database_exists?
+      else
+        connection.table_exists?(ActiveRecord::SchemaMigration.table_name)
+      end
+    rescue ActiveRecord::NoDatabaseError
+      false
     end
 
-    def self.past_migrations(sort = nil)
-      sort = sort.downcase if sort
-      db_list_data =
-        if DataMigrate::DataSchemaMigration.table_exists?
-          DataMigrate::DataSchemaMigration.normalized_versions.sort
-        else
-          []
-        end
-      db_list_schema = ActiveRecord::SchemaMigration.normalized_versions.sort.sort
-      migrations = db_list_data.map do |d|
-        {
-          version: d.to_i, kind: :data
-        }
-      end +
-                   db_list_schema.map do |d|
-                     {
-                       version: d.to_i, kind: :schema
-                     }
-                   end
+    def primary?(db_config)
+      if db_config.respond_to?(:primary?)  # Rails 7.0+
+        db_config.primary?
+      else
+        db_config.name == "primary"
+      end
+    end
 
-      sort == "asc" ? sort_migrations(migrations) : sort_migrations(migrations).reverse
+    def dump_schema_after_migration?
+      if ActiveRecord.respond_to?(:dump_schema_after_migration)
+        ActiveRecord.dump_schema_after_migration
+      else
+        ActiveRecord::Base.dump_schema_after_migration
+      end
     end
 
+    def schema_format
+      if ActiveRecord.respond_to?(:schema_format)
+        ActiveRecord.schema_format
+      else
+        ActiveRecord::Base.schema_format
+      end
+    end
   end
 end

data/lib/data_migrate/migration_context.rb

@@ -11,7 +11,8 @@ module DataMigrate
                               migrations
                             end
 
-      DataMigrator.new(:up, selected_migrations, target_version).migrate
+      data_migrator = DataMigrate::RailsHelper.data_migrator(:up, selected_migrations, DataMigrate::RailsHelper.schema_migration, DataMigrate::RailsHelper.internal_metadata, target_version)
+      data_migrator.migrate
     end
 
     def down(target_version = nil)
@@ -22,11 +23,13 @@ module DataMigrate
           migrations
         end
 
-      DataMigrator.new(:down, selected_migrations, target_version).migrate
+      data_migrator = DataMigrate::RailsHelper.data_migrator(:down, selected_migrations, DataMigrate::RailsHelper.schema_migration, DataMigrate::RailsHelper.internal_metadata, target_version)
+      data_migrator.migrate
     end
 
     def run(direction, target_version)
-      DataMigrator.new(direction, migrations, target_version).run
+      data_migrator = DataMigrate::RailsHelper.data_migrator(direction, migrations, DataMigrate::RailsHelper.schema_migration, DataMigrate::RailsHelper.internal_metadata, target_version)
+      data_migrator.run
     end
 
     def current_version
@@ -40,12 +43,12 @@ module DataMigrate
     end
 
     def migrations_status
-      db_list = DataSchemaMigration.normalized_versions
+      db_list = DataMigrate::RailsHelper.data_schema_migration.normalized_versions
 
       file_list = migration_files.map do |file|
         version, name, scope = parse_migration_filename(file)
         raise ActiveRecord::IllegalMigrationNameError.new(file) unless version
-        version = ActiveRecord::SchemaMigration.normalize_migration_number(version)
+        version = DataMigrate::RailsHelper.schema_migration.normalize_migration_number(version)
         status = db_list.delete(version) ? "up" : "down"
         [status, version, (name + scope).humanize]
       end.compact
@@ -60,15 +63,15 @@ module DataMigrate
     private
 
     def get_all_versions
-      if DataMigrate::DataSchemaMigration.table_exists?
-        DataSchemaMigration.normalized_versions.map(&:to_i)
+      if DataMigrate::RailsHelper.data_schema_migration.table_exists?
+        DataMigrate::RailsHelper.data_schema_migration.normalized_versions.map(&:to_i)
       else
         []
       end
     end
 
     def move(direction, steps)
-      migrator = DataMigrator.new(direction, migrations)
+      migrator = DataMigrate::RailsHelper.data_migrator(direction, migrations)
 
       if current_version != 0 && !migrator.current_migration
         raise ActiveRecord::UnknownMigrationVersionError.new(current_version)

data/lib/data_migrate/rails_helper.rb

@@ -0,0 +1,91 @@
+module DataMigrate
+  class RailsHelper
+    class << self
+      def rails_version_equal_to_or_higher_than_7_2
+        return @equal_to_or_higher_than_7_2 if defined?(@equal_to_or_higher_than_7_2)
+
+        @equal_to_or_higher_than_7_2 = Gem::Dependency.new("railties", ">= 7.2.0.alpha").match?("railties", Gem.loaded_specs["railties"].version, true)
+      end
+
+      def rails_version_equal_to_or_higher_than_7_1
+        return @equal_to_or_higher_than_7_1 if defined?(@equal_to_or_higher_than_7_1)
+
+        @equal_to_or_higher_than_7_1 = Gem::Dependency.new("railties", ">= 7.1.0.alpha").match?("railties", Gem.loaded_specs["railties"].version, true)
+      end
+
+      def rails_version_equal_to_or_higher_than_7_0
+        return @rails_version_equal_to_or_higher_than_7_0 if defined?(@rails_version_equal_to_or_higher_than_7_0)
+
+        @rails_version_equal_to_or_higher_than_7_0 = Gem::Dependency.new("railties", ">= 7.0").match?("railties", Gem.loaded_specs["railties"].version, true)
+      end
+
+      def internal_metadata
+        if rails_version_equal_to_or_higher_than_7_2
+          ActiveRecord::Base.connection_pool.internal_metadata
+        elsif rails_version_equal_to_or_higher_than_7_1
+          ActiveRecord::Base.connection.internal_metadata
+        else
+          ActiveRecord::InternalMetadata
+        end
+      end
+
+      def schema_migration
+        if rails_version_equal_to_or_higher_than_7_2
+          ActiveRecord::Base.connection_pool.schema_migration
+        elsif rails_version_equal_to_or_higher_than_7_1
+          ActiveRecord::Base.connection.schema_migration
+        else
+          ActiveRecord::SchemaMigration
+        end
+      end
+
+      def schema_migration_versions
+        if rails_version_equal_to_or_higher_than_7_1
+          schema_migration.versions
+        else
+          schema_migration.all.pluck(:version)
+        end
+      end
+
+      def schema_create_version(version)
+        if rails_version_equal_to_or_higher_than_7_1
+          schema_migration.create_version(version)
+        else
+          schema_migration.create(version: version)
+        end
+      end
+
+      def data_schema_delete_version(version)
+        if rails_version_equal_to_or_higher_than_7_1
+          data_schema_migration.delete_version(version)
+        else
+          data_schema_migration.where(version: version.to_s).delete_all
+        end
+      end
+
+      def data_schema_migration
+        if rails_version_equal_to_or_higher_than_7_2
+          DataMigrate::DataSchemaMigration.new(ActiveRecord::Tasks::DatabaseTasks.migration_connection_pool)
+        elsif rails_version_equal_to_or_higher_than_7_1
+          DataMigrate::DataSchemaMigration.new(ActiveRecord::Tasks::DatabaseTasks.migration_connection)
+        else
+          DataMigrate::DataSchemaMigration
+        end
+      end
+
+      def data_migrator(
+        direction,
+        migrations,
+        schema_migration = DataMigrate::RailsHelper.schema_migration,
+        internal_metadata = DataMigrate::RailsHelper.internal_metadata,
+        target_version = nil
+      )
+        if rails_version_equal_to_or_higher_than_7_1
+          DataMigrate::DataMigrator.new(direction, migrations, schema_migration, internal_metadata, target_version)
+        else
+          DataMigrate::DataMigrator.new(direction, migrations, schema_migration, target_version)
+        end
+      end
+    end
+  end
+end

data/lib/data_migrate/schema_dumper.rb

@@ -26,7 +26,7 @@ module DataMigrate
 
     def initialize(connection)
       @connection = connection
-      all_versions =  DataSchemaMigration.normalized_versions
+      all_versions =  DataMigrate::RailsHelper.data_schema_migration.normalized_versions
 
       @version = begin
                     all_versions.max

data/lib/data_migrate/{schema_migration_six.rb → schema_migration.rb}

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module DataMigrate
   # Helper class to getting access to db schema
   # to allow data/schema combiation tasks
@@ -5,14 +7,14 @@ module DataMigrate
     def self.pending_schema_migrations
       all_migrations = DataMigrate::MigrationContext.new(migrations_paths).migrations
       sort_migrations(
-        ActiveRecord::Migrator.new(:up, all_migrations, ActiveRecord::Base.connection.schema_migration).
+        ActiveRecord::Migrator.new(:up, all_migrations, DataMigrate::RailsHelper.schema_migration, DataMigrate::RailsHelper.internal_metadata).
         pending_migrations.
         map {|m| { version: m.version, kind: :schema }}
       )
     end
 
     def self.run(direction, migration_paths, version)
-      ActiveRecord::MigrationContext.new(migration_paths, ActiveRecord::Base.connection.schema_migration).run(direction, version)
+      ActiveRecord::MigrationContext.new(migration_paths, DataMigrate::RailsHelper.schema_migration).run(direction, version)
     end
 
     def self.sort_migrations(set1, set2 = nil)
@@ -22,9 +24,10 @@ module DataMigrate
 
     def self.migrations_paths
       spec_name = DataMigrate.config.spec_name
-      if spec_name && Rails.version > '6.1'
+      # The positional argument true is to include pre-release versions, such as 7.1.0.alpha
+      if spec_name && Gem::Dependency.new("railties", ">= 7.0").match?("railties", Gem.loaded_specs["railties"].version, true)
         ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, name: spec_name).migrations_paths
-      elsif spec_name
+      elsif spec_name && Gem::Dependency.new("railties", "~> 6.1").match?("railties", Gem.loaded_specs["railties"].version)
         ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, spec_name: spec_name).migrations_paths
       else
         Rails.application.config.paths["db/migrate"].to_a

data/lib/data_migrate/{status_service_five.rb → status_service.rb}

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module DataMigrate
   class StatusService
     class << self
@@ -22,22 +24,18 @@ module DataMigrate
     private
 
     def table_name
-      DataMigrate::DataSchemaMigration.table_name
+      DataMigrate::RailsHelper.data_schema_migration.table_name
     end
 
     def output(stream)
-      unless DataMigrate::DataSchemaMigration.table_exists?
+      unless DataMigrate::RailsHelper.data_schema_migration.table_exists?
         stream.puts "Data migrations table does not exist yet."
         return
       end
 
       # output
-      if ActiveRecord.version >= Gem::Version.new('7.0')
-        stream.puts "\ndatabase: #{ActiveRecord::Base.connection_db_config.configuration_hash[:database]}\n\n"
-      else
-        stream.puts "\ndatabase: #{ActiveRecord::Base.connection_config[:database]}\n\n"
-      end
-      stream.puts "#{'Status'.center(8)}  #{'Migration ID'.ljust(14)}  Migration Name"
+      stream.puts "\ndatabase: #{database_name}\n\n"
+      stream.puts "#{"Status".center(8)}  #{"Migration ID".ljust(14)}  Migration Name"
       stream.puts "-" * 50
       db_list.each do |status, version, name|
         stream.puts "#{status.center(8)}  #{version.ljust(14)}  #{name}"
@@ -45,6 +43,14 @@ module DataMigrate
       stream.puts
     end
 
+    def database_name
+      if Gem::Dependency.new("railties", "~> 7.0").match?("railties", Gem.loaded_specs["railties"].version)
+        ActiveRecord::Base.connection_db_config.configuration_hash[:database]
+      elsif Gem::Dependency.new("railties", "~> 6.1").match?("railties", Gem.loaded_specs["railties"].version)
+        ActiveRecord::Base.connection_config[:database]
+      end
+    end
+
     def db_list
       DataMigrate::DataMigrator.migrations_status
     end