data_migrate 3.2.0 → 11.3.0

data/lib/data_migrate/data_migrator.rb

@@ -1,84 +1,80 @@
-require 'active_record'
+# frozen_string_literal: true
 
-module DataMigrate
+require "active_record"
+require "data_migrate/config"
 
+module DataMigrate
   class DataMigrator < ActiveRecord::Migrator
-
-    def record_version_state_after_migrating(version)
-      if down?
-        migrated.delete(version)
-        DataMigrate::DataSchemaMigration.where(:version => version.to_s).delete_all
-      else
-        migrated << version
-        DataMigrate::DataSchemaMigration.create!(:version => version.to_s)
-      end
+    def load_migrated
+      @migrated_versions =
+        DataMigrate::RailsHelper.data_schema_migration.normalized_versions.map(&:to_i).sort
     end
 
     class << self
-      def get_all_versions(connection = ActiveRecord::Base.connection)
-        if table_exists?(connection, schema_migrations_table_name)
-          # Certain versions of the gem wrote data migration versions into
-          # schema_migrations table. After the fix, it was corrected to write into
-          # data_migrations. However, not to break anything we are going to
-          # get versions from both tables.
-          #
-          # This may cause some problems:
-          # Eg. rake data:versions will show version from the schema_migrations table
-          # which may be a version of actual schema migration and not data migration
-          DataMigrate::DataSchemaMigration.all.map { |x| x.version.to_i }.sort +
-            ActiveRecord::SchemaMigration.all.map { |x| x.version.to_i }.sort
-        else
-          []
-        end
+      def migrations_paths
+        Array.wrap(DataMigrate.config.data_migrations_path)
       end
 
-      def schema_migrations_table_name
-        ActiveRecord::Base.table_name_prefix + 'data_migrations' + ActiveRecord::Base.table_name_suffix
+      def create_data_schema_table
+        DataMigrate::RailsHelper.data_schema_migration.create_table
       end
 
-      def migrations_path
-        'db/data'
+      def current_version
+        DataMigrate::MigrationContext.new(migrations_paths).current_version
       end
 
-      def assure_data_schema_table
-        ActiveRecord::Base.establish_connection(db_config)
-        sm_table = DataMigrate::DataMigrator.schema_migrations_table_name
+      ##
+      # Compares the given filename with what we expect data migration
+      # filenames to be, eg the "20091231235959_some_name.rb" pattern
+      # @param (String) filename
+      # @return (MatchData)
+      def match(filename)
+        /(\d{14})_(.+)\.rb$/.match(filename)
+      end
 
-        unless table_exists?(ActiveRecord::Base.connection, sm_table)
-          create_table(sm_table)
-        end
+      def needs_migration?
+        DataMigrate::DatabaseTasks.pending_migrations.count.positive?
+      end
+      ##
+      # Provides the full migrations_path filepath
+      # @return (String)
+      def full_migrations_path
+        File.join(Rails.root, *migrations_paths.split(File::SEPARATOR))
       end
 
-      private
+      def migrations_status
+        DataMigrate::MigrationContext.new(migrations_paths).migrations_status
+      end
 
-      def create_table(sm_table)
-        ActiveRecord::Base.connection.create_table(sm_table, :id => false) do |schema_migrations_table|
-            schema_migrations_table.column :version, :string, :null => false
-          end
+      # TODO: this was added to be backward compatible, need to re-evaluate
+      def migrations(_migrations_paths)
+        DataMigrate::MigrationContext.new(_migrations_paths).migrations
+      end
 
-          suffix = ActiveRecord::Base.table_name_suffix
-          prefix = ActiveRecord::Base.table_name_prefix
-          index_name = "#{prefix}unique_data_migrations#{suffix}"
+      #TODO: this was added to be backward compatible, need to re-evaluate
+      def run(direction, migration_paths, version)
+        # Ensure all Active Record model cache is reset for each data migration
+        # As recommended in: https://github.com/rails/rails/blob/da21c2e9812e5eb0698fba4a9aa38632fc004432/activerecord/lib/active_record/migration.rb#L467-L470
+        ActiveRecord::Base.descendants.each(&:reset_column_information)
 
-          ActiveRecord::Base.connection.add_index sm_table, :version,
-            :unique => true,
-            :name => index_name
+        DataMigrate::MigrationContext.new(migration_paths).run(direction, version)
       end
 
-      def table_exists?(connection, table_name)
-        # Avoid the warning that table_exists? prints in Rails 5.0 due a change in behavior between
-        # Rails 5.0 and Rails 5.1 of this method with respect to database views.
-        if ActiveRecord.version >= Gem::Version.new('5.0') && ActiveRecord.version < Gem::Version.new('5.1')
-          connection.data_source_exists?(table_name)
-        else
-          connection.table_exists?(schema_migrations_table_name)
-        end
+      def rollback(migrations_path, steps)
+        DataMigrate::MigrationContext.new(migrations_path).rollback(steps)
       end
+    end
 
-      def db_config
-        ActiveRecord::Base.configurations[Rails.env || 'development'] || ENV["DATABASE_URL"]
-      end
+    private
 
+    def record_version_state_after_migrating(version)
+      if down?
+        migrated.delete(version)
+        DataMigrate::RailsHelper.data_schema_delete_version(version.to_s)
+      else
+        migrated << version
+        DataMigrate::RailsHelper.data_schema_migration.create_version(version.to_s)
+      end
     end
   end
 end

data/lib/data_migrate/data_schema.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module DataMigrate
+  ##
+  # Provides the definition method for data_schema.rb
+  class Data < ActiveRecord::Schema
+    # This method is based on the following two methods
+    #   ActiveRecord::Schema#define
+    #   ActiveRecord::ConnectionAdapters::SchemaStatements
+    #     #assume_migrated_upto_version
+    def define(info)
+      DataMigrate::DataMigrator.create_data_schema_table
+
+      return if info[:version].blank?
+
+      version = info[:version].to_i
+
+      unless migrated.include?(version)
+        execute "INSERT INTO #{sm_table} (version) VALUES ('#{version}')"
+      end
+
+      insert(version)
+    end
+
+    private
+
+    def migrated
+      @migrated ||= select_values("SELECT version FROM #{sm_table}").map(&:to_i)
+    end
+
+    def versions
+      @versions ||= Set.new.tap do |versions|
+        DataMigrate::DataMigrator.migrations_paths.each do |path|
+          Dir.foreach(path) do |file|
+            match_data = DataMigrate::DataMigrator.match(file)
+            versions << match_data[1].to_i if match_data
+          end
+        end
+      end
+    end
+
+    def insert(version)
+      inserted = Set.new
+      (versions - migrated).each do |v|
+        if inserted.include?(v)
+          raise "Duplicate data migration #{v}. Please renumber your data " \
+            "migrations to resolve the conflict."
+        elsif v < version
+          execute "INSERT INTO #{sm_table} (version) VALUES ('#{v}')"
+          inserted << v
+        end
+      end
+    end
+
+    def sm_table
+      quote_table_name(table_name)
+    end
+
+    def table_name
+      DataMigrate::RailsHelper.data_schema_migration.table_name
+    end
+  end
+end

data/lib/data_migrate/data_schema_migration.rb

@@ -1,14 +1,31 @@
 module DataMigrate
-  class DataSchemaMigration < ::ActiveRecord::SchemaMigration
-    class << self
+  class DataSchemaMigration < ActiveRecord::SchemaMigration
+    # In Rails 7.1+, ActiveRecord::SchemaMigration methods are instance methods
+    # So we only load the appropriate methods depending on Rails version.
+    if DataMigrate::RailsHelper.rails_version_equal_to_or_higher_than_7_1
       def table_name
-        ActiveRecord::Base.table_name_prefix + 'data_migrations' + ActiveRecord::Base.table_name_suffix
+        ActiveRecord::Base.table_name_prefix + DataMigrate.config.data_migrations_table_name + ActiveRecord::Base.table_name_suffix
       end
 
-      def index_name
-        "#{table_name_prefix}unique_data_migrations#{table_name_suffix}"
+      def primary_key
+        "version"
+      end
+    else
+      class << self
+        def table_name
+          ActiveRecord::Base.table_name_prefix + DataMigrate.config.data_migrations_table_name + ActiveRecord::Base.table_name_suffix
+        end
+
+        def primary_key
+          "version"
+        end
+
+        def create_version(version)
+          # Note that SchemaMigration.create_version in Rails 7.1 does not
+          # raise an error if validations fail but we retain this behaviour for now.
+          create!(version: version)
+        end
       end
     end
   end
 end
-

data/lib/data_migrate/database_configurations_wrapper.rb

@@ -0,0 +1,11 @@
+module DataMigrate
+  # This wrapper is used to differentiate between
+  # a data and schema db config when running migrations
+  class DatabaseConfigurationWrapper
+    attr_reader :db_config
+
+    def initialize(db_config)
+      @db_config = db_config
+    end
+  end
+end

data/lib/data_migrate/database_tasks.rb

@@ -0,0 +1,290 @@
+# frozen_string_literal: true
+
+require "data_migrate/config"
+
+module DataMigrate
+  ##
+  # This class extends DatabaseTasks to add a schema_file method.
+  module DatabaseTasks
+    extend ActiveRecord::Tasks::DatabaseTasks
+    extend self
+
+    if respond_to?(:register_task)
+      register_task(/mysql/,        "ActiveRecord::Tasks::MySQLDatabaseTasks")
+      register_task(/trilogy/,      "ActiveRecord::Tasks::MySQLDatabaseTasks")
+      register_task(/postgresql/,   "ActiveRecord::Tasks::PostgreSQLDatabaseTasks")
+      register_task(/sqlite/,       "ActiveRecord::Tasks::SQLiteDatabaseTasks")
+    end
+
+    # These method are only introduced in Rails 7.1
+    unless respond_to?(:with_temporary_pool_for_each)
+      def with_temporary_pool_for_each(env: ActiveRecord::Tasks::DatabaseTasks.env, name: nil, &block) # :nodoc:
+        if name
+          db_config = ActiveRecord::Base.configurations.configs_for(env_name: env, name: name)
+           with_temporary_pool(db_config, &block)
+        else
+          ActiveRecord::Base.configurations.configs_for(env_name: env, name: name).each do |db_config|
+             with_temporary_pool(db_config, &block)
+          end
+        end
+      end
+
+      def with_temporary_connection(db_config, &block) # :nodoc:
+        with_temporary_pool(db_config) do |pool|
+          pool.with_connection(&block)
+        end
+      end
+
+      def migration_class # :nodoc:
+        ActiveRecord::Base
+      end
+
+      def migration_connection # :nodoc:
+        migration_class.connection
+      end
+
+      private def with_temporary_pool(db_config)
+        original_db_config = migration_class.connection_db_config
+        pool = migration_class.connection_handler.establish_connection(db_config)
+
+        yield pool
+      ensure
+        migration_class.connection_handler.establish_connection(original_db_config)
+      end
+    end
+
+    def db_configs_with_versions
+      db_configs_with_versions = Hash.new { |h, k| h[k] = [] }
+
+      with_temporary_pool_for_each do |pool|
+        db_config = pool.db_config
+        if db_config.primary?
+          versions_to_run = DataMigrate::DatabaseTasks.pending_data_migrations.map { |m| m[:version] }
+          target_version = ActiveRecord::Tasks::DatabaseTasks.target_version
+
+          versions_to_run.each do |version|
+            next if target_version && target_version != version
+            db_configs_with_versions[version] << DatabaseConfigurationWrapper.new(db_config)
+          end
+        end
+      end
+
+      db_configs_with_versions
+    end
+
+    def schema_file(_format = nil)
+      File.join(db_dir, "data_schema.rb")
+    end
+
+    def schema_file_type(_format = nil)
+      "data_schema.rb"
+    end
+
+    # This method is removed in Rails 7.0
+    def dump_filename(spec_name, format = ActiveRecord::Base.schema_format)
+      filename = if spec_name == "primary"
+        schema_file_type(format)
+      else
+        "#{spec_name}_#{schema_file_type(format)}"
+      end
+
+      ENV["DATA_SCHEMA"] || File.join(db_dir, filename)
+    end
+
+    def check_schema_file(filename)
+      unless File.exist?(filename)
+        message = +%{#{filename} doesn't exist yet. Run `rake data:migrate` to create it, then try again.}
+        Kernel.abort message
+      end
+    end
+
+    def pending_migrations
+      sort_migrations(
+        pending_schema_migrations,
+        pending_data_migrations
+      )
+    end
+
+    def sort_migrations(*migrations)
+      migrations.flatten.sort { |a, b| sort_string(a) <=> sort_string(b) }
+    end
+
+    def sort_string migration
+      "#{migration[:version]}_#{migration[:kind] == :data ? 1 : 0}"
+    end
+
+    def data_migrations_path
+      ::DataMigrate.config.data_migrations_path
+    end
+
+    def run_migration(migration, direction)
+      if migration[:kind] == :data
+        ::ActiveRecord::Migration.write("== %s %s" % ['Data', "=" * 71])
+        ::DataMigrate::DataMigrator.run(direction, data_migrations_path, migration[:version])
+      else
+        ::ActiveRecord::Migration.write("== %s %s" % ['Schema', "=" * 69])
+        ::DataMigrate::SchemaMigration.run(
+          direction,
+          ::DataMigrate::SchemaMigration.migrations_paths,
+          migration[:version]
+        )
+      end
+    end
+
+    def schema_dump_path(db_config, format = ActiveRecord.schema_format)
+      return ENV["DATA_SCHEMA"] if ENV["DATA_SCHEMA"]
+
+      # We only require a schema.rb file for the primary database
+      return unless db_config.primary?
+
+      File.join(File.dirname(ActiveRecord::Tasks::DatabaseTasks.schema_dump_path(db_config, format)), schema_file_type)
+    end
+
+    # Override this method from `ActiveRecord::Tasks::DatabaseTasks`
+    # to ensure that the sha saved in ar_internal_metadata table
+    # is from the original schema.rb file
+    def schema_sha1(file)
+      ActiveRecord::Tasks::DatabaseTasks.schema_dump_path(ActiveRecord::Base.configurations.configs_for(env_name: ActiveRecord::Tasks::DatabaseTasks.env, name: "primary"))
+    end
+
+    def forward(step = 1)
+      DataMigrate::DataMigrator.create_data_schema_table
+      migrations = pending_migrations.reverse.pop(step).reverse
+      migrations.each do | pending_migration |
+        if pending_migration[:kind] == :data
+          ActiveRecord::Migration.write("== %s %s" % ["Data", "=" * 71])
+          DataMigrate::DataMigrator.run(:up, data_migrations_path, pending_migration[:version])
+        elsif pending_migration[:kind] == :schema
+          ActiveRecord::Migration.write("== %s %s" % ["Schema", "=" * 69])
+          DataMigrate::SchemaMigration.run(:up, DataMigrate::SchemaMigration.migrations_paths, pending_migration[:version])
+        end
+      end
+    end
+
+    def pending_data_migrations
+      data_migrations = DataMigrate::DataMigrator.migrations(data_migrations_path)
+      data_migrator = DataMigrate::RailsHelper.data_migrator(:up, data_migrations)
+      sort_migrations(
+        data_migrator.pending_migrations.map { |m| { version: m.version, name: m.name, kind: :data } }
+      )
+    end
+
+    def pending_schema_migrations
+      ::DataMigrate::SchemaMigration.pending_schema_migrations
+    end
+
+    def past_migrations(sort = nil)
+      data_versions = DataMigrate::RailsHelper.data_schema_migration.table_exists? ? DataMigrate::RailsHelper.data_schema_migration.normalized_versions : []
+      schema_versions = DataMigrate::RailsHelper.schema_migration.normalized_versions
+      migrations = data_versions.map { |v| { version: v.to_i, kind: :data } } + schema_versions.map { |v| { version: v.to_i, kind: :schema } }
+
+      sort&.downcase == "asc" ? sort_migrations(migrations) : sort_migrations(migrations).reverse
+    end
+
+    def self.migrate_with_data
+      DataMigrate::DataMigrator.create_data_schema_table
+
+      ActiveRecord::Migration.verbose = ENV["VERBOSE"] ? ENV["VERBOSE"] == "true" : true
+
+      # 7.2 removes the param for db_configs_with_versions in https://github.com/rails/rails/commit/9572fcb4a0bd5396436689a6a42613886871cd81
+      # 7.1 stable backported the change in https://github.com/rails/rails/commit/c53ec4b60980036b43528829d4b0b7457f759224
+      schema_mapped_versions = if Gem::Dependency.new("railties", ">= 7.1.4").match?("railties", Gem.loaded_specs["railties"].version, true)
+        ActiveRecord::Tasks::DatabaseTasks.db_configs_with_versions
+      else
+        db_configs = ActiveRecord::Base.configurations.configs_for(env_name: ActiveRecord::Tasks::DatabaseTasks.env)
+
+        ActiveRecord::Tasks::DatabaseTasks.db_configs_with_versions(db_configs)
+      end
+
+      data_mapped_versions = DataMigrate::DatabaseTasks.db_configs_with_versions
+
+      mapped_versions = schema_mapped_versions.merge(data_mapped_versions) do |_key, schema_db_configs, data_db_configs|
+        schema_db_configs + data_db_configs
+      end
+
+      mapped_versions.sort.each do |version, db_configs|
+        db_configs.each do |db_config|
+          if is_data_migration = db_config.is_a?(DataMigrate::DatabaseConfigurationWrapper)
+            db_config = db_config.db_config
+          end
+
+          DataMigrate::DatabaseTasks.with_temporary_connection(db_config) do
+            if is_data_migration
+              DataMigrate::DataMigrator.run(:up, DataMigrate::DatabaseTasks.data_migrations_path, version)
+            else
+              ActiveRecord::Tasks::DatabaseTasks.migrate(version)
+            end
+          end
+        end
+      end
+    end
+
+    def self.prepare_all_with_data
+      seed = false
+
+      each_current_configuration(env) do |db_config|
+        next unless primary?(db_config)
+
+        with_temporary_pool(db_config) do |pool|
+          connection = pool.respond_to?(:lease_connection) ? pool.lease_connection : pool.connection
+          unless database_exists?(connection)
+            create(db_config)
+            if File.exist?(schema_dump_path(db_config))
+              load_schema(db_config, schema_format, nil)
+              load_schema_current(
+                :ruby,
+                ENV["DATA_SCHEMA"]
+              )
+            end
+
+            seed = true
+          end
+
+          migrate_with_data
+          if dump_schema_after_migration?
+            dump_schema(db_config)
+            DataMigrate::Tasks::DataMigrateTasks.dump
+          end
+        end
+      end
+
+      load_seed if seed
+    end
+
+    private
+
+    def database_exists?(connection)
+      if connection.respond_to?(:database_exists?)  # Rails 7.1+
+        connection.database_exists?
+      else
+        connection.table_exists?(ActiveRecord::SchemaMigration.table_name)
+      end
+    rescue ActiveRecord::NoDatabaseError
+      false
+    end
+
+    def primary?(db_config)
+      if db_config.respond_to?(:primary?)  # Rails 7.0+
+        db_config.primary?
+      else
+        db_config.name == "primary"
+      end
+    end
+
+    def dump_schema_after_migration?
+      if ActiveRecord.respond_to?(:dump_schema_after_migration)
+        ActiveRecord.dump_schema_after_migration
+      else
+        ActiveRecord::Base.dump_schema_after_migration
+      end
+    end
+
+    def schema_format
+      if ActiveRecord.respond_to?(:schema_format)
+        ActiveRecord.schema_format
+      else
+        ActiveRecord::Base.schema_format
+      end
+    end
+  end
+end

data/lib/data_migrate/migration_context.rb

@@ -0,0 +1,93 @@
+module DataMigrate
+  class MigrationContext < ActiveRecord::MigrationContext
+    def initialize(migrations_paths = "db/data")
+      @migrations_paths = migrations_paths || "db/data"
+    end
+
+    def up(target_version = nil)
+      selected_migrations = if block_given?
+                              migrations.select { |m| yield m }
+                            else
+                              migrations
+                            end
+
+      data_migrator = DataMigrate::RailsHelper.data_migrator(:up, selected_migrations, DataMigrate::RailsHelper.schema_migration, DataMigrate::RailsHelper.internal_metadata, target_version)
+      data_migrator.migrate
+    end
+
+    def down(target_version = nil)
+      selected_migrations =
+        if block_given?
+          migrations.select { |m| yield m }
+        else
+          migrations
+        end
+
+      data_migrator = DataMigrate::RailsHelper.data_migrator(:down, selected_migrations, DataMigrate::RailsHelper.schema_migration, DataMigrate::RailsHelper.internal_metadata, target_version)
+      data_migrator.migrate
+    end
+
+    def run(direction, target_version)
+      data_migrator = DataMigrate::RailsHelper.data_migrator(direction, migrations, DataMigrate::RailsHelper.schema_migration, DataMigrate::RailsHelper.internal_metadata, target_version)
+      data_migrator.run
+    end
+
+    def current_version
+      get_all_versions.max || 0
+    rescue ActiveRecord::NoDatabaseError
+    end
+
+    def migration_files
+      paths = Array(migrations_paths)
+      Dir[*paths.flat_map { |path| "#{path}/**/[0-9]*_*.rb" }]
+    end
+
+    def migrations_status
+      db_list = DataMigrate::RailsHelper.data_schema_migration.normalized_versions
+
+      file_list = migration_files.map do |file|
+        version, name, scope = parse_migration_filename(file)
+        raise ActiveRecord::IllegalMigrationNameError.new(file) unless version
+        version = DataMigrate::RailsHelper.schema_migration.normalize_migration_number(version)
+        status = db_list.delete(version) ? "up" : "down"
+        [status, version, (name + scope).humanize]
+      end.compact
+
+      db_list.map! do |version|
+        ["up", version, "********** NO FILE **********"]
+      end
+
+      (db_list + file_list).sort_by { |_, version, _| version }
+    end
+
+    private
+
+    def get_all_versions
+      if DataMigrate::RailsHelper.data_schema_migration.table_exists?
+        DataMigrate::RailsHelper.data_schema_migration.normalized_versions.map(&:to_i)
+      else
+        []
+      end
+    end
+
+    def move(direction, steps)
+      migrator = DataMigrate::RailsHelper.data_migrator(direction, migrations)
+
+      if current_version != 0 && !migrator.current_migration
+        raise ActiveRecord::UnknownMigrationVersionError.new(current_version)
+      end
+
+      start_index =
+        if current_version.zero?
+          0
+        else
+          migrator.migrations.index(migrator.current_migration)
+        end
+
+      finish = migrator.migrations[start_index + steps]
+      version = finish ? finish.version : 0
+      send(direction, version)
+    end
+
+  end
+end