data_migrate 10.0.0.rc1 → 11.0.0.rc

data/gemfiles/rails_7.2.gemfile.lock

@@ -0,0 +1,265 @@
+PATH
+  remote: ..
+  specs:
+    data_migrate (11.0.0.rc)
+      activerecord (>= 6.1)
+      railties (>= 6.1)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    actioncable (7.2.0)
+      actionpack (= 7.2.0)
+      activesupport (= 7.2.0)
+      nio4r (~> 2.0)
+      websocket-driver (>= 0.6.1)
+      zeitwerk (~> 2.6)
+    actionmailbox (7.2.0)
+      actionpack (= 7.2.0)
+      activejob (= 7.2.0)
+      activerecord (= 7.2.0)
+      activestorage (= 7.2.0)
+      activesupport (= 7.2.0)
+      mail (>= 2.8.0)
+    actionmailer (7.2.0)
+      actionpack (= 7.2.0)
+      actionview (= 7.2.0)
+      activejob (= 7.2.0)
+      activesupport (= 7.2.0)
+      mail (>= 2.8.0)
+      rails-dom-testing (~> 2.2)
+    actionpack (7.2.0)
+      actionview (= 7.2.0)
+      activesupport (= 7.2.0)
+      nokogiri (>= 1.8.5)
+      racc
+      rack (>= 2.2.4, < 3.2)
+      rack-session (>= 1.0.1)
+      rack-test (>= 0.6.3)
+      rails-dom-testing (~> 2.2)
+      rails-html-sanitizer (~> 1.6)
+      useragent (~> 0.16)
+    actiontext (7.2.0)
+      actionpack (= 7.2.0)
+      activerecord (= 7.2.0)
+      activestorage (= 7.2.0)
+      activesupport (= 7.2.0)
+      globalid (>= 0.6.0)
+      nokogiri (>= 1.8.5)
+    actionview (7.2.0)
+      activesupport (= 7.2.0)
+      builder (~> 3.1)
+      erubi (~> 1.11)
+      rails-dom-testing (~> 2.2)
+      rails-html-sanitizer (~> 1.6)
+    activejob (7.2.0)
+      activesupport (= 7.2.0)
+      globalid (>= 0.3.6)
+    activemodel (7.2.0)
+      activesupport (= 7.2.0)
+    activerecord (7.2.0)
+      activemodel (= 7.2.0)
+      activesupport (= 7.2.0)
+      timeout (>= 0.4.0)
+    activestorage (7.2.0)
+      actionpack (= 7.2.0)
+      activejob (= 7.2.0)
+      activerecord (= 7.2.0)
+      activesupport (= 7.2.0)
+      marcel (~> 1.0)
+    activesupport (7.2.0)
+      base64
+      bigdecimal
+      concurrent-ruby (~> 1.0, >= 1.3.1)
+      connection_pool (>= 2.2.5)
+      drb
+      i18n (>= 1.6, < 2)
+      logger (>= 1.4.2)
+      minitest (>= 5.1)
+      securerandom (>= 0.3)
+      tzinfo (~> 2.0, >= 2.0.5)
+    appraisal (2.5.0)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    ast (2.4.2)
+    base64 (0.2.0)
+    bigdecimal (3.1.8)
+    builder (3.3.0)
+    childprocess (5.0.0)
+    coderay (1.1.3)
+    concurrent-ruby (1.3.4)
+    connection_pool (2.4.1)
+    crass (1.0.6)
+    date (3.3.4)
+    diff-lcs (1.5.1)
+    drb (2.2.1)
+    erubi (1.13.0)
+    globalid (1.2.1)
+      activesupport (>= 6.1)
+    i18n (1.14.5)
+      concurrent-ruby (~> 1.0)
+    iniparse (1.5.0)
+    io-console (0.7.2)
+    irb (1.14.0)
+      rdoc (>= 4.0.0)
+      reline (>= 0.4.2)
+    json (2.7.2)
+    language_server-protocol (3.17.0.3)
+    logger (1.6.0)
+    loofah (2.22.0)
+      crass (~> 1.0.2)
+      nokogiri (>= 1.12.0)
+    mail (2.8.1)
+      mini_mime (>= 0.1.1)
+      net-imap
+      net-pop
+      net-smtp
+    marcel (1.0.4)
+    method_source (1.1.0)
+    mini_mime (1.1.5)
+    minitest (5.25.1)
+    net-imap (0.4.14)
+      date
+      net-protocol
+    net-pop (0.1.2)
+      net-protocol
+    net-protocol (0.2.2)
+      timeout
+    net-smtp (0.5.0)
+      net-protocol
+    nio4r (2.7.3)
+    nokogiri (1.16.7-aarch64-linux)
+      racc (~> 1.4)
+    nokogiri (1.16.7-arm64-darwin)
+      racc (~> 1.4)
+    nokogiri (1.16.7-x86_64-linux)
+      racc (~> 1.4)
+    overcommit (0.63.0)
+      childprocess (>= 0.6.3, < 6)
+      iniparse (~> 1.4)
+      rexml (~> 3.2)
+    parallel (1.24.0)
+    parser (3.3.1.0)
+      ast (~> 2.4.1)
+      racc
+    pry (0.14.2)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    psych (5.1.2)
+      stringio
+    racc (1.8.1)
+    rack (3.1.7)
+    rack-session (2.0.0)
+      rack (>= 3.0.0)
+    rack-test (2.1.0)
+      rack (>= 1.3)
+    rackup (2.1.0)
+      rack (>= 3)
+      webrick (~> 1.8)
+    rails (7.2.0)
+      actioncable (= 7.2.0)
+      actionmailbox (= 7.2.0)
+      actionmailer (= 7.2.0)
+      actionpack (= 7.2.0)
+      actiontext (= 7.2.0)
+      actionview (= 7.2.0)
+      activejob (= 7.2.0)
+      activemodel (= 7.2.0)
+      activerecord (= 7.2.0)
+      activestorage (= 7.2.0)
+      activesupport (= 7.2.0)
+      bundler (>= 1.15.0)
+      railties (= 7.2.0)
+    rails-dom-testing (2.2.0)
+      activesupport (>= 5.0.0)
+      minitest
+      nokogiri (>= 1.6)
+    rails-html-sanitizer (1.6.0)
+      loofah (~> 2.21)
+      nokogiri (~> 1.14)
+    railties (7.2.0)
+      actionpack (= 7.2.0)
+      activesupport (= 7.2.0)
+      irb (~> 1.13)
+      rackup (>= 1.0.0)
+      rake (>= 12.2)
+      thor (~> 1.0, >= 1.2.2)
+      zeitwerk (~> 2.6)
+    rainbow (3.1.1)
+    rake (13.2.1)
+    rb-readline (0.5.5)
+    rdoc (6.7.0)
+      psych (>= 4.0.0)
+    regexp_parser (2.9.2)
+    reline (0.5.9)
+      io-console (~> 0.5)
+    rexml (3.2.8)
+      strscan (>= 3.0.9)
+    rspec (3.13.0)
+      rspec-core (~> 3.13.0)
+      rspec-expectations (~> 3.13.0)
+      rspec-mocks (~> 3.13.0)
+    rspec-core (3.13.0)
+      rspec-support (~> 3.13.0)
+    rspec-expectations (3.13.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.13.0)
+    rspec-mocks (3.13.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.13.0)
+    rspec-support (3.13.1)
+    rubocop (1.64.0)
+      json (~> 2.3)
+      language_server-protocol (>= 3.17.0)
+      parallel (~> 1.10)
+      parser (>= 3.3.0.2)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 2.4.0, < 3.0)
+    rubocop-ast (1.31.3)
+      parser (>= 3.3.1.0)
+    ruby-progressbar (1.13.0)
+    securerandom (0.3.1)
+    sqlite3 (2.0.2-aarch64-linux-gnu)
+    sqlite3 (2.0.2-arm64-darwin)
+    sqlite3 (2.0.2-x86_64-linux-gnu)
+    stringio (3.1.1)
+    strscan (3.1.0)
+    thor (1.3.1)
+    timecop (0.9.8)
+    timeout (0.4.1)
+    tzinfo (2.0.6)
+      concurrent-ruby (~> 1.0)
+    unicode-display_width (2.5.0)
+    useragent (0.16.10)
+    webrick (1.8.1)
+    websocket-driver (0.7.6)
+      websocket-extensions (>= 0.1.0)
+    websocket-extensions (0.1.5)
+    zeitwerk (2.6.17)
+
+PLATFORMS
+  aarch64-linux
+  arm64-darwin-23
+  x86_64-linux
+
+DEPENDENCIES
+  appraisal
+  data_migrate!
+  overcommit
+  pry
+  rails (~> 7.2.0)
+  rake
+  rb-readline
+  rspec
+  rspec-core
+  rubocop
+  sqlite3
+  timecop
+
+BUNDLED WITH
+   2.4.17

data/lib/data_migrate/data_migrator.rb

@@ -5,32 +5,20 @@ require "data_migrate/config"
 
 module DataMigrate
   class DataMigrator < ActiveRecord::Migrator
-    def self.migrations_paths
-      [DataMigrate.config.data_migrations_path]
-    end
-
-    def self.assure_data_schema_table
-      DataMigrate::DataSchemaMigration.create_table
-    end
-
-    def initialize(direction, migrations, target_version = nil)
-      @direction         = direction
-      @target_version    = target_version
-      @migrated_versions = nil
-      @migrations        = migrations
-
-      validate(@migrations)
-
-      DataMigrate::DataSchemaMigration.create_table
-      ActiveRecord::InternalMetadata.create_table
-    end
-
     def load_migrated
       @migrated_versions =
-        DataMigrate::DataSchemaMigration.normalized_versions.map(&:to_i).sort
+        DataMigrate::RailsHelper.data_schema_migration.normalized_versions.map(&:to_i).sort
     end
 
     class << self
+      def migrations_paths
+        [DataMigrate.config.data_migrations_path]
+      end
+
+      def create_data_schema_table
+        DataMigrate::RailsHelper.data_schema_migration.create_table
+      end
+
       def current_version
         DataMigrate::MigrationContext.new(migrations_paths).current_version
       end
@@ -66,6 +54,10 @@ module DataMigrate
 
       #TODO: this was added to be backward compatible, need to re-evaluate
       def run(direction, migration_paths, version)
+        # Ensure all Active Record model cache is reset for each data migration
+        # As recommended in: https://github.com/rails/rails/blob/da21c2e9812e5eb0698fba4a9aa38632fc004432/activerecord/lib/active_record/migration.rb#L467-L470
+        ActiveRecord::Base.descendants.each(&:reset_column_information)
+
         DataMigrate::MigrationContext.new(migration_paths).run(direction, version)
       end
 
@@ -79,10 +71,10 @@ module DataMigrate
     def record_version_state_after_migrating(version)
       if down?
         migrated.delete(version)
-        DataMigrate::DataSchemaMigration.where(version: version.to_s).delete_all
+        DataMigrate::RailsHelper.data_schema_delete_version(version.to_s)
       else
         migrated << version
-        DataMigrate::DataSchemaMigration.create!(version: version.to_s)
+        DataMigrate::RailsHelper.data_schema_migration.create_version(version.to_s)
       end
     end
   end

data/lib/data_migrate/data_schema.rb

@@ -9,7 +9,7 @@ module DataMigrate
     #   ActiveRecord::ConnectionAdapters::SchemaStatements
     #     #assume_migrated_upto_version
     def define(info)
-      DataMigrate::DataMigrator.assure_data_schema_table
+      DataMigrate::DataMigrator.create_data_schema_table
 
       return if info[:version].blank?
 
@@ -57,7 +57,7 @@ module DataMigrate
     end
 
     def table_name
-      DataMigrate::DataSchemaMigration.table_name
+      DataMigrate::RailsHelper.data_schema_migration.table_name
     end
   end
 end

data/lib/data_migrate/data_schema_migration.rb

@@ -1,12 +1,29 @@
 module DataMigrate
-  class DataSchemaMigration
-    class << self
-      delegate :table_name, :primary_key, :create_table, :normalized_versions, :create, :create!, :table_exists?, :exists?, :where, to: :instance
+  class DataSchemaMigration < ActiveRecord::SchemaMigration
+    # In Rails 7.1+, ActiveRecord::SchemaMigration methods are instance methods
+    # So we only load the appropriate methods depending on Rails version.
+    if DataMigrate::RailsHelper.rails_version_equal_to_or_higher_than_7_1
+      def table_name
+        ActiveRecord::Base.table_name_prefix + 'data_migrations' + ActiveRecord::Base.table_name_suffix
+      end
+
+      def primary_key
+        "version"
+      end
+    else
+      class << self
+        def table_name
+          ActiveRecord::Base.table_name_prefix + 'data_migrations' + ActiveRecord::Base.table_name_suffix
+        end
+
+        def primary_key
+          "version"
+        end
 
-      def instance
-        @instance ||= Class.new(::ActiveRecord::SchemaMigration) do
-          define_singleton_method(:table_name) { ActiveRecord::Base.table_name_prefix + 'data_migrations' + ActiveRecord::Base.table_name_suffix }
-          define_singleton_method(:primary_key) { "version" }
+        def create_version(version)
+          # Note that SchemaMigration.create_version in Rails 7.1 does not
+          # raise an error if validations fail but we retain this behaviour for now.
+          create!(version: version)
         end
       end
     end

data/lib/data_migrate/database_configurations_wrapper.rb

@@ -0,0 +1,11 @@
+module DataMigrate
+  # This wrapper is used to differentiate between
+  # a data and schema db config when running migrations
+  class DatabaseConfigurationWrapper
+    attr_reader :db_config
+
+    def initialize(db_config)
+      @db_config = db_config
+    end
+  end
+end

data/lib/data_migrate/database_tasks.rb

@@ -5,67 +5,143 @@ require "data_migrate/config"
 module DataMigrate
   ##
   # This class extends DatabaseTasks to add a schema_file method.
-  class DatabaseTasks
+  module DatabaseTasks
     extend ActiveRecord::Tasks::DatabaseTasks
-
-    class << self
-      def schema_file_type(_format = nil)
-        "data_schema.rb"
+    extend self
+
+    # These method are only introduced in Rails 7.1
+    unless respond_to?(:with_temporary_connection_for_each)
+      def with_temporary_connection_for_each(env: ActiveRecord::Tasks::DatabaseTasks.env, name: nil, &block) # :nodoc:
+        if name
+          db_config = ActiveRecord::Base.configurations.configs_for(env_name: env, name: name)
+          with_temporary_connection(db_config, &block)
+        else
+          ActiveRecord::Base.configurations.configs_for(env_name: env, name: name).each do |db_config|
+            with_temporary_connection(db_config, &block)
+          end
+        end
       end
 
-      def dump_filename(spec_name, format = ActiveRecord::Base.schema_format)
-        filename = if spec_name == "primary"
-          schema_file_type(format)
-        else
-          "#{spec_name}_#{schema_file_type(format)}"
+      def with_temporary_connection(db_config) # :nodoc:
+        with_temporary_pool(db_config) do |pool|
+          yield pool.connection
         end
+      end
 
-        ENV["DATA_SCHEMA"] || File.join(db_dir, filename)
+      def migration_class # :nodoc:
+        ActiveRecord::Base
       end
 
-      def check_schema_file(filename)
-        unless File.exist?(filename)
-          message = +%{#{filename} doesn't exist yet. Run `rake data:migrate` to create it, then try again.}
-          Kernel.abort message
-        end
+      def migration_connection # :nodoc:
+        migration_class.connection
       end
 
-      def pending_migrations
-        sort_migrations(
-          pending_schema_migrations,
-          pending_data_migrations
-        )
+      private def with_temporary_pool(db_config)
+        original_db_config = migration_class.connection_db_config
+        pool = migration_class.connection_handler.establish_connection(db_config)
+
+        yield pool
+      ensure
+        migration_class.connection_handler.establish_connection(original_db_config)
       end
+    end
+
+    def db_configs_with_versions
+      db_configs_with_versions = Hash.new { |h, k| h[k] = [] }
+
+      with_temporary_connection_for_each do |conn|
+        db_config = conn.pool.db_config
+        if db_config.primary?
+          versions_to_run = DataMigrate::DatabaseTasks.pending_data_migrations.map { |m| m[:version] }
+          target_version = ActiveRecord::Tasks::DatabaseTasks.target_version
 
-      def sort_migrations(*migrations)
-        migrations.flatten.sort { |a, b|  sort_string(a) <=> sort_string(b) }
+          versions_to_run.each do |version|
+            next if target_version && target_version != version
+            db_configs_with_versions[version] << DatabaseConfigurationWrapper.new(db_config)
+          end
+        end
       end
 
-      def sort_string migration
-        "#{migration[:version]}_#{migration[:kind] == :data ? 1 : 0}"
+      db_configs_with_versions
+    end
+
+    def schema_file(_format = nil)
+      File.join(db_dir, "data_schema.rb")
+    end
+
+    def schema_file_type(_format = nil)
+      "data_schema.rb"
+    end
+
+    # This method is removed in Rails 7.0
+    def dump_filename(spec_name, format = ActiveRecord::Base.schema_format)
+      filename = if spec_name == "primary"
+        schema_file_type(format)
+      else
+        "#{spec_name}_#{schema_file_type(format)}"
       end
 
-      def data_migrations_path
-        ::DataMigrate.config.data_migrations_path
+      ENV["DATA_SCHEMA"] || File.join(db_dir, filename)
+    end
+
+    def check_schema_file(filename)
+      unless File.exist?(filename)
+        message = +%{#{filename} doesn't exist yet. Run `rake data:migrate` to create it, then try again.}
+        Kernel.abort message
       end
+    end
 
-      def run_migration(migration, direction)
-        if migration[:kind] == :data
-          ::ActiveRecord::Migration.write("== %s %s" % ['Data', "=" * 71])
-          ::DataMigrate::DataMigrator.run(direction, data_migrations_path, migration[:version])
-        else
-          ::ActiveRecord::Migration.write("== %s %s" % ['Schema', "=" * 69])
-          ::DataMigrate::SchemaMigration.run(
-            direction,
-            ::DataMigrate::SchemaMigration.migrations_paths,
-            migration[:version]
-          )
-        end
+    def pending_migrations
+      sort_migrations(
+        pending_schema_migrations,
+        pending_data_migrations
+      )
+    end
+
+    def sort_migrations(*migrations)
+      migrations.flatten.sort { |a, b|  sort_string(a) <=> sort_string(b) }
+    end
+
+    def sort_string migration
+      "#{migration[:version]}_#{migration[:kind] == :data ? 1 : 0}"
+    end
+
+    def data_migrations_path
+      ::DataMigrate.config.data_migrations_path
+    end
+
+    def run_migration(migration, direction)
+      if migration[:kind] == :data
+        ::ActiveRecord::Migration.write("== %s %s" % ['Data', "=" * 71])
+        ::DataMigrate::DataMigrator.run(direction, data_migrations_path, migration[:version])
+      else
+        ::ActiveRecord::Migration.write("== %s %s" % ['Schema', "=" * 69])
+        ::DataMigrate::SchemaMigration.run(
+          direction,
+          ::DataMigrate::SchemaMigration.migrations_paths,
+          migration[:version]
+        )
       end
     end
 
-    def self.forward(step = 1)
-      DataMigrate::DataMigrator.assure_data_schema_table
+    def schema_dump_path(db_config, format = ActiveRecord.schema_format)
+      return ENV["DATA_SCHEMA"] if ENV["DATA_SCHEMA"]
+
+      # We only require a schema.rb file for the primary database
+      return unless db_config.primary?
+
+      File.join(File.dirname(ActiveRecord::Tasks::DatabaseTasks.schema_dump_path(db_config, format)), schema_file_type)
+    end
+
+    # Override this method from `ActiveRecord::Tasks::DatabaseTasks`
+    # to ensure that the sha saved in ar_internal_metadata table
+    # is from the original schema.rb file
+    def schema_sha1(file)
+      ActiveRecord::Tasks::DatabaseTasks.schema_dump_path(ActiveRecord::Base.configurations.configs_for(env_name: ActiveRecord::Tasks::DatabaseTasks.env, name: "primary"))
+    end
+
+    def forward(step = 1)
+      DataMigrate::DataMigrator.create_data_schema_table
       migrations = pending_migrations.reverse.pop(step).reverse
       migrations.each do | pending_migration |
         if pending_migration[:kind] == :data
@@ -78,22 +154,92 @@ module DataMigrate
       end
     end
 
-    def self.pending_data_migrations
+    def pending_data_migrations
       data_migrations = DataMigrate::DataMigrator.migrations(data_migrations_path)
-      sort_migrations(DataMigrate::DataMigrator.new(:up, data_migrations ).
-        pending_migrations.map {|m| { version: m.version, name: m.name, kind: :data }})
+      data_migrator = DataMigrate::RailsHelper.data_migrator(:up, data_migrations)
+      sort_migrations(
+        data_migrator.pending_migrations.map { |m| { version: m.version, name: m.name, kind: :data } }
+        )
     end
 
-    def self.pending_schema_migrations
+    def pending_schema_migrations
       ::DataMigrate::SchemaMigration.pending_schema_migrations
     end
 
-    def self.past_migrations(sort = nil)
-      data_versions = DataMigrate::DataSchemaMigration.table_exists? ? DataMigrate::DataSchemaMigration.normalized_versions : []
-      schema_versions = ActiveRecord::SchemaMigration.normalized_versions
+    def past_migrations(sort = nil)
+      data_versions = DataMigrate::RailsHelper.data_schema_migration.table_exists? ? DataMigrate::RailsHelper.data_schema_migration.normalized_versions : []
+      schema_versions = DataMigrate::RailsHelper.schema_migration.normalized_versions
       migrations = data_versions.map { |v| { version: v.to_i, kind: :data } } + schema_versions.map { |v| { version: v.to_i, kind: :schema } }
 
       sort&.downcase == "asc" ? sort_migrations(migrations) : sort_migrations(migrations).reverse
     end
+
+    def self.migrate_with_data
+      DataMigrate::DataMigrator.create_data_schema_table
+
+      ActiveRecord::Migration.verbose = ENV["VERBOSE"] ? ENV["VERBOSE"] == "true" : true
+
+      db_configs = ActiveRecord::Base.configurations.configs_for(env_name: ActiveRecord::Tasks::DatabaseTasks.env)
+
+      schema_mapped_versions = ActiveRecord::Tasks::DatabaseTasks.db_configs_with_versions(db_configs)
+      data_mapped_versions = DataMigrate::DatabaseTasks.db_configs_with_versions
+
+      mapped_versions = schema_mapped_versions.merge(data_mapped_versions) do |_key, schema_db_configs, data_db_configs|
+        schema_db_configs + data_db_configs
+      end
+
+      mapped_versions.sort.each do |version, db_configs|
+        db_configs.each do |db_config|
+          if is_data_migration = db_config.is_a?(DataMigrate::DatabaseConfigurationWrapper)
+            db_config = db_config.db_config
+          end
+
+          DataMigrate::DatabaseTasks.with_temporary_connection(db_config) do
+            if is_data_migration
+              DataMigrate::DataMigrator.run(:up, DataMigrate::DatabaseTasks.data_migrations_path, version)
+            else
+              ActiveRecord::Tasks::DatabaseTasks.migrate(version)
+            end
+          end
+        end
+      end
+    end
+
+    def self.prepare_all_with_data
+      seed = false
+
+      each_current_configuration(env) do |db_config|
+        next unless db_config.primary?
+
+        with_temporary_pool(db_config) do
+          begin
+            database_initialized = migration_connection.schema_migration.table_exists?
+          rescue ActiveRecord::NoDatabaseError
+            create(db_config)
+            retry
+          end
+
+          unless database_initialized
+            if File.exist?(schema_dump_path(db_config))
+              load_schema(db_config, ActiveRecord.schema_format, nil)
+              load_schema_current(
+                :ruby,
+                ENV["DATA_SCHEMA"]
+              )
+            end
+
+            seed = true
+          end
+
+          migrate_with_data
+          if ActiveRecord.dump_schema_after_migration
+            dump_schema(db_config)
+            DataMigrate::Tasks::DataMigrateTasks.dump
+          end
+        end
+      end
+
+      load_seed if seed
+    end
   end
 end