data_migrate 9.2.0 → 9.3.0

data/gemfiles/rails_7.0.gemfile.lock

@@ -1,74 +1,74 @@
 PATH
   remote: ..
   specs:
-    data_migrate (9.2.0)
+    data_migrate (9.3.0)
       activerecord (>= 6.1)
       railties (>= 6.1)
 
 GEM
   remote: http://rubygems.org/
   specs:
-    actioncable (7.0.6)
-      actionpack (= 7.0.6)
-      activesupport (= 7.0.6)
+    actioncable (7.0.8.1)
+      actionpack (= 7.0.8.1)
+      activesupport (= 7.0.8.1)
       nio4r (~> 2.0)
       websocket-driver (>= 0.6.1)
-    actionmailbox (7.0.6)
-      actionpack (= 7.0.6)
-      activejob (= 7.0.6)
-      activerecord (= 7.0.6)
-      activestorage (= 7.0.6)
-      activesupport (= 7.0.6)
+    actionmailbox (7.0.8.1)
+      actionpack (= 7.0.8.1)
+      activejob (= 7.0.8.1)
+      activerecord (= 7.0.8.1)
+      activestorage (= 7.0.8.1)
+      activesupport (= 7.0.8.1)
       mail (>= 2.7.1)
       net-imap
       net-pop
       net-smtp
-    actionmailer (7.0.6)
-      actionpack (= 7.0.6)
-      actionview (= 7.0.6)
-      activejob (= 7.0.6)
-      activesupport (= 7.0.6)
+    actionmailer (7.0.8.1)
+      actionpack (= 7.0.8.1)
+      actionview (= 7.0.8.1)
+      activejob (= 7.0.8.1)
+      activesupport (= 7.0.8.1)
       mail (~> 2.5, >= 2.5.4)
       net-imap
       net-pop
       net-smtp
       rails-dom-testing (~> 2.0)
-    actionpack (7.0.6)
-      actionview (= 7.0.6)
-      activesupport (= 7.0.6)
+    actionpack (7.0.8.1)
+      actionview (= 7.0.8.1)
+      activesupport (= 7.0.8.1)
       rack (~> 2.0, >= 2.2.4)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actiontext (7.0.6)
-      actionpack (= 7.0.6)
-      activerecord (= 7.0.6)
-      activestorage (= 7.0.6)
-      activesupport (= 7.0.6)
+    actiontext (7.0.8.1)
+      actionpack (= 7.0.8.1)
+      activerecord (= 7.0.8.1)
+      activestorage (= 7.0.8.1)
+      activesupport (= 7.0.8.1)
       globalid (>= 0.6.0)
       nokogiri (>= 1.8.5)
-    actionview (7.0.6)
-      activesupport (= 7.0.6)
+    actionview (7.0.8.1)
+      activesupport (= 7.0.8.1)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.1, >= 1.2.0)
-    activejob (7.0.6)
-      activesupport (= 7.0.6)
+    activejob (7.0.8.1)
+      activesupport (= 7.0.8.1)
       globalid (>= 0.3.6)
-    activemodel (7.0.6)
-      activesupport (= 7.0.6)
-    activerecord (7.0.6)
-      activemodel (= 7.0.6)
-      activesupport (= 7.0.6)
-    activestorage (7.0.6)
-      actionpack (= 7.0.6)
-      activejob (= 7.0.6)
-      activerecord (= 7.0.6)
-      activesupport (= 7.0.6)
+    activemodel (7.0.8.1)
+      activesupport (= 7.0.8.1)
+    activerecord (7.0.8.1)
+      activemodel (= 7.0.8.1)
+      activesupport (= 7.0.8.1)
+    activestorage (7.0.8.1)
+      actionpack (= 7.0.8.1)
+      activejob (= 7.0.8.1)
+      activerecord (= 7.0.8.1)
+      activesupport (= 7.0.8.1)
       marcel (~> 1.0)
       mini_mime (>= 1.1.0)
-    activesupport (7.0.6)
+    activesupport (7.0.8.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -81,19 +81,19 @@ GEM
     builder (3.2.4)
     childprocess (4.1.0)
     coderay (1.1.3)
-    concurrent-ruby (1.2.2)
+    concurrent-ruby (1.2.3)
     crass (1.0.6)
-    date (3.3.3)
+    date (3.3.4)
     diff-lcs (1.5.0)
     erubi (1.12.0)
-    globalid (1.1.0)
-      activesupport (>= 5.0)
+    globalid (1.2.1)
+      activesupport (>= 6.1)
     i18n (1.14.1)
       concurrent-ruby (~> 1.0)
     iniparse (1.5.0)
     json (2.6.3)
     language_server-protocol (3.17.0.3)
-    loofah (2.21.3)
+    loofah (2.22.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.12.0)
     mail (2.8.1)
@@ -103,21 +103,21 @@ GEM
       net-smtp
     marcel (1.0.2)
     method_source (1.0.0)
-    mini_mime (1.1.2)
-    minitest (5.18.1)
-    net-imap (0.3.6)
+    mini_mime (1.1.5)
+    minitest (5.22.2)
+    net-imap (0.4.10)
       date
       net-protocol
     net-pop (0.1.2)
       net-protocol
-    net-protocol (0.2.1)
+    net-protocol (0.2.2)
       timeout
-    net-smtp (0.3.3)
+    net-smtp (0.4.0.1)
       net-protocol
-    nio4r (2.5.9)
-    nokogiri (1.15.3-arm64-darwin)
+    nio4r (2.7.0)
+    nokogiri (1.16.2-arm64-darwin)
       racc (~> 1.4)
-    nokogiri (1.15.3-x86_64-linux)
+    nokogiri (1.16.2-x86_64-linux)
       racc (~> 1.4)
     overcommit (0.60.0)
       childprocess (>= 0.6.3, < 5)
@@ -130,40 +130,40 @@ GEM
     pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    racc (1.7.1)
-    rack (2.2.7)
+    racc (1.7.3)
+    rack (2.2.8.1)
     rack-test (2.1.0)
       rack (>= 1.3)
-    rails (7.0.6)
-      actioncable (= 7.0.6)
-      actionmailbox (= 7.0.6)
-      actionmailer (= 7.0.6)
-      actionpack (= 7.0.6)
-      actiontext (= 7.0.6)
-      actionview (= 7.0.6)
-      activejob (= 7.0.6)
-      activemodel (= 7.0.6)
-      activerecord (= 7.0.6)
-      activestorage (= 7.0.6)
-      activesupport (= 7.0.6)
+    rails (7.0.8.1)
+      actioncable (= 7.0.8.1)
+      actionmailbox (= 7.0.8.1)
+      actionmailer (= 7.0.8.1)
+      actionpack (= 7.0.8.1)
+      actiontext (= 7.0.8.1)
+      actionview (= 7.0.8.1)
+      activejob (= 7.0.8.1)
+      activemodel (= 7.0.8.1)
+      activerecord (= 7.0.8.1)
+      activestorage (= 7.0.8.1)
+      activesupport (= 7.0.8.1)
       bundler (>= 1.15.0)
-      railties (= 7.0.6)
-    rails-dom-testing (2.1.1)
+      railties (= 7.0.8.1)
+    rails-dom-testing (2.2.0)
       activesupport (>= 5.0.0)
       minitest
       nokogiri (>= 1.6)
     rails-html-sanitizer (1.6.0)
       loofah (~> 2.21)
       nokogiri (~> 1.14)
-    railties (7.0.6)
-      actionpack (= 7.0.6)
-      activesupport (= 7.0.6)
+    railties (7.0.8.1)
+      actionpack (= 7.0.8.1)
+      activesupport (= 7.0.8.1)
       method_source
       rake (>= 12.2)
       thor (~> 1.0)
       zeitwerk (~> 2.5)
     rainbow (3.1.1)
-    rake (13.0.6)
+    rake (13.1.0)
     rb-readline (0.5.5)
     regexp_parser (2.8.1)
     rexml (3.2.5)
@@ -194,21 +194,22 @@ GEM
     rubocop-ast (1.29.0)
       parser (>= 3.2.1.0)
     ruby-progressbar (1.13.0)
-    sqlite3 (1.6.3-arm64-darwin)
-    sqlite3 (1.6.3-x86_64-linux)
-    thor (1.2.2)
+    sqlite3 (1.7.2-arm64-darwin)
+    sqlite3 (1.7.2-x86_64-linux)
+    thor (1.3.1)
     timecop (0.9.6)
-    timeout (0.4.0)
+    timeout (0.4.1)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     unicode-display_width (2.4.2)
-    websocket-driver (0.7.5)
+    websocket-driver (0.7.6)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
-    zeitwerk (2.6.8)
+    zeitwerk (2.6.13)
 
 PLATFORMS
   arm64-darwin-22
+  arm64-darwin-23
   x86_64-linux
 
 DEPENDENCIES
@@ -216,14 +217,14 @@ DEPENDENCIES
   data_migrate!
   overcommit
   pry
-  rails (~> 7.0)
+  rails (~> 7.0.0)
   rake
   rb-readline
   rspec
   rspec-core
   rubocop
-  sqlite3 (~> 1.4)
+  sqlite3
   timecop
 
 BUNDLED WITH
-   2.4.14
+   2.5.6

data/gemfiles/rails_7.1.gemfile

@@ -2,7 +2,6 @@
 
 source "http://rubygems.org"
 
-gem "sqlite3", "~> 1.4"
 gem "rails", "7.1.0"
 
 gemspec path: "../"

data/gemfiles/rails_7.1.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    data_migrate (9.2.0)
+    data_migrate (9.3.0)
       activerecord (>= 6.1)
       railties (>= 6.1)
 
@@ -86,17 +86,17 @@ GEM
       rake
       thor (>= 0.14.0)
     ast (2.4.2)
-    base64 (0.1.1)
-    bigdecimal (3.1.4)
+    base64 (0.2.0)
+    bigdecimal (3.1.6)
     builder (3.2.4)
     childprocess (4.1.0)
     coderay (1.1.3)
-    concurrent-ruby (1.2.2)
+    concurrent-ruby (1.2.3)
     connection_pool (2.4.1)
     crass (1.0.6)
-    date (3.3.3)
+    date (3.3.4)
     diff-lcs (1.5.0)
-    drb (2.1.1)
+    drb (2.2.0)
       ruby2_keywords
     erubi (1.12.0)
     globalid (1.2.1)
@@ -104,13 +104,13 @@ GEM
     i18n (1.14.1)
       concurrent-ruby (~> 1.0)
     iniparse (1.5.0)
-    io-console (0.6.0)
-    irb (1.8.1)
+    io-console (0.7.2)
+    irb (1.11.2)
       rdoc
-      reline (>= 0.3.8)
+      reline (>= 0.4.2)
     json (2.6.3)
     language_server-protocol (3.17.0.3)
-    loofah (2.21.3)
+    loofah (2.22.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.12.0)
     mail (2.8.1)
@@ -118,24 +118,24 @@ GEM
       net-imap
       net-pop
       net-smtp
-    marcel (1.0.2)
+    marcel (1.0.4)
     method_source (1.0.0)
     mini_mime (1.1.5)
-    minitest (5.20.0)
-    mutex_m (0.1.2)
-    net-imap (0.4.0)
+    minitest (5.22.2)
+    mutex_m (0.2.0)
+    net-imap (0.4.10)
       date
       net-protocol
     net-pop (0.1.2)
       net-protocol
-    net-protocol (0.2.1)
+    net-protocol (0.2.2)
       timeout
-    net-smtp (0.4.0)
+    net-smtp (0.4.0.1)
       net-protocol
-    nio4r (2.5.9)
-    nokogiri (1.15.4-arm64-darwin)
+    nio4r (2.7.0)
+    nokogiri (1.16.2-arm64-darwin)
       racc (~> 1.4)
-    nokogiri (1.15.4-x86_64-linux)
+    nokogiri (1.16.2-x86_64-linux)
       racc (~> 1.4)
     overcommit (0.60.0)
       childprocess (>= 0.6.3, < 5)
@@ -148,10 +148,10 @@ GEM
     pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    psych (5.1.0)
+    psych (5.1.2)
       stringio
-    racc (1.7.1)
-    rack (3.0.8)
+    racc (1.7.3)
+    rack (3.0.9.1)
     rack-session (2.0.0)
       rack (>= 3.0.0)
     rack-test (2.1.0)
@@ -189,12 +189,12 @@ GEM
       thor (~> 1.0, >= 1.2.2)
       zeitwerk (~> 2.6)
     rainbow (3.1.1)
-    rake (13.0.6)
+    rake (13.1.0)
     rb-readline (0.5.5)
-    rdoc (6.5.0)
+    rdoc (6.6.2)
       psych (>= 4.0.0)
     regexp_parser (2.8.1)
-    reline (0.3.9)
+    reline (0.4.3)
       io-console (~> 0.5)
     rexml (3.2.5)
     rspec (3.12.0)
@@ -225,12 +225,12 @@ GEM
       parser (>= 3.2.1.0)
     ruby-progressbar (1.13.0)
     ruby2_keywords (0.0.5)
-    sqlite3 (1.6.3-arm64-darwin)
-    sqlite3 (1.6.3-x86_64-linux)
-    stringio (3.0.8)
-    thor (1.2.2)
+    sqlite3 (1.7.2-arm64-darwin)
+    sqlite3 (1.7.2-x86_64-linux)
+    stringio (3.1.0)
+    thor (1.3.1)
     timecop (0.9.6)
-    timeout (0.4.0)
+    timeout (0.4.1)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     unicode-display_width (2.4.2)
@@ -238,10 +238,11 @@ GEM
     websocket-driver (0.7.6)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
-    zeitwerk (2.6.12)
+    zeitwerk (2.6.13)
 
 PLATFORMS
   arm64-darwin-22
+  arm64-darwin-23
   x86_64-linux
 
 DEPENDENCIES
@@ -255,7 +256,7 @@ DEPENDENCIES
   rspec
   rspec-core
   rubocop
-  sqlite3 (~> 1.4)
+  sqlite3
   timecop
 
 BUNDLED WITH

data/lib/data_migrate/database_configurations_wrapper.rb

@@ -0,0 +1,11 @@
+module DataMigrate
+  # This wrapper is used to differentiate between
+  # a data and schema db config when running migrations
+  class DatabaseConfigurationWrapper
+    attr_reader :db_config
+
+    def initialize(db_config)
+      @db_config = db_config
+    end
+  end
+end

data/lib/data_migrate/database_tasks.rb

@@ -5,87 +5,142 @@ require "data_migrate/config"
 module DataMigrate
   ##
   # This class extends DatabaseTasks to add a schema_file method.
-  class DatabaseTasks
+  module DatabaseTasks
     extend ActiveRecord::Tasks::DatabaseTasks
-
-    class << self
-      def schema_file(_format = nil)
-        File.join(db_dir, "data_schema.rb")
-      end
-
-      def schema_file_type(_format = nil)
-        "data_schema.rb"
-      end
-
-      # This method is removed in Rails 7.0
-      def dump_filename(spec_name, format = ActiveRecord::Base.schema_format)
-        filename = if spec_name == "primary"
-          schema_file_type(format)
+    extend self
+
+    # These method are only introduced in Rails 7.1
+    unless respond_to?(:with_temporary_connection_for_each)
+      def with_temporary_connection_for_each(env: ActiveRecord::Tasks::DatabaseTasks.env, name: nil, &block) # :nodoc:
+        if name
+          db_config = ActiveRecord::Base.configurations.configs_for(env_name: env, name: name)
+          with_temporary_connection(db_config, &block)
         else
-          "#{spec_name}_#{schema_file_type(format)}"
+          ActiveRecord::Base.configurations.configs_for(env_name: env, name: name).each do |db_config|
+            with_temporary_connection(db_config, &block)
+          end
         end
-
-        ENV["DATA_SCHEMA"] || File.join(db_dir, filename)
       end
 
-      def check_schema_file(filename)
-        unless File.exist?(filename)
-          message = +%{#{filename} doesn't exist yet. Run `rake data:migrate` to create it, then try again.}
-          Kernel.abort message
+      def with_temporary_connection(db_config) # :nodoc:
+        with_temporary_pool(db_config) do |pool|
+          yield pool.connection
         end
       end
 
-      def pending_migrations
-        sort_migrations(
-          pending_schema_migrations,
-          pending_data_migrations
-        )
+      def migration_class # :nodoc:
+        ActiveRecord::Base
       end
 
-      def sort_migrations(*migrations)
-        migrations.flatten.sort { |a, b|  sort_string(a) <=> sort_string(b) }
+      def migration_connection # :nodoc:
+        migration_class.connection
       end
 
-      def sort_string migration
-        "#{migration[:version]}_#{migration[:kind] == :data ? 1 : 0}"
-      end
+      private def with_temporary_pool(db_config)
+        original_db_config = migration_class.connection_db_config
+        pool = migration_class.connection_handler.establish_connection(db_config)
 
-      def data_migrations_path
-        ::DataMigrate.config.data_migrations_path
+        yield pool
+      ensure
+        migration_class.connection_handler.establish_connection(original_db_config)
       end
+    end
 
-      def run_migration(migration, direction)
-        if migration[:kind] == :data
-          ::ActiveRecord::Migration.write("== %s %s" % ['Data', "=" * 71])
-          ::DataMigrate::DataMigrator.run(direction, data_migrations_path, migration[:version])
-        else
-          ::ActiveRecord::Migration.write("== %s %s" % ['Schema', "=" * 69])
-          ::DataMigrate::SchemaMigration.run(
-            direction,
-            ::DataMigrate::SchemaMigration.migrations_paths,
-            migration[:version]
-          )
+    def db_configs_with_versions
+      db_configs_with_versions = Hash.new { |h, k| h[k] = [] }
+
+      with_temporary_connection_for_each do |conn|
+        db_config = conn.pool.db_config
+        if db_config.primary?
+          versions_to_run = DataMigrate::DatabaseTasks.pending_data_migrations.map { |m| m[:version] }
+          target_version = ActiveRecord::Tasks::DatabaseTasks.target_version
+
+          versions_to_run.each do |version|
+            next if target_version && target_version != version
+            db_configs_with_versions[version] << DatabaseConfigurationWrapper.new(db_config)
+          end
         end
       end
 
-      def schema_dump_path(db_config, format = ActiveRecord.schema_format)
-        return ENV["DATA_SCHEMA"] if ENV["DATA_SCHEMA"]
+      db_configs_with_versions
+    end
 
-        # We only require a schema.rb file for the primary database
-        return unless db_config.primary?
+    def schema_file(_format = nil)
+      File.join(db_dir, "data_schema.rb")
+    end
 
-        File.join(File.dirname(ActiveRecord::Tasks::DatabaseTasks.schema_dump_path(db_config, format)), schema_file_type)
+    def schema_file_type(_format = nil)
+      "data_schema.rb"
+    end
+
+    # This method is removed in Rails 7.0
+    def dump_filename(spec_name, format = ActiveRecord::Base.schema_format)
+      filename = if spec_name == "primary"
+        schema_file_type(format)
+      else
+        "#{spec_name}_#{schema_file_type(format)}"
       end
 
-      # Override this method from `ActiveRecord::Tasks::DatabaseTasks`
-      # to ensure that the sha saved in ar_internal_metadata table
-      # is from the original schema.rb file
-      def schema_sha1(file)
-        ActiveRecord::Tasks::DatabaseTasks.schema_dump_path(ActiveRecord::Base.configurations.configs_for(env_name: ActiveRecord::Tasks::DatabaseTasks.env, name: "primary"))
+      ENV["DATA_SCHEMA"] || File.join(db_dir, filename)
+    end
+
+    def check_schema_file(filename)
+      unless File.exist?(filename)
+        message = +%{#{filename} doesn't exist yet. Run `rake data:migrate` to create it, then try again.}
+        Kernel.abort message
       end
     end
 
-    def self.forward(step = 1)
+    def pending_migrations
+      sort_migrations(
+        pending_schema_migrations,
+        pending_data_migrations
+      )
+    end
+
+    def sort_migrations(*migrations)
+      migrations.flatten.sort { |a, b|  sort_string(a) <=> sort_string(b) }
+    end
+
+    def sort_string migration
+      "#{migration[:version]}_#{migration[:kind] == :data ? 1 : 0}"
+    end
+
+    def data_migrations_path
+      ::DataMigrate.config.data_migrations_path
+    end
+
+    def run_migration(migration, direction)
+      if migration[:kind] == :data
+        ::ActiveRecord::Migration.write("== %s %s" % ['Data', "=" * 71])
+        ::DataMigrate::DataMigrator.run(direction, data_migrations_path, migration[:version])
+      else
+        ::ActiveRecord::Migration.write("== %s %s" % ['Schema', "=" * 69])
+        ::DataMigrate::SchemaMigration.run(
+          direction,
+          ::DataMigrate::SchemaMigration.migrations_paths,
+          migration[:version]
+        )
+      end
+    end
+
+    def schema_dump_path(db_config, format = ActiveRecord.schema_format)
+      return ENV["DATA_SCHEMA"] if ENV["DATA_SCHEMA"]
+
+      # We only require a schema.rb file for the primary database
+      return unless db_config.primary?
+
+      File.join(File.dirname(ActiveRecord::Tasks::DatabaseTasks.schema_dump_path(db_config, format)), schema_file_type)
+    end
+
+    # Override this method from `ActiveRecord::Tasks::DatabaseTasks`
+    # to ensure that the sha saved in ar_internal_metadata table
+    # is from the original schema.rb file
+    def schema_sha1(file)
+      ActiveRecord::Tasks::DatabaseTasks.schema_dump_path(ActiveRecord::Base.configurations.configs_for(env_name: ActiveRecord::Tasks::DatabaseTasks.env, name: "primary"))
+    end
+
+    def forward(step = 1)
       DataMigrate::DataMigrator.create_data_schema_table
       migrations = pending_migrations.reverse.pop(step).reverse
       migrations.each do | pending_migration |
@@ -99,7 +154,7 @@ module DataMigrate
       end
     end
 
-    def self.pending_data_migrations
+    def pending_data_migrations
       data_migrations = DataMigrate::DataMigrator.migrations(data_migrations_path)
       data_migrator = DataMigrate::RailsHelper.data_migrator(:up, data_migrations)
       sort_migrations(
@@ -107,11 +162,11 @@ module DataMigrate
         )
     end
 
-    def self.pending_schema_migrations
+    def pending_schema_migrations
       ::DataMigrate::SchemaMigration.pending_schema_migrations
     end
 
-    def self.past_migrations(sort = nil)
+    def past_migrations(sort = nil)
       data_versions = DataMigrate::RailsHelper.data_schema_migration.table_exists? ? DataMigrate::RailsHelper.data_schema_migration.normalized_versions : []
       schema_versions = DataMigrate::RailsHelper.schema_migration.normalized_versions
       migrations = data_versions.map { |v| { version: v.to_i, kind: :data } } + schema_versions.map { |v| { version: v.to_i, kind: :schema } }

data/lib/data_migrate/test.rb

@@ -0,0 +1,14 @@
+module Base
+  extend self
+
+  def foo
+    puts "Base#foo called"
+  end
+end
+
+module Child
+  extend Base
+  extend self
+
+  puts "foo: #{respond_to?(:foo)}"
+end