data_migrate 9.2.0 → 9.4.2

data/lib/data_migrate/database_tasks.rb

@@ -5,87 +5,142 @@ require "data_migrate/config"
 module DataMigrate
   ##
   # This class extends DatabaseTasks to add a schema_file method.
-  class DatabaseTasks
+  module DatabaseTasks
     extend ActiveRecord::Tasks::DatabaseTasks
-
-    class << self
-      def schema_file(_format = nil)
-        File.join(db_dir, "data_schema.rb")
-      end
-
-      def schema_file_type(_format = nil)
-        "data_schema.rb"
-      end
-
-      # This method is removed in Rails 7.0
-      def dump_filename(spec_name, format = ActiveRecord::Base.schema_format)
-        filename = if spec_name == "primary"
-          schema_file_type(format)
+    extend self
+
+    # These method are only introduced in Rails 7.1
+    unless respond_to?(:with_temporary_connection_for_each)
+      def with_temporary_connection_for_each(env: ActiveRecord::Tasks::DatabaseTasks.env, name: nil, &block) # :nodoc:
+        if name
+          db_config = ActiveRecord::Base.configurations.configs_for(env_name: env, name: name)
+          with_temporary_connection(db_config, &block)
         else
-          "#{spec_name}_#{schema_file_type(format)}"
+          ActiveRecord::Base.configurations.configs_for(env_name: env, name: name).each do |db_config|
+            with_temporary_connection(db_config, &block)
+          end
         end
-
-        ENV["DATA_SCHEMA"] || File.join(db_dir, filename)
       end
 
-      def check_schema_file(filename)
-        unless File.exist?(filename)
-          message = +%{#{filename} doesn't exist yet. Run `rake data:migrate` to create it, then try again.}
-          Kernel.abort message
+      def with_temporary_connection(db_config) # :nodoc:
+        with_temporary_pool(db_config) do |pool|
+          yield pool.connection
         end
       end
 
-      def pending_migrations
-        sort_migrations(
-          pending_schema_migrations,
-          pending_data_migrations
-        )
+      def migration_class # :nodoc:
+        ActiveRecord::Base
       end
 
-      def sort_migrations(*migrations)
-        migrations.flatten.sort { |a, b|  sort_string(a) <=> sort_string(b) }
+      def migration_connection # :nodoc:
+        migration_class.connection
       end
 
-      def sort_string migration
-        "#{migration[:version]}_#{migration[:kind] == :data ? 1 : 0}"
-      end
+      private def with_temporary_pool(db_config)
+        original_db_config = migration_class.connection_db_config
+        pool = migration_class.connection_handler.establish_connection(db_config)
 
-      def data_migrations_path
-        ::DataMigrate.config.data_migrations_path
+        yield pool
+      ensure
+        migration_class.connection_handler.establish_connection(original_db_config)
       end
+    end
 
-      def run_migration(migration, direction)
-        if migration[:kind] == :data
-          ::ActiveRecord::Migration.write("== %s %s" % ['Data', "=" * 71])
-          ::DataMigrate::DataMigrator.run(direction, data_migrations_path, migration[:version])
-        else
-          ::ActiveRecord::Migration.write("== %s %s" % ['Schema', "=" * 69])
-          ::DataMigrate::SchemaMigration.run(
-            direction,
-            ::DataMigrate::SchemaMigration.migrations_paths,
-            migration[:version]
-          )
+    def db_configs_with_versions
+      db_configs_with_versions = Hash.new { |h, k| h[k] = [] }
+
+      with_temporary_connection_for_each do |conn|
+        db_config = conn.pool.db_config
+        if db_config.primary?
+          versions_to_run = DataMigrate::DatabaseTasks.pending_data_migrations.map { |m| m[:version] }
+          target_version = ActiveRecord::Tasks::DatabaseTasks.target_version
+
+          versions_to_run.each do |version|
+            next if target_version && target_version != version
+            db_configs_with_versions[version] << DatabaseConfigurationWrapper.new(db_config)
+          end
         end
       end
 
-      def schema_dump_path(db_config, format = ActiveRecord.schema_format)
-        return ENV["DATA_SCHEMA"] if ENV["DATA_SCHEMA"]
+      db_configs_with_versions
+    end
 
-        # We only require a schema.rb file for the primary database
-        return unless db_config.primary?
+    def schema_file(_format = nil)
+      File.join(db_dir, "data_schema.rb")
+    end
 
-        File.join(File.dirname(ActiveRecord::Tasks::DatabaseTasks.schema_dump_path(db_config, format)), schema_file_type)
+    def schema_file_type(_format = nil)
+      "data_schema.rb"
+    end
+
+    # This method is removed in Rails 7.0
+    def dump_filename(spec_name, format = ActiveRecord::Base.schema_format)
+      filename = if spec_name == "primary"
+        schema_file_type(format)
+      else
+        "#{spec_name}_#{schema_file_type(format)}"
       end
 
-      # Override this method from `ActiveRecord::Tasks::DatabaseTasks`
-      # to ensure that the sha saved in ar_internal_metadata table
-      # is from the original schema.rb file
-      def schema_sha1(file)
-        ActiveRecord::Tasks::DatabaseTasks.schema_dump_path(ActiveRecord::Base.configurations.configs_for(env_name: ActiveRecord::Tasks::DatabaseTasks.env, name: "primary"))
+      ENV["DATA_SCHEMA"] || File.join(db_dir, filename)
+    end
+
+    def check_schema_file(filename)
+      unless File.exist?(filename)
+        message = +%{#{filename} doesn't exist yet. Run `rake data:migrate` to create it, then try again.}
+        Kernel.abort message
       end
     end
 
-    def self.forward(step = 1)
+    def pending_migrations
+      sort_migrations(
+        pending_schema_migrations,
+        pending_data_migrations
+      )
+    end
+
+    def sort_migrations(*migrations)
+      migrations.flatten.sort { |a, b|  sort_string(a) <=> sort_string(b) }
+    end
+
+    def sort_string migration
+      "#{migration[:version]}_#{migration[:kind] == :data ? 1 : 0}"
+    end
+
+    def data_migrations_path
+      ::DataMigrate.config.data_migrations_path
+    end
+
+    def run_migration(migration, direction)
+      if migration[:kind] == :data
+        ::ActiveRecord::Migration.write("== %s %s" % ['Data', "=" * 71])
+        ::DataMigrate::DataMigrator.run(direction, data_migrations_path, migration[:version])
+      else
+        ::ActiveRecord::Migration.write("== %s %s" % ['Schema', "=" * 69])
+        ::DataMigrate::SchemaMigration.run(
+          direction,
+          ::DataMigrate::SchemaMigration.migrations_paths,
+          migration[:version]
+        )
+      end
+    end
+
+    def schema_dump_path(db_config, format = ActiveRecord.schema_format)
+      return ENV["DATA_SCHEMA"] if ENV["DATA_SCHEMA"]
+
+      # We only require a schema.rb file for the primary database
+      return unless db_config.primary?
+
+      File.join(File.dirname(ActiveRecord::Tasks::DatabaseTasks.schema_dump_path(db_config, format)), schema_file_type)
+    end
+
+    # Override this method from `ActiveRecord::Tasks::DatabaseTasks`
+    # to ensure that the sha saved in ar_internal_metadata table
+    # is from the original schema.rb file
+    def schema_sha1(file)
+      ActiveRecord::Tasks::DatabaseTasks.schema_dump_path(ActiveRecord::Base.configurations.configs_for(env_name: ActiveRecord::Tasks::DatabaseTasks.env, name: "primary"))
+    end
+
+    def forward(step = 1)
       DataMigrate::DataMigrator.create_data_schema_table
       migrations = pending_migrations.reverse.pop(step).reverse
       migrations.each do | pending_migration |
@@ -99,7 +154,7 @@ module DataMigrate
       end
     end
 
-    def self.pending_data_migrations
+    def pending_data_migrations
       data_migrations = DataMigrate::DataMigrator.migrations(data_migrations_path)
       data_migrator = DataMigrate::RailsHelper.data_migrator(:up, data_migrations)
       sort_migrations(
@@ -107,16 +162,84 @@ module DataMigrate
         )
     end
 
-    def self.pending_schema_migrations
+    def pending_schema_migrations
       ::DataMigrate::SchemaMigration.pending_schema_migrations
     end
 
-    def self.past_migrations(sort = nil)
+    def past_migrations(sort = nil)
       data_versions = DataMigrate::RailsHelper.data_schema_migration.table_exists? ? DataMigrate::RailsHelper.data_schema_migration.normalized_versions : []
       schema_versions = DataMigrate::RailsHelper.schema_migration.normalized_versions
       migrations = data_versions.map { |v| { version: v.to_i, kind: :data } } + schema_versions.map { |v| { version: v.to_i, kind: :schema } }
 
       sort&.downcase == "asc" ? sort_migrations(migrations) : sort_migrations(migrations).reverse
     end
+
+    def self.migrate_with_data
+      DataMigrate::DataMigrator.create_data_schema_table
+
+      ActiveRecord::Migration.verbose = ENV["VERBOSE"] ? ENV["VERBOSE"] == "true" : true
+
+      db_configs = ActiveRecord::Base.configurations.configs_for(env_name: ActiveRecord::Tasks::DatabaseTasks.env)
+
+      schema_mapped_versions = ActiveRecord::Tasks::DatabaseTasks.db_configs_with_versions(db_configs)
+      data_mapped_versions = DataMigrate::DatabaseTasks.db_configs_with_versions
+
+      mapped_versions = schema_mapped_versions.merge(data_mapped_versions) do |_key, schema_db_configs, data_db_configs|
+        schema_db_configs + data_db_configs
+      end
+
+      mapped_versions.sort.each do |version, db_configs|
+        db_configs.each do |db_config|
+          if is_data_migration = db_config.is_a?(DataMigrate::DatabaseConfigurationWrapper)
+            db_config = db_config.db_config
+          end
+
+          DataMigrate::DatabaseTasks.with_temporary_connection(db_config) do
+            if is_data_migration
+              DataMigrate::DataMigrator.run(:up, DataMigrate::DatabaseTasks.data_migrations_path, version)
+            else
+              ActiveRecord::Tasks::DatabaseTasks.migrate(version)
+            end
+          end
+        end
+      end
+    end
+
+    def self.prepare_all_with_data
+      seed = false
+
+      each_current_configuration(env) do |db_config|
+        next unless db_config.primary?
+
+        with_temporary_pool(db_config) do
+          begin
+            database_initialized = migration_connection.schema_migration.table_exists?
+          rescue ActiveRecord::NoDatabaseError
+            create(db_config)
+            retry
+          end
+
+          unless database_initialized
+            if File.exist?(schema_dump_path(db_config))
+              load_schema(db_config, ActiveRecord.schema_format, nil)
+              load_schema_current(
+                :ruby,
+                ENV["DATA_SCHEMA"]
+              )
+            end
+
+            seed = true
+          end
+
+          migrate_with_data
+          if ActiveRecord.dump_schema_after_migration
+            dump_schema(db_config)
+            DataMigrate::Tasks::DataMigrateTasks.dump
+          end
+        end
+      end
+
+      load_seed if seed
+    end
   end
 end

data/lib/data_migrate/test.rb

@@ -0,0 +1,14 @@
+module Base
+  extend self
+
+  def foo
+    puts "Base#foo called"
+  end
+end
+
+module Child
+  extend Base
+  extend self
+
+  puts "foo: #{respond_to?(:foo)}"
+end

data/lib/data_migrate/version.rb

@@ -1,3 +1,3 @@
 module DataMigrate
-  VERSION = "9.2.0".freeze
+  VERSION = "9.4.2".freeze
 end

data/lib/data_migrate.rb

@@ -12,6 +12,7 @@ require File.join(File.dirname(__FILE__), "data_migrate", "railtie")
 require File.join(File.dirname(__FILE__), "data_migrate", "tasks/data_migrate_tasks")
 require File.join(File.dirname(__FILE__), "data_migrate", "config")
 require File.join(File.dirname(__FILE__), "data_migrate", "schema_migration")
+require File.join(File.dirname(__FILE__), "data_migrate", "database_configurations_wrapper")
 
 module DataMigrate
   def self.root

data/tasks/databases.rake

@@ -5,52 +5,8 @@ require 'data_migrate/tasks/data_migrate_tasks'
 namespace :db do
   namespace :migrate do
     desc "Migrate the database data and schema (options: VERSION=x, VERBOSE=false)."
-    task :with_data => :environment do
-      DataMigrate::DataMigrator.create_data_schema_table
-
-      ActiveRecord::Migration.verbose = ENV["VERBOSE"] ? ENV["VERBOSE"] == "true" : true
-      target_version = ENV["VERSION"] ? ENV["VERSION"].to_i : nil
-      migrations = []
-
-      if target_version.nil?
-        migrations = DataMigrate::DatabaseTasks.pending_migrations.map{ |m| m.merge(:direction =>:up) }
-      else
-        current_schema_version = ActiveRecord::Migrator.current_version
-        schema_migrations = if target_version > current_schema_version
-                              DataMigrate::DatabaseTasks.pending_schema_migrations.keep_if{ |m| m[:version] <= target_version }.map{ |m| m.merge(:direction =>:up) }
-                            elsif target_version < current_schema_version
-                              DataMigrate::DatabaseTasks.past_migrations.keep_if{ |m| m[:version] > target_version }.map{ |m| m.merge(:direction =>:down) }
-                            else # ==
-                              []
-                            end
-
-        current_data_version = DataMigrate::DataMigrator.current_version
-        data_migrations = if target_version > current_data_version
-                            DataMigrate::DatabaseTasks.pending_data_migrations.keep_if{ |m| m[:version] <= target_version }.map{ |m| m.merge(:direction =>:up) }
-                          elsif target_version < current_data_version
-                            DataMigrate::DatabaseTasks.past_migrations.keep_if{ |m| m[:version] > target_version }.map{ |m| m.merge(:direction =>:down) }
-                          else # ==
-                            []
-                          end
-        migrations = if schema_migrations.empty?
-                       data_migrations
-                     elsif data_migrations.empty?
-                       schema_migrations
-                     elsif target_version > current_data_version && target_version > current_schema_version
-                       DataMigrate::DatabaseTasks.sort_migrations data_migrations, schema_migrations
-                     elsif target_version < current_data_version && target_version < current_schema_version
-                       DataMigrate::DatabaseTasks.sort_migrations(data_migrations, schema_migrations).reverse
-                     elsif target_version > current_data_version && target_version < current_schema_version
-                       schema_migrations + data_migrations
-                     elsif target_version < current_data_version && target_version > current_schema_version
-                       schema_migrations + data_migrations
-                     end
-      end
-
-      migrations.each do |migration|
-        DataMigrate::DatabaseTasks.run_migration(migration, migration[:direction])
-      end
-
+    task :with_data => :load_config do
+      DataMigrate::DatabaseTasks.migrate_with_data
       Rake::Task["db:_dump"].invoke
       Rake::Task["data:dump"].invoke
     end
@@ -190,6 +146,13 @@ namespace :db do
       end
     end
   end
+
+  namespace :prepare do
+    desc "Runs setup if database does not exist, or runs data and schema migrations if it does"
+    task with_data: :environment do
+      DataMigrate::DatabaseTasks.prepare_all_with_data
+    end
+  end
 end
 
 namespace :data do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: data_migrate
 version: !ruby/object:Gem::Version
-  version: 9.2.0
+  version: 9.4.2
 platform: ruby
 authors:
 - Andrew J Vargo
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-10-05 00:00:00.000000000 Z
+date: 2024-08-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -128,16 +128,16 @@ dependencies:
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.6
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.6
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: timecop
   requirement: !ruby/object:Gem::Requirement
@@ -202,7 +202,6 @@ files:
 - Changelog.md
 - Gemfile
 - Gemfile.lock
-- Gemfile.rails6.1
 - LICENSE
 - README.md
 - Rakefile
@@ -220,6 +219,7 @@ files:
 - lib/data_migrate/data_migrator.rb
 - lib/data_migrate/data_schema.rb
 - lib/data_migrate/data_schema_migration.rb
+- lib/data_migrate/database_configurations_wrapper.rb
 - lib/data_migrate/database_tasks.rb
 - lib/data_migrate/migration_context.rb
 - lib/data_migrate/rails_helper.rb
@@ -228,6 +228,7 @@ files:
 - lib/data_migrate/schema_migration.rb
 - lib/data_migrate/status_service.rb
 - lib/data_migrate/tasks/data_migrate_tasks.rb
+- lib/data_migrate/test.rb
 - lib/data_migrate/version.rb
 - lib/generators/data_migrate.rb
 - lib/generators/data_migration/data_migration_generator.rb
@@ -277,7 +278,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
+rubygems_version: 3.4.19
 signing_key: 
 specification_version: 4
 summary: Rake tasks to migrate data alongside schema changes.

data/Gemfile.rails6.1

@@ -1,11 +0,0 @@
-source "http://rubygems.org"
-
-# Specify your gem's dependencies in data_migrate.gemspec
-%w[
-  activerecord
-  railties
-].each do |rails_gem|
-  gem rails_gem, '~> 6.1.0'
-end
-gem 'sqlite3', "~> 1.4"
-gemspec