data_migrate 9.4.0 → 10.0.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8b1c79f9a3a6b30aeb19f85dd46e0c7d29885f3552787924f27eeb80f1738e95
-  data.tar.gz: 56ff3a64d6cde029b8a6e9f57c82ebf15ad86aa0140520d5a08c67a8a30e9869
+  metadata.gz: d212f0e6fa0c9b767cbdf17854c7c52d640a9c8611d0b10ed6c0460de68985cd
+  data.tar.gz: ac338ef447f1942d64eb5cf6c0fe520f71089968c04fb03fb829a1318534b4eb
 SHA512:
-  metadata.gz: 6b8d1bb7fd343e44f0cdf623fda3e52875e4efee1327bc4fbf2e3700b1071a96e3b50bb88a63ef4f94c98fcdb068c81f47a4a94d3ee80e91f4d9b47d84893234
-  data.tar.gz: bbbcc92d53620c487cdce34a99dd5c41ddb156fa6a8a1a1b8304f1750af2c696df2c1f12079d9ec320b7f1a13497ab35ea20d19a0e1c5f33b3e7a7b2804a8ca0
+  metadata.gz: 01ae940433e4cd51a2f421297fb3ffb72372503971ee906164129c035c89aedb6c18d5003cd083930c24245f16728c38c93aed94fb8d9b2f7d0288895362a3d4
+  data.tar.gz: 9d94ff40ed274256247960c0dd1325645b638ac2198d29b3bce2c3c11f842b4b0a2b381bd73fcb17af308ab366cdca16c7f99c9e2fa778da7ab6103352e50ba1

data/.github/workflows/build.yml

@@ -11,14 +11,14 @@ jobs:
       matrix:
         os: [ "ubuntu-20.04" ]
         ruby:
+          - '2.7'
           - '3.0'
           - '3.1'
           - '3.2'
-          - '3.3'
         gemfile:
+          - gemfiles/rails_6.0.gemfile
           - gemfiles/rails_6.1.gemfile
           - gemfiles/rails_7.0.gemfile
-          - gemfiles/rails_7.1.gemfile
     runs-on: ubuntu-latest
     env:
       BUNDLE_GEMFILE: ${{ matrix.gemfile }}
@@ -32,4 +32,4 @@ jobs:
           ruby-version: ${{ matrix.ruby }}
           bundler-cache: true
       - name: Run tests
-        run: bundle exec rspec
+        run: bundle exec rspec

data/.github/workflows/gempush.yml

@@ -1,8 +1,10 @@
-name: Release Gem
+name: Ruby Gem
 
 on:
-  release:
-    types: [released]
+  push:
+    branches:
+      - main
+      - chaunce-multiple_connection_support
 
 jobs:
   build:
@@ -11,10 +13,10 @@ jobs:
 
     steps:
       - uses: actions/checkout@master
-      - name: Set up Ruby 3.2
-        uses: ruby/setup-ruby@v1
+      - name: Set up Ruby 2.6
+        uses: actions/setup-ruby@v1
         with:
-          ruby-version: '3.2'
+          version: 2.6.x
 
       - name: Publish to RubyGems
         run: |

data/.gitignore

@@ -1,12 +1,11 @@
 /.rvmrc
 *.gem
+*.lock
 gemfiles/.bundle
 spec/db/test.db
 spec/db/other_test.db
 spec/db/data_schema.rb
 .vscode/
 .DS_Store
-.ruby-gemset
-.ruby-version
 .idea/
-vendor/
+vendor/

data/.ruby-version

@@ -0,0 +1 @@
+2.7.5

data/.travis.yml

@@ -0,0 +1,14 @@
+language: ruby
+rvm:
+  - 2.7
+  - 3.0
+  - 3.2
+script: bundle exec rspec
+gemfile:
+  - gemfiles/rails_6.0.gemfile
+  - gemfiles/rails_6.1.gemfile
+  - gemfiles/rails_7.0.gemfile
+jobs:
+  exclude:
+    - rvm: 3.0
+    - rvm: 3.2

data/Appraisals

@@ -1,11 +1,11 @@
+appraise 'rails-6.0' do
+  gem 'rails', '~> 6.0.0'
+end
+
 appraise 'rails-6.1' do
   gem 'rails', '~> 6.1.0'
 end
 
 appraise 'rails-7.0' do
-  gem 'rails', '~> 7.0.0'
-end
-
-appraise 'rails-7.1' do
-  gem 'rails', '7.1.0'
+  gem 'rails', '~> 7.0'
 end

data/Changelog.md

@@ -1,51 +1,13 @@
 # Changelog
 
-## 9.4.0
-- Reset model schema cache before each data migration https://github.com/ilyakatz/data-migrate/pull/307
-- Run load_config rake task before db:migrate:with_data https://github.com/ilyakatz/data-migrate/pull/308
-
-## 9.3.0
-- Improve with_data Rake task for multiple database https://github.com/ilyakatz/data-migrate/pull/296
-
-## 9.2.0
-- Support Rails 7.1 https://github.com/ilyakatz/data-migrate/pull/278
-- Build and test against 7.1.0.rc1 https://github.com/ilyakatz/data-migrate/pull/286
-
-## 9.1.0
-
-- Fix a bug that caused `schema_sha1` in `ar_internal_metadata` to be reset to the `data_schema.rb` file. (#272)
-- Remove the need for empty data_schema files for non-primary databases. (#273)
-
-## [YANKED] 10.0.3.rc
-
-- Remove all travis references [leoarnold](https//:github.com/leoarnold)
-- Changing to rc because of ongoing discussion how to properly handle multiple database environments
-
-## [YANKED] 10.0.2
-
-Change "rails" dependencies to "railties"
-
-## [YANKED] 10.0.1
-
-- Bug fix for Rails 6 config [chaunce](https//:github.com/chaunce)
-- Railties bug fix by [opti](https://github.com/opti)
-
-## [YANKED] 10.0.0
-
-Releasing 10.0.0
-
-!!! Breaking changes !!!
-
-- This version introduces a breaking change which may lead to undesired
-behavior in multi-database environments. See https://github.com/ilyakatz/data-migrate/issues/181
-
-## [YANKED] 10.0.0.rc1
+## 10.0.0.rc1
 
 - Changes by [chaunce](https//:github.com/chaunce)
 - Multiple databases support
 - Refactor to clean things up
 - Deprecate rails 5.2 support for real
 
+
 ## 9.0.0
 
 Ruby 3.2 support [mehanoid](https://github.com/mehanoid)

data/Gemfile

@@ -1,3 +1,4 @@
 source "http://rubygems.org"
 
+gem 'sqlite3', "~> 1.4"
 gemspec

data/Gemfile.rails6.1

@@ -0,0 +1,11 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in data_migrate.gemspec
+%w[
+  activerecord
+  railties
+].each do |rails_gem|
+  gem rails_gem, '~> 6.1.0'
+end
+gem 'sqlite3', "~> 1.4"
+gemspec

data/README.md

@@ -2,7 +2,7 @@
 
 - [![Version](http://img.shields.io/gem/v/data_migrate.svg?style=flat-square)](https://rubygems.org/gems/data_migrate)
 - [![License](http://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat-square)](http://opensource.org/licenses/MIT)
-- ![.github/workflows/build.yml](https://github.com/ilyakatz/data-migrate/actions/workflows/.github/workflows/build.yml/badge.svg)
+- [![Travis](https://img.shields.io/travis/ilyakatz/data-migrate.svg)](https://travis-ci.org/ilyakatz/data-migrate)
 - [![Reviewed by Hound](https://img.shields.io/badge/Reviewed_by-Hound-8E64B0.svg)](https://houndci.com)
 
 Run data migrations alongside schema migrations.
@@ -12,7 +12,7 @@ migrations, except they should be reserved for data migrations. For
 instance, if you realize you need to titleize all your titles, this
 is the place to do it.
 
-![directory tree example](./screenshot.png)
+![Travis](./screenshot.png)
 
 ## Why should I use this?
 
@@ -36,7 +36,7 @@ table to track all migrations.
 
 ## Rails Support
 
-Support Rails 6.1 through 7.1
+Support Rails 6.0 through 7.0
 
 
 #### v1
@@ -96,9 +96,7 @@ Tasks work as they would with the 'vanilla' db version. The 'with_data' addition
 
 With 'up' and 'down', you can specify the option 'BOTH', which defaults to false. Using true, will migrate both the data and schema (in the desired direction) if they both match the version provided. Again, going up, schema is given precedence. Down its data.
 
-When using `rake db:migrate:with_data` migrations will be run in ascending order by their version. For example, if you have a data migration with version 20230410000000 and a schema migration with version 20230415000000, expect the data migration to run first.
-
-`rake db:migrate:status:with_data` provides an additional column to indicate which type of migration.
+`rake db:migrate:status:with_data` provides and additional column to indicate which type of migration.
 
 ### Configuration
 
@@ -164,9 +162,9 @@ Run tests for a specific version of Rails
 
 ```
 bundle exec appraisal install
+bundle exec appraisal rails-6.0 rspec
 bundle exec appraisal rails-6.1 rspec
 bundle exec appraisal rails-7.0 rspec
-bundle exec appraisal rails-7.1 rspec
 ```
 
 ## Thanks

data/data_migrate.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |s|
     activerecord
     railties
   ].each do |rails_gem|
-    s.add_dependency(rails_gem, '>= 6.1')
+    s.add_dependency(rails_gem, '>= 6.0')
   end
 
   s.add_development_dependency "appraisal"
@@ -28,7 +28,7 @@ Gem::Specification.new do |s|
   s.add_development_dependency "rspec-core"
   s.add_development_dependency "pry"
   s.add_development_dependency "rb-readline"
-  s.add_development_dependency "sqlite3"
+  s.add_development_dependency "sqlite3", "~> 1.3.6"
   s.add_development_dependency "timecop"
   s.add_development_dependency "rubocop"
   s.add_development_dependency "overcommit"

data/gemfiles/{rails_7.1.gemfile → rails_6.0.gemfile}

@@ -2,6 +2,7 @@
 
 source "http://rubygems.org"
 
-gem "rails", "7.1.0"
+gem "rails", "~> 6.0.0"
+gem "sqlite3", "~> 1.4"
 
 gemspec path: "../"

data/gemfiles/rails_6.1.gemfile

@@ -3,5 +3,6 @@
 source "http://rubygems.org"
 
 gem "rails", "~> 6.1.0"
+gem "sqlite3", "~> 1.4"
 
 gemspec path: "../"

data/gemfiles/rails_7.0.gemfile

@@ -2,6 +2,7 @@
 
 source "http://rubygems.org"
 
-gem "rails", "~> 7.0.0"
+gem "sqlite3", "~> 1.4"
+gem "rails", "~> 7.0"
 
 gemspec path: "../"

data/lib/data_migrate/data_migrator.rb

@@ -5,20 +5,32 @@ require "data_migrate/config"
 
 module DataMigrate
   class DataMigrator < ActiveRecord::Migrator
+    def self.migrations_paths
+      [DataMigrate.config.data_migrations_path]
+    end
+
+    def self.assure_data_schema_table
+      DataMigrate::DataSchemaMigration.create_table
+    end
+
+    def initialize(direction, migrations, target_version = nil)
+      @direction         = direction
+      @target_version    = target_version
+      @migrated_versions = nil
+      @migrations        = migrations
+
+      validate(@migrations)
+
+      DataMigrate::DataSchemaMigration.create_table
+      ActiveRecord::InternalMetadata.create_table
+    end
+
     def load_migrated
       @migrated_versions =
-        DataMigrate::RailsHelper.data_schema_migration.normalized_versions.map(&:to_i).sort
+        DataMigrate::DataSchemaMigration.normalized_versions.map(&:to_i).sort
     end
 
     class << self
-      def migrations_paths
-        [DataMigrate.config.data_migrations_path]
-      end
-
-      def create_data_schema_table
-        DataMigrate::RailsHelper.data_schema_migration.create_table
-      end
-
       def current_version
         DataMigrate::MigrationContext.new(migrations_paths).current_version
       end
@@ -54,10 +66,6 @@ module DataMigrate
 
       #TODO: this was added to be backward compatible, need to re-evaluate
       def run(direction, migration_paths, version)
-        # Ensure all Active Record model cache is reset for each data migration
-        # As recommended in: https://github.com/rails/rails/blob/da21c2e9812e5eb0698fba4a9aa38632fc004432/activerecord/lib/active_record/migration.rb#L467-L470
-        ActiveRecord::Base.descendants.each(&:reset_column_information)
-
         DataMigrate::MigrationContext.new(migration_paths).run(direction, version)
       end
 
@@ -71,10 +79,10 @@ module DataMigrate
     def record_version_state_after_migrating(version)
       if down?
         migrated.delete(version)
-        DataMigrate::RailsHelper.data_schema_delete_version(version.to_s)
+        DataMigrate::DataSchemaMigration.where(version: version.to_s).delete_all
       else
         migrated << version
-        DataMigrate::RailsHelper.data_schema_migration.create_version(version.to_s)
+        DataMigrate::DataSchemaMigration.create!(version: version.to_s)
       end
     end
   end

data/lib/data_migrate/data_schema.rb

@@ -9,7 +9,7 @@ module DataMigrate
     #   ActiveRecord::ConnectionAdapters::SchemaStatements
     #     #assume_migrated_upto_version
     def define(info)
-      DataMigrate::DataMigrator.create_data_schema_table
+      DataMigrate::DataMigrator.assure_data_schema_table
 
       return if info[:version].blank?
 
@@ -57,7 +57,7 @@ module DataMigrate
     end
 
     def table_name
-      DataMigrate::RailsHelper.data_schema_migration.table_name
+      DataMigrate::DataSchemaMigration.table_name
     end
   end
 end

data/lib/data_migrate/data_schema_migration.rb

@@ -1,29 +1,12 @@
 module DataMigrate
-  class DataSchemaMigration < ActiveRecord::SchemaMigration
-    # In Rails 7.1+, ActiveRecord::SchemaMigration methods are instance methods
-    # So we only load the appropriate methods depending on Rails version.
-    if DataMigrate::RailsHelper.rails_version_equal_to_or_higher_than_7_1
-      def table_name
-        ActiveRecord::Base.table_name_prefix + 'data_migrations' + ActiveRecord::Base.table_name_suffix
-      end
-
-      def primary_key
-        "version"
-      end
-    else
-      class << self
-        def table_name
-          ActiveRecord::Base.table_name_prefix + 'data_migrations' + ActiveRecord::Base.table_name_suffix
-        end
-
-        def primary_key
-          "version"
-        end
+  class DataSchemaMigration
+    class << self
+      delegate :table_name, :primary_key, :create_table, :normalized_versions, :create, :create!, :table_exists?, :exists?, :where, to: :instance
 
-        def create_version(version)
-          # Note that SchemaMigration.create_version in Rails 7.1 does not
-          # raise an error if validations fail but we retain this behaviour for now.
-          create!(version: version)
+      def instance
+        @instance ||= Class.new(::ActiveRecord::SchemaMigration) do
+          define_singleton_method(:table_name) { ActiveRecord::Base.table_name_prefix + 'data_migrations' + ActiveRecord::Base.table_name_suffix }
+          define_singleton_method(:primary_key) { "version" }
         end
       end
     end

data/lib/data_migrate/database_tasks.rb

@@ -5,143 +5,67 @@ require "data_migrate/config"
 module DataMigrate
   ##
   # This class extends DatabaseTasks to add a schema_file method.
-  module DatabaseTasks
+  class DatabaseTasks
     extend ActiveRecord::Tasks::DatabaseTasks
-    extend self
-
-    # These method are only introduced in Rails 7.1
-    unless respond_to?(:with_temporary_connection_for_each)
-      def with_temporary_connection_for_each(env: ActiveRecord::Tasks::DatabaseTasks.env, name: nil, &block) # :nodoc:
-        if name
-          db_config = ActiveRecord::Base.configurations.configs_for(env_name: env, name: name)
-          with_temporary_connection(db_config, &block)
-        else
-          ActiveRecord::Base.configurations.configs_for(env_name: env, name: name).each do |db_config|
-            with_temporary_connection(db_config, &block)
-          end
-        end
+
+    class << self
+      def schema_file_type(_format = nil)
+        "data_schema.rb"
       end
 
-      def with_temporary_connection(db_config) # :nodoc:
-        with_temporary_pool(db_config) do |pool|
-          yield pool.connection
+      def dump_filename(spec_name, format = ActiveRecord::Base.schema_format)
+        filename = if spec_name == "primary"
+          schema_file_type(format)
+        else
+          "#{spec_name}_#{schema_file_type(format)}"
         end
-      end
 
-      def migration_class # :nodoc:
-        ActiveRecord::Base
+        ENV["DATA_SCHEMA"] || File.join(db_dir, filename)
       end
 
-      def migration_connection # :nodoc:
-        migration_class.connection
+      def check_schema_file(filename)
+        unless File.exist?(filename)
+          message = +%{#{filename} doesn't exist yet. Run `rake data:migrate` to create it, then try again.}
+          Kernel.abort message
+        end
       end
 
-      private def with_temporary_pool(db_config)
-        original_db_config = migration_class.connection_db_config
-        pool = migration_class.connection_handler.establish_connection(db_config)
-
-        yield pool
-      ensure
-        migration_class.connection_handler.establish_connection(original_db_config)
+      def pending_migrations
+        sort_migrations(
+          pending_schema_migrations,
+          pending_data_migrations
+        )
       end
-    end
-
-    def db_configs_with_versions
-      db_configs_with_versions = Hash.new { |h, k| h[k] = [] }
 
-      with_temporary_connection_for_each do |conn|
-        db_config = conn.pool.db_config
-        if db_config.primary?
-          versions_to_run = DataMigrate::DatabaseTasks.pending_data_migrations.map { |m| m[:version] }
-          target_version = ActiveRecord::Tasks::DatabaseTasks.target_version
-
-          versions_to_run.each do |version|
-            next if target_version && target_version != version
-            db_configs_with_versions[version] << DatabaseConfigurationWrapper.new(db_config)
-          end
-        end
+      def sort_migrations(*migrations)
+        migrations.flatten.sort { |a, b|  sort_string(a) <=> sort_string(b) }
       end
 
-      db_configs_with_versions
-    end
-
-    def schema_file(_format = nil)
-      File.join(db_dir, "data_schema.rb")
-    end
-
-    def schema_file_type(_format = nil)
-      "data_schema.rb"
-    end
-
-    # This method is removed in Rails 7.0
-    def dump_filename(spec_name, format = ActiveRecord::Base.schema_format)
-      filename = if spec_name == "primary"
-        schema_file_type(format)
-      else
-        "#{spec_name}_#{schema_file_type(format)}"
+      def sort_string migration
+        "#{migration[:version]}_#{migration[:kind] == :data ? 1 : 0}"
       end
 
-      ENV["DATA_SCHEMA"] || File.join(db_dir, filename)
-    end
-
-    def check_schema_file(filename)
-      unless File.exist?(filename)
-        message = +%{#{filename} doesn't exist yet. Run `rake data:migrate` to create it, then try again.}
-        Kernel.abort message
+      def data_migrations_path
+        ::DataMigrate.config.data_migrations_path
       end
-    end
-
-    def pending_migrations
-      sort_migrations(
-        pending_schema_migrations,
-        pending_data_migrations
-      )
-    end
-
-    def sort_migrations(*migrations)
-      migrations.flatten.sort { |a, b|  sort_string(a) <=> sort_string(b) }
-    end
 
-    def sort_string migration
-      "#{migration[:version]}_#{migration[:kind] == :data ? 1 : 0}"
-    end
-
-    def data_migrations_path
-      ::DataMigrate.config.data_migrations_path
-    end
-
-    def run_migration(migration, direction)
-      if migration[:kind] == :data
-        ::ActiveRecord::Migration.write("== %s %s" % ['Data', "=" * 71])
-        ::DataMigrate::DataMigrator.run(direction, data_migrations_path, migration[:version])
-      else
-        ::ActiveRecord::Migration.write("== %s %s" % ['Schema', "=" * 69])
-        ::DataMigrate::SchemaMigration.run(
-          direction,
-          ::DataMigrate::SchemaMigration.migrations_paths,
-          migration[:version]
-        )
+      def run_migration(migration, direction)
+        if migration[:kind] == :data
+          ::ActiveRecord::Migration.write("== %s %s" % ['Data', "=" * 71])
+          ::DataMigrate::DataMigrator.run(direction, data_migrations_path, migration[:version])
+        else
+          ::ActiveRecord::Migration.write("== %s %s" % ['Schema', "=" * 69])
+          ::DataMigrate::SchemaMigration.run(
+            direction,
+            ::DataMigrate::SchemaMigration.migrations_paths,
+            migration[:version]
+          )
+        end
       end
     end
 
-    def schema_dump_path(db_config, format = ActiveRecord.schema_format)
-      return ENV["DATA_SCHEMA"] if ENV["DATA_SCHEMA"]
-
-      # We only require a schema.rb file for the primary database
-      return unless db_config.primary?
-
-      File.join(File.dirname(ActiveRecord::Tasks::DatabaseTasks.schema_dump_path(db_config, format)), schema_file_type)
-    end
-
-    # Override this method from `ActiveRecord::Tasks::DatabaseTasks`
-    # to ensure that the sha saved in ar_internal_metadata table
-    # is from the original schema.rb file
-    def schema_sha1(file)
-      ActiveRecord::Tasks::DatabaseTasks.schema_dump_path(ActiveRecord::Base.configurations.configs_for(env_name: ActiveRecord::Tasks::DatabaseTasks.env, name: "primary"))
-    end
-
-    def forward(step = 1)
-      DataMigrate::DataMigrator.create_data_schema_table
+    def self.forward(step = 1)
+      DataMigrate::DataMigrator.assure_data_schema_table
       migrations = pending_migrations.reverse.pop(step).reverse
       migrations.each do | pending_migration |
         if pending_migration[:kind] == :data
@@ -154,21 +78,19 @@ module DataMigrate
       end
     end
 
-    def pending_data_migrations
+    def self.pending_data_migrations
       data_migrations = DataMigrate::DataMigrator.migrations(data_migrations_path)
-      data_migrator = DataMigrate::RailsHelper.data_migrator(:up, data_migrations)
-      sort_migrations(
-        data_migrator.pending_migrations.map { |m| { version: m.version, name: m.name, kind: :data } }
-        )
+      sort_migrations(DataMigrate::DataMigrator.new(:up, data_migrations ).
+        pending_migrations.map {|m| { version: m.version, name: m.name, kind: :data }})
     end
 
-    def pending_schema_migrations
+    def self.pending_schema_migrations
       ::DataMigrate::SchemaMigration.pending_schema_migrations
     end
 
-    def past_migrations(sort = nil)
-      data_versions = DataMigrate::RailsHelper.data_schema_migration.table_exists? ? DataMigrate::RailsHelper.data_schema_migration.normalized_versions : []
-      schema_versions = DataMigrate::RailsHelper.schema_migration.normalized_versions
+    def self.past_migrations(sort = nil)
+      data_versions = DataMigrate::DataSchemaMigration.table_exists? ? DataMigrate::DataSchemaMigration.normalized_versions : []
+      schema_versions = ActiveRecord::SchemaMigration.normalized_versions
       migrations = data_versions.map { |v| { version: v.to_i, kind: :data } } + schema_versions.map { |v| { version: v.to_i, kind: :schema } }
 
       sort&.downcase == "asc" ? sort_migrations(migrations) : sort_migrations(migrations).reverse

data/lib/data_migrate/legacy_migrator.rb

@@ -0,0 +1,22 @@
+module DataMigrate
+  class LegacyMigrator
+    def initialize(migrations_paths = "db/data")
+      @migrations_paths = migrations_paths || "db/data"
+    end
+
+    def migrate
+      dates =
+        DataMigrate::DataMigrator.migrations(@migrations_paths).collect(&:version)
+      legacy = ActiveRecord::SchemaMigration.where(version: dates)
+      legacy.each do |v|
+        begin
+          version = v.version
+          puts "Creating #{version} in data schema"
+          DataMigrate::DataSchemaMigration.create(version: version)
+        rescue ActiveRecord::RecordNotUnique
+          nil
+        end
+      end
+    end
+  end
+end