data_migrate 5.0.2 → 5.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: c3db86bc3ba15d15bed751ad312b844a2644f61982085863fb5a25456c1f1bf6
-  data.tar.gz: f5c22f029369bb4174c6f33020e5ff558b89ea9c2ade09cde3b9a22a96c22c67
+SHA1:
+  metadata.gz: 374e93835d9a65e56af9ce534dd650c286a3dbcf
+  data.tar.gz: 48a7b7a1b7870b28f4cc34bff8468cc797649c8d
 SHA512:
-  metadata.gz: ba1af7cfb8c5d7f3ce4c5f96e06c16e84598f7a436e7c75920889d1d9c5429ca8c97e29c5a6b012aaea556e8423ea039ba3917a0ae515371463d055d5a83d918
-  data.tar.gz: e733b1743936438b68b1c8168fe4bb0305e2d960c5410588d3f15ab76cbf3c053beb2e86610b1e1f499b05381651eed799d820ac5c5dae5e57264f4a5fa80553
+  metadata.gz: 3b53c608118790f93587e29221f7d5aad40fcbe782f081f314d3955228f6ad45f2fbe90b565221aeb90fb38fe6642883f850b31ae7ec8d925b7b34c7a07cc2b7
+  data.tar.gz: 1500d7f084fcdc4b8f0a442925a695c90b495a148da5ba6021add79d432c8826c6bc2b965bc178595796d92fe8d990f931860f02e0fadcc68674546e366bf2bf

data/.gitignore

@@ -4,6 +4,5 @@
 gemfiles/.bundle
 spec/db/test.db
 .vscode/
-tasks/databases.rake
 .DS_Store
 .idea/

data/.travis.yml

@@ -1,16 +1,12 @@
 language: ruby
 rvm:
-  - 2.2.6
-  - 2.3.4
+  - 2.2.9
+  - 2.3.7
+  - 2.4.4
+  - 2.5.1
 script: bundle exec rspec
 gemfile:
   - gemfiles/rails_4.2.gemfile
   - gemfiles/rails_5.0.gemfile
   - gemfiles/rails_5.1.gemfile
   - gemfiles/rails_5.2.gemfile
-matrix:
-  exclude:
-    - rvm: 2.0.0-p648
-      gemfile: gemfiles/rails_5.0.gemfile
-    - rvm: 2.1.10
-      gemfile: gemfiles/rails_5.0.gemfile

data/Changelog.md

@@ -1,6 +1,9 @@
 Changelog
 =========
 
+## 5.1.0
+Fixes to `db:schema:load:with_data` + `db:structure:load:with_data` definition, thanks to [craineum](https://github.com/craineum)
+
 ## 5.0.0
 
 Remove support for legacy migrations (from v2).

data/README.md

@@ -131,7 +131,8 @@ You can generate a data migration as you would a schema migration:
     rake db:migrate:up:with_data      # Runs the "up" for a given migration VERSION
     rake db:migrate:with_data         # Migrate the database data and schema (options: VERSION=x, VERBOSE=false)
     rake db:rollback:with_data        # Rolls the schema back to the previous version (specify steps w/ STEP=n)
-    rake db:schema:load:with_data     # Load both schema.rb and data_schema.rb file into the database
+    rake db:schema:load:with_data     # Load both schema.rb and data_schema.rb files into the database
+    rake db:structure:load:with_data  # Load both structure.sql and data_schema.rb files into the database
     rake db:version:with_data         # Retrieves the current schema version numbers for data and schema migrations
 
 Tasks work as they would with the 'vanilla' db version. The 'with_data' addition to the 'db' tasks will run the task in the context of both the data and schema migrations. That is, rake db:rollback:with_data will check to see if it was a schema or data migration invoked last, and do that. Tasks invoked in that space also have an additional line of output, indicating if the action is performed on data or schema.

data/lib/data_migrate/data_migrator_five.rb

@@ -7,6 +7,7 @@ module DataMigrate
     self.migrations_paths = ["db/data"]
 
     def self.assure_data_schema_table
+      ActiveRecord::Base.establish_connection(db_config)
       DataMigrate::DataSchemaMigration.create_table
     end
 
@@ -41,6 +42,13 @@ module DataMigrate
         /(\d{14})_(.+)\.rb/.match(filename)
       end
 
+      ##
+      # Provides the full migrations_path filepath
+      # @return (String)
+      def full_migrations_path
+        File.join(Rails.root, *migrations_paths.split(File::SEPARATOR))
+      end
+
       def migrations_status
         DataMigrate::MigrationContext.new(migrations_paths).migrations_status
       end
@@ -58,6 +66,11 @@ module DataMigrate
       def rollback(migrations_path, steps)
         DataMigrate::MigrationContext.new(migrations_path).rollback(steps)
       end
+
+      def db_config
+        ActiveRecord::Base.configurations[Rails.env || "development"] ||
+          ENV["DATABASE_URL"]
+      end
     end
 
     private

data/lib/data_migrate/database_tasks.rb

@@ -2,11 +2,12 @@
 
 module DataMigrate
   ##
-  # This class extends DatabaseTasks to add a data_schema_file method.
+  # This class extends DatabaseTasks to add a schema_file method.
   class DatabaseTasks
     extend ActiveRecord::Tasks::DatabaseTasks
 
-    def self.data_schema_file
+    # This overrides ActiveRecord::Tasks::DatabaseTasks
+    def self.schema_file(_format = nil)
       File.join(db_dir, "data_schema.rb")
     end
 

data/lib/data_migrate/version.rb

@@ -1,3 +1,3 @@
 module DataMigrate
-  VERSION = "5.0.2".freeze
+  VERSION = "5.1.0".freeze
 end

data/spec/data_migrate/data_spec.rb

@@ -31,8 +31,6 @@ describe DataMigrate::Data do
     before do
       allow(DataMigrate::DataMigrator).
         to receive(:db_config) { db_config }
-      ActiveRecord::Base.establish_connection(db_config)
-      ActiveRecord::SchemaMigration.create_table
     end
 
     after do

data/spec/data_migrate/database_tasks_spec.rb

@@ -48,9 +48,9 @@ describe DataMigrate::DatabaseTasks do
     ActiveRecord::Base.establish_connection(db_config)
   end
 
-  describe :data_schema_file do
+  describe :schema_file do
     it "returns the correct data schema file path" do
-      expect(subject.data_schema_file).to eq "db/data_schema.rb"
+      expect(subject.schema_file(nil)).to eq "db/data_schema.rb"
     end
   end
 

data/spec/spec_helper.rb

@@ -7,3 +7,9 @@ require 'timecop'
 if Rails::VERSION::MAJOR == 5 &&  Rails::VERSION::MINOR == 2
   DataMigrate::DataMigrator.migrations_paths = ["spec/db/data"]
 end
+
+RSpec.configure do |config|
+  config.mock_with :rspec do |mocks|
+    mocks.verify_partial_doubles = true
+  end
+end

data/tasks/databases.rake

@@ -0,0 +1,391 @@
+require 'data_migrate/tasks/data_migrate_tasks'
+
+namespace :db do
+  namespace :migrate do
+    desc "Migrate the database data and schema (options: VERSION=x, VERBOSE=false)."
+    task :with_data => :environment do
+      assure_data_schema_table
+
+      ActiveRecord::Migration.verbose = ENV["VERBOSE"] ? ENV["VERBOSE"] == "true" : true
+      target_version = ENV["VERSION"] ? ENV["VERSION"].to_i : nil
+      migrations = []
+
+      if target_version.nil?
+        migrations = pending_migrations.map{ |m| m.merge(:direction =>:up) }
+      else
+        current_schema_version = ActiveRecord::Migrator.current_version
+        schema_migrations = if target_version > current_schema_version
+                              pending_schema_migrations.keep_if{ |m| m[:version] <= target_version }.map{ |m| m.merge(:direction =>:up) }
+                            elsif target_version < current_schema_version
+                              past_migrations.keep_if{ |m| m[:version] > target_version }.map{ |m| m.merge(:direction =>:down) }
+                            else # ==
+                              []
+                            end
+
+        current_data_version = ActiveRecord::Migrator.current_version
+        data_migrations = if target_version > current_data_version
+                            pending_data_migrations.keep_if{ |m| m[:version] <= target_version }.map{ |m| m.merge(:direction =>:up) }
+                          elsif target_version < current_data_version
+                            past_migrations.keep_if{ |m| m[:version] > target_version }.map{ |m| m.merge(:direction =>:down) }
+                          else # ==
+                            []
+                          end
+        migrations = if schema_migrations.empty?
+                       data_migrations
+                     elsif data_migrations.empty?
+                       schema_migrations
+                     elsif target_version > current_data_version && target_version > current_schema_version
+                       sort_migrations data_migrations, schema_migrations
+                     elsif target_version < current_data_version && target_version < current_schema_version
+                       sort_migrations(data_migrations, schema_migrations).reverse
+                     elsif target_version > current_data_version && target_version < current_schema_version
+                       schema_migrations + data_migrations
+                     elsif target_version < current_data_version && target_version > current_schema_version
+                       schema_migrations + data_migrations
+                     end
+      end
+
+      migrations.each do |migration|
+        if migration[:kind] == :data
+          ActiveRecord::Migration.write("== %s %s" % ['Data', "=" * 71])
+          DataMigrate::DataMigrator.run(migration[:direction], "db/data/", migration[:version])
+        else
+          ActiveRecord::Migration.write("== %s %s" % ['Schema', "=" * 69])
+          DataMigrate::SchemaMigration.run(
+            migration[:direction],
+            Rails.application.config.paths["db/migrate"],
+            migration[:version]
+          )
+        end
+      end
+
+      Rake::Task["db:_dump"].invoke
+      Rake::Task["data:dump"].invoke
+    end
+
+    namespace :redo do
+      desc 'Rollbacks the database one migration and re migrate up (options: STEP=x, VERSION=x).'
+      task :with_data => :environment do
+      assure_data_schema_table
+        if ENV["VERSION"]
+          Rake::Task["db:migrate:down:with_data"].invoke
+          Rake::Task["db:migrate:up:with_data"].invoke
+        else
+          Rake::Task["db:rollback:with_data"].invoke
+          Rake::Task["db:migrate:with_data"].invoke
+        end
+      end
+    end
+
+    namespace :up do
+      desc 'Runs the "up" for a given migration VERSION. (options both=false)'
+      task :with_data => :environment do
+        version = ENV["VERSION"] ? ENV["VERSION"].to_i : nil
+        raise "VERSION is required" unless version
+        assure_data_schema_table
+        run_both = ENV["BOTH"] == "true"
+        migrations = pending_migrations.keep_if{|m| m[:version] == version}
+
+        unless run_both || migrations.size < 2
+          migrations = migrations.slice(0,1)
+        end
+
+        migrations.each do |migration|
+          if migration[:kind] == :data
+            ActiveRecord::Migration.write("== %s %s" % ['Data', "=" * 71])
+            DataMigrate::DataMigrator.run(:up, "db/data/", migration[:version])
+          else
+            ActiveRecord::Migration.write("== %s %s" % ['Schema', "=" * 69])
+            DataMigrate::SchemaMigration.run(:up, "db/migrate/", migration[:version])
+          end
+        end
+
+        Rake::Task["db:_dump"].invoke
+        Rake::Task["data:dump"].invoke
+      end
+    end
+
+    namespace :down do
+      desc 'Runs the "down" for a given migration VERSION. (option BOTH=false)'
+      task :with_data => :environment do
+        version = ENV["VERSION"] ? ENV["VERSION"].to_i : nil
+        raise "VERSION is required" unless version
+        assure_data_schema_table
+        run_both = ENV["BOTH"] == "true"
+        migrations = past_migrations.keep_if{|m| m[:version] == version}
+
+        unless run_both || migrations.size < 2
+          migrations = migrations.slice(0,1)
+        end
+
+        migrations.each do |migration|
+          if migration[:kind] == :data
+            ActiveRecord::Migration.write("== %s %s" % ['Data', "=" * 71])
+            DataMigrate::DataMigrator.run(:down, "db/data/", migration[:version])
+          else
+            ActiveRecord::Migration.write("== %s %s" % ['Schema', "=" * 69])
+            DataMigrate::SchemaMigration.run(:down, "db/migrate/", migration[:version])
+          end
+        end
+
+        Rake::Task["db:_dump"].invoke
+        Rake::Task["data:dump"].invoke
+      end
+    end
+
+    namespace :status do
+      desc "Display status of data and schema migrations"
+      task :with_data => :environment do
+        config = connect_to_database
+        next unless config
+
+        db_list_data = ActiveRecord::Base.connection.select_values(
+          "SELECT version FROM #{DataMigrate::DataSchemaMigration.table_name}"
+        )
+        db_list_schema = ActiveRecord::Base.connection.select_values(
+          "SELECT version FROM #{ActiveRecord::SchemaMigration.schema_migrations_table_name}"
+        )
+        file_list = []
+
+        Dir.foreach(File.join(Rails.root, 'db', 'data')) do |file|
+          # only files matching "20091231235959_some_name.rb" pattern
+          if match_data = /(\d{14})_(.+)\.rb/.match(file)
+            status = db_list_data.delete(match_data[1]) ? 'up' : 'down'
+            file_list << [status, match_data[1], match_data[2], 'data']
+          end
+        end
+
+        Dir.foreach(File.join(Rails.root, 'db', 'migrate')) do |file|
+          # only files matching "20091231235959_some_name.rb" pattern
+          if match_data = /(\d{14})_(.+)\.rb/.match(file)
+            status = db_list_schema.delete(match_data[1]) ? 'up' : 'down'
+            file_list << [status, match_data[1], match_data[2], 'schema']
+          end
+        end
+
+        file_list.sort!{|a,b| "#{a[1]}_#{a[3] == 'data' ? 1 : 0}" <=> "#{b[1]}_#{b[3] == 'data' ? 1 : 0}" }
+
+        # output
+        puts "\ndatabase: #{config['database']}\n\n"
+        puts "#{"Status".center(8)} #{"Type".center(8)}  #{"Migration ID".ljust(14)} Migration Name"
+        puts "-" * 60
+        file_list.each do |file|
+          puts "#{file[0].center(8)} #{file[3].center(8)} #{file[1].ljust(14)}  #{file[2].humanize}"
+        end
+        db_list_schema.each do |version|
+          puts "#{'up'.center(8)}  #{version.ljust(14)}  *** NO SCHEMA FILE ***"
+        end
+        db_list_data.each do |version|
+          puts "#{'up'.center(8)}  #{version.ljust(14)}  *** NO DATA FILE ***"
+        end
+        puts
+      end
+    end
+  end # END OF MIGRATE NAME SPACE
+
+  namespace :rollback do
+    desc 'Rolls the schema back to the previous version (specify steps w/ STEP=n).'
+    task :with_data => :environment do
+      step = ENV['STEP'] ? ENV['STEP'].to_i : 1
+      assure_data_schema_table
+      past_migrations[0..(step - 1)].each do | past_migration |
+        if past_migration[:kind] == :data
+          ActiveRecord::Migration.write("== %s %s" % ['Data', "=" * 71])
+          DataMigrate::DataMigrator.run(:down, "db/data/", past_migration[:version])
+        elsif past_migration[:kind] == :schema
+          ActiveRecord::Migration.write("== %s %s" % ['Schema', "=" * 69])
+          ActiveRecord::Migrator.run(:down, "db/migrate/", past_migration[:version])
+        end
+      end
+
+      Rake::Task["db:_dump"].invoke
+      Rake::Task["data:dump"].invoke
+    end
+  end
+
+  namespace :forward do
+    desc 'Pushes the schema to the next version (specify steps w/ STEP=n).'
+    task :with_data => :environment do
+      assure_data_schema_table
+      step = ENV['STEP'] ? ENV['STEP'].to_i : 1
+      DataMigrate::DatabaseTasks.forward(step)
+      Rake::Task["db:_dump"].invoke
+      Rake::Task["data:dump"].invoke
+    end
+  end
+
+  namespace :version do
+    desc "Retrieves the current schema version numbers for data and schema migrations"
+    task :with_data => :environment do
+      assure_data_schema_table
+      puts "Current Schema version: #{ActiveRecord::Migrator.current_version}"
+      puts "Current Data version: #{DataMigrate::DataMigrator.current_version}"
+    end
+  end
+
+  namespace :schema do
+    namespace :load do
+      desc "Load both schema.rb and data_schema.rb file into the database"
+      task with_data: :environment do
+        Rake::Task["db:schema:load"].invoke
+
+        DataMigrate::DatabaseTasks.load_schema_current(
+          :ruby,
+          ENV["DATA_SCHEMA"]
+        )
+      end
+    end
+  end
+
+  namespace :structure do
+    namespace :load do
+      desc "Load both structure.sql and data_schema.rb file into the database"
+      task with_data: :environment do
+        Rake::Task["db:structure:load"].invoke
+
+        DataMigrate::DatabaseTasks.load_schema_current(
+          :ruby,
+          ENV["DATA_SCHEMA"]
+        )
+      end
+    end
+  end
+end
+
+namespace :data do
+  desc 'Migrate data migrations (options: VERSION=x, VERBOSE=false)'
+  task :migrate => :environment do
+    DataMigrate::Tasks::DataMigrateTasks.migrate
+    Rake::Task["data:dump"].invoke
+  end
+
+  namespace :migrate do
+    desc  'Rollbacks the database one migration and re migrate up (options: STEP=x, VERSION=x).'
+    task :redo => :environment do
+      assure_data_schema_table
+      if ENV["VERSION"]
+        Rake::Task["data:migrate:down"].invoke
+        Rake::Task["data:migrate:up"].invoke
+      else
+        Rake::Task["data:rollback"].invoke
+        Rake::Task["data:migrate"].invoke
+      end
+    end
+
+    desc 'Runs the "up" for a given migration VERSION.'
+    task :up => :environment do
+      assure_data_schema_table
+      version = ENV["VERSION"] ? ENV["VERSION"].to_i : nil
+      raise "VERSION is required" unless version
+      DataMigrate::DataMigrator.run(:up, "db/data/", version)
+      Rake::Task["data:dump"].invoke
+    end
+
+    desc 'Runs the "down" for a given migration VERSION.'
+    task :down => :environment do
+      version = ENV["VERSION"] ? ENV["VERSION"].to_i : nil
+      raise "VERSION is required" unless version
+      assure_data_schema_table
+      DataMigrate::DataMigrator.run(:down, "db/data/", version)
+      Rake::Task["data:dump"].invoke
+    end
+
+    desc "Display status of data migrations"
+    task :status => :environment do
+      config = ActiveRecord::Base.configurations[Rails.env || 'development']
+      ActiveRecord::Base.establish_connection(config)
+      connection = ActiveRecord::Base.connection
+      puts "\ndatabase: #{config['database']}\n\n"
+      DataMigrate::StatusService.dump(connection)
+    end
+  end
+
+  desc 'Rolls the schema back to the previous version (specify steps w/ STEP=n).'
+  task :rollback => :environment do
+    assure_data_schema_table
+    step = ENV['STEP'] ? ENV['STEP'].to_i : 1
+    DataMigrate::DataMigrator.rollback('db/data/', step)
+    Rake::Task["data:dump"].invoke
+  end
+
+  desc 'Pushes the schema to the next version (specify steps w/ STEP=n).'
+  task :forward => :environment do
+    assure_data_schema_table
+    step = ENV['STEP'] ? ENV['STEP'].to_i : 1
+    # TODO: No worky for .forward
+    # DataMigrate::DataMigrator.forward('db/data/', step)
+    migrations = pending_data_migrations.reverse.pop(step).reverse
+    migrations.each do | pending_migration |
+      DataMigrate::DataMigrator.run(:up, "db/data/", pending_migration[:version])
+    end
+    Rake::Task["data:dump"].invoke
+  end
+
+  desc "Retrieves the current schema version number for data migrations"
+  task :version => :environment do
+    assure_data_schema_table
+    puts "Current data version: #{DataMigrate::DataMigrator.current_version}"
+  end
+
+  desc "Create a db/data_schema.rb file that stores the current data version"
+  task dump: :environment do
+    if ActiveRecord::Base.dump_schema_after_migration
+      filename = DataMigrate::DatabaseTasks.schema_file
+      File.open(filename, "w:utf-8") do |file|
+        DataMigrate::SchemaDumper.dump(ActiveRecord::Base.connection, file)
+      end
+    end
+
+    # Allow this task to be called as many times as required. An example
+    # is the migrate:redo task, which calls other two internally
+    # that depend on this one.
+    Rake::Task["data:dump"].reenable
+  end
+end
+
+def pending_migrations
+  DataMigrate::DatabaseTasks.sort_migrations(
+    DataMigrate::DatabaseTasks.pending_schema_migrations,
+    DataMigrate::DatabaseTasks.pending_data_migrations
+  )
+end
+
+def pending_data_migrations
+  DataMigrate::DatabaseTasks.pending_data_migrations
+end
+
+def pending_schema_migrations
+  DataMigrate::DatabaseTasks.pending_schema_migrations
+end
+
+def sort_migrations set_1, set_2=nil
+  migrations = set_1 + (set_2 || [])
+  migrations.sort{|a,b|  sort_string(a) <=> sort_string(b)}
+end
+
+def sort_string migration
+  "#{migration[:version]}_#{migration[:kind] == :data ? 1 : 0}"
+end
+
+def connect_to_database
+  config = ActiveRecord::Base.configurations[Rails.env || 'development']
+  ActiveRecord::Base.establish_connection(config)
+
+  unless DataMigrate::DataSchemaMigration.table_exists?
+    puts 'Data migrations table does not exist yet.'
+    config = nil
+  end
+  unless ActiveRecord::SchemaMigration.table_exists?
+    puts 'Schema migrations table does not exist yet.'
+    config = nil
+  end
+  config
+end
+
+def past_migrations(sort=nil)
+  DataMigrate::DatabaseTasks.past_migrations(sort)
+end
+
+def assure_data_schema_table
+  DataMigrate::DataMigrator.assure_data_schema_table
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: data_migrate
 version: !ruby/object:Gem::Version
-  version: 5.0.2
+  version: 5.1.0
 platform: ruby
 authors:
 - Andrew J Vargo
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-08-22 00:00:00.000000000 Z
+date: 2018-09-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -275,7 +275,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: data_migrate
-rubygems_version: 2.7.4
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: Rake tasks to migrate data alongside schema changes.