mysql_big_table_migration 0.1.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2cf0a701c21368899da7dd1cb5ca1004a403513b
+  data.tar.gz: f2d290bdf85b48ba4140e360e730c7de86a50b1f
+SHA512:
+  metadata.gz: caf05416fd733107981ddee51522b5c893c47d612da106525c5658503ee52ce1887a84d56b50b2bdc961ae714e479fea5a3c9bad9b78acb9c6e6631e440ad429
+  data.tar.gz: f19ea5701c1c20fd916368b5eb8b7e9fa84634340c2a03cb55de79e6df1a6b954a25490d487c6203a3a4f5597668e17fb787e59018aa3abdb9f7d8e87214f885

data/.gitignore

@@ -0,0 +1,5 @@
+*.*~
+*.swp
+test/debug.log
+Gemfile.lock
+pkg/

data/.travis.yml

@@ -0,0 +1,23 @@
+language: ruby
+env:
+  - DB=mysql
+rvm:
+  - 1.9
+  - 2.1
+  - 2.2
+  - rbx-2.5.2
+bundler_args: --without production
+before_script:
+  - mysql -e 'create database mysql_big_table_migration_test'
+before_install:
+  - gem update --system
+services:
+  - mysql
+script:
+  - bundle install
+  - bundle exec rake test
+cache:
+  - bundler
+os:
+  - linux
+  - osx

data/CHANGELOG

@@ -0,0 +1,17 @@
+TODO:
+
+* Support change_table :foo do |t| style migrations (with option to activate temp table)
+* Support ActiveRecord::Migration::CommandRecorder (rake db:rollback)
+* Ensure index names based on original table
+
+0.1.2:
+
+* Moved repo from thickpaddy/mysql_big_table_migration to MakeYourLaws/mysql_big_table_migration
+* Added dependencies on ActiveRecord & Rails, removed unused rdoc, updated to minitest, moved from Rails::VERSION to ActiveRecord::VERSION
+* Fixed tests - mysql can't convert 'foo0' to an integer
+* Moved dependencies into gemspec, updated Gemfile.lock
+* Pushed to RubyGems
+
+<= 0.1.1:
+
+See https://github.com/thickpaddy/mysql_big_table_migration/commits/master

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2010 [name of plugin creator]
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,58 @@
+[![Build Status](https://travis-ci.org/MakeYourLaws/mysql_big_table_migration.svg)](https://travis-ci.org/MakeYourLaws/mysql_big_table_migration)
+
+MysqlBigTableMigration
+======================
+
+A Rails plugin that adds methods to ActiveRecord::Migration to allow columns
+and indexes to be added to and removed from large tables with millions of
+rows in MySQL, without leaving processes seemingly stalled in state "copy
+to tmp table".
+
+For each of the standard transformations that operate on columns or indexes,
+this plugin adds a "using_tmp_table" version. These methods create a
+temporary table with the same structure as the table to be altered, applies
+the transformation to the temp table, copies data from the source table to
+the temp table and then replaces the source table with the temporary one.
+
+While it does try to ensure that data is consistent at the end of the entire
+process by locking tables and looking for rows created or modified during
+copying, this is NOT TRANSACTION SAFE as it (a) relies on timestamp columns
+and (b) doesn't handle rows that have been deleted from the source table
+after being copied to the temporary table.
+
+Installation
+============
+
+	source 'https://rubygems.org' do
+	  ...
+	  gem 'mysql_big_table_migration'
+	  ...
+	end
+
+
+Example
+=======
+
+
+	class AddIndexOnSomeColumnToSomeTable < ActiveRecord::Migration
+	  def self.up
+	    add_index_using_tmp_table :some_table, :some_column
+	  end
+
+	  def self.down
+	    remove_index_using_tmp_table :some_table, :some_column
+	  end
+	end
+
+
+Copyright (c) 2010 Mark Woods, released under the MIT license
+
+Testing
+========
+
+You will need to bundle install dependencies for the project, as well as create a test database. To install dependencies run `bundle install`. To create the database run the following in a MySQL prompt as an admin user:
+
+	CREATE DATABASE mysql_big_table_migration_test;
+	GRANT ALL PRIVILEGES ON *.* TO 'dev'@'localhost' IDENTIFIED BY 'password';
+
+After bundling dependencies, and setting up the test database, run the tests with `rake test`

data/Rakefile

@@ -0,0 +1,15 @@
+require 'rake'
+require 'rake/testtask'
+
+desc 'Default: run unit tests.'
+task :default => :test
+
+desc 'Test the mysql_big_table_migration plugin.'
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.pattern = 'test/*_test.rb'
+  t.verbose = true
+end
+
+Bundler::GemHelper.install_tasks
+Dir.glob('tasks/*.rake').each { |r| import r }

data/init.rb

@@ -0,0 +1 @@
+require 'mysql_big_table_migration'

data/install.rb

@@ -0,0 +1 @@
+# Install hook code here

data/lib/mysql_big_table_migration.rb

@@ -0,0 +1,195 @@
+module MySQLBigTableMigration
+
+  SUPPORTED_ADAPTERS = [
+    "ActiveRecord::ConnectionAdapters::MysqlAdapter",
+    "ActiveRecord::ConnectionAdapters::Mysql2Adapter"
+  ]
+
+  def rename_column(table, old_name, new_name)
+    push_state unless @state_stack.try(:any?)
+    current_state[:renames] ||= {}
+    current_state[:renames][old_name] = new_name
+    super
+  end
+
+  def add_column_using_tmp_table(table_name, *args)
+    with_tmp_table(table_name) do |tmp_table_name|
+      begin
+        add_column tmp_table_name, *args
+      rescue ActiveRecord::StatementInvalid => e
+        raise unless e.message.include?("Duplicate column name")
+      end
+    end
+  end
+
+  def remove_column_using_tmp_table(table_name, column_name)
+    with_tmp_table(table_name) do |tmp_table_name|
+      begin
+        remove_column tmp_table_name, column_name
+      rescue ActiveRecord::StatementInvalid => e
+        raise unless e.message.include?("check that column/key exists")
+      end
+    end
+  end
+
+  def rename_column_using_tmp_table(table_name, column_name, new_column_name)
+    with_tmp_table(table_name) { |tmp_table_name| rename_column(tmp_table_name, column_name, new_column_name) }
+  end
+
+  def change_column_using_tmp_table(table_name, column_name, type, options={})
+    with_tmp_table(table_name) { |tmp_table_name| change_column(tmp_table_name, column_name, type, options) }
+  end
+
+  def add_index_using_tmp_table(table_name, column_name, options={})
+    # generate the index name using the original table name if no name provided
+    options[:name] = index_name(table_name, :column => Array(column_name)) if options[:name].nil?
+    with_tmp_table(table_name) { |tmp_table_name| add_index(tmp_table_name, column_name, options) }
+  end
+
+  def remove_index_using_tmp_table(table_name, options={})
+    with_tmp_table(table_name) { |tmp_table_name| remove_index(tmp_table_name, :name => index_name(table_name, options)) }
+  end
+
+  def with_tmp_table(table_name)
+    raise ArgumentError, "block expected" unless block_given?
+
+    unless SUPPORTED_ADAPTERS.include? connection.class.name
+      puts "Warning: Unsupported connection adapter '#{connection.class.name}' for MySQL Big Table Migration Plugin"
+      puts "         Migration methods will still be executed, but without using a temp table."
+      yield table_name
+      return
+    end
+
+    table_name = table_name.to_s
+    new_table_name = "tmp_new_" + table_name
+    old_table_name = "tmp_old_" + table_name
+
+    begin
+      say "Creating temporary table #{new_table_name} like #{table_name}..."
+      connection.execute("CREATE TABLE #{new_table_name} LIKE #{table_name}")
+
+      # yield the temporary table name to the block, which should alter the table using standard migration methods
+      push_state
+      yield new_table_name
+      state = pop_state
+
+      rename_columns ||= state[:renames] || {}
+
+      # get column names to copy *after* yielding to block - could drop a column from new table
+      # note: do not get column names using the column_names method, we need to make sure we avoid obtaining a cached array of column names
+      old_column_names = []
+      each_result_hash(connection.execute("DESCRIBE #{table_name}")) { |row| old_column_names << row['Field'] } # see ruby mysql docs for more info
+      new_column_names = []
+      each_result_hash(connection.execute("DESCRIBE #{new_table_name}")) { |row| new_column_names << row['Field'] }
+
+      shared_columns = old_column_names & new_column_names
+
+      old_column_list = column_list(shared_columns + rename_columns.keys)
+      new_column_list = column_list(shared_columns + rename_columns.values)
+
+      timestamp_before_migration = fetch_result_row(connection.execute("SELECT CURRENT_TIMESTAMP"))[0] # note: string, not time object
+      max_id_before_migration = fetch_result_row(connection.execute("SELECT MAX(id) FROM #{table_name}"))[0].to_i
+
+      if max_id_before_migration == 0
+        say "Source table is empty, no rows to copy into temporary table"
+      else
+        batch_size = mysql_big_table_migration_bach_size
+        start = fetch_result_row(connection.execute("SELECT MIN(id) FROM #{table_name}"))[0].to_i
+        counter = start
+        say "Inserting into temporary table in batches of #{batch_size}..."
+        say "Approximately #{max_id_before_migration-start+1} rows to process, first row has id #{start}", true
+        while counter <= ( max = fetch_result_row(connection.execute("SELECT MAX(id) FROM #{table_name}"))[0].to_i )
+          percentage_complete = mysql_big_table_migration_completion start, counter, max
+          say "Processing rows with ids between #{counter} and #{(counter+batch_size)-1} (#{percentage_complete}% complete)", true
+          connection.execute("INSERT INTO #{new_table_name} (#{new_column_list}) SELECT #{old_column_list} FROM #{table_name} WHERE id >= #{counter} AND id < #{counter + batch_size}")
+          counter = counter + batch_size
+        end
+        say "Finished inserting into temporary table"
+      end
+
+    rescue Exception => e
+      drop_table new_table_name
+      raise
+    end
+
+    say "Replacing source table with temporary table..."
+    rename_table table_name, old_table_name
+    rename_table new_table_name, table_name
+
+    say "Cleaning up, checking for rows created/updated during migration, dropping old table..."
+    begin
+      connection.execute("LOCK TABLES #{table_name} WRITE, #{old_table_name} READ")
+      recently_created_or_updated_conditions = "id > #{max_id_before_migration}"
+      recently_created_or_updated_conditions << " OR updated_at > '#{timestamp_before_migration}'" if old_column_names.include?("updated_at")
+      connection.execute("REPLACE INTO #{table_name} (#{new_column_list}) SELECT #{old_column_list} FROM #{old_table_name} WHERE #{recently_created_or_updated_conditions}")
+    rescue Exception => e
+      puts "Failed to lock tables and do final cleanup. This may not be anything to worry about, especially on an infrequently used table."
+      puts "ERROR MESSAGE: " + e.message
+    ensure
+      connection.execute("UNLOCK TABLES")
+    end
+    drop_table old_table_name
+  end
+
+  private
+
+  def mysql_big_table_migration_completion(start, counter, max)
+    number_done = counter - start + 1
+    number_to_do = max - start + 1
+    (number_done * 100 / number_to_do.to_f).to_i
+  end
+
+  def mysql_big_table_migration_bach_size
+    10000
+  end
+
+  def connection
+    ActiveRecord::Base.connection
+  end
+
+  def each_result_hash(result, &block)
+    case connection.class.name
+    when "ActiveRecord::ConnectionAdapters::MysqlAdapter"
+      result.each_hash(&block)
+    when "ActiveRecord::ConnectionAdapters::Mysql2Adapter"
+      result.each(:as => :hash, &block)
+    end
+  end
+
+  def fetch_result_row(result)
+    case connection.class.name
+    when "ActiveRecord::ConnectionAdapters::MysqlAdapter"
+      result.fetch_row
+    when "ActiveRecord::ConnectionAdapters::Mysql2Adapter"
+      result.first
+    end
+  end
+
+  def column_list(column_names)
+    "`" + column_names.join("`, `") + "`"
+  end
+
+  def push_state
+    @state_stack ||= []
+    @state_stack.push({})
+  end
+
+  def pop_state
+    @state_stack.pop
+  end
+
+  def current_state
+    @state_stack.last
+  end
+
+end
+
+if Object.const_defined?("ActiveRecord")
+  class ActiveRecord::Migration
+    if ActiveRecord::VERSION::STRING < "3.0"
+      extend MySQLBigTableMigration
+    else
+      include MySQLBigTableMigration
+    end
+  end
+end

data/lib/mysql_big_table_migration/version.rb

@@ -0,0 +1,3 @@
+module MySQLBigTableMigration
+  VERSION = "0.1.2" unless defined? MySQLBigTableMigration::VERSION
+end

data/mysql_big_table_migration.gemspec

@@ -0,0 +1,36 @@
+$LOAD_PATH.unshift 'lib'
+require "mysql_big_table_migration/version"
+
+Gem::Specification.new do |s|
+  s.name              = "mysql_big_table_migration"
+  s.version           = MySQLBigTableMigration::VERSION
+  s.platform          = Gem::Platform::RUBY
+  s.date              = Time.now.strftime('%Y-%m-%d')
+  s.summary           = "allow columns and indexes to be added to and removed from large tables"
+  s.homepage          = "http://github.com/MakeYourLaws/mysql_big_table_migration"
+  s.email             = "sai@makeyourlaws.org"
+  s.authors           = [ "Mark Woods" ]
+  s.has_rdoc          = false
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test}/*`.split("\n")
+  # s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency "activerecord"
+  s.add_dependency "activesupport"
+  s.add_development_dependency "bundler"
+  s.add_development_dependency "fileutils"
+  s.add_development_dependency "logger"
+  s.add_development_dependency "minitest"
+  s.add_development_dependency "mysql"
+  s.add_development_dependency "mysql2"
+  s.add_development_dependency "rake"
+
+  s.description       = <<desc
+  A Rails plugin that adds methods to ActiveRecord::Migration to allow columns
+  and indexes to be added to and removed from large tables with millions of
+  rows in MySQL, without leaving processes seemingly stalled in state "copy
+  to tmp table".
+desc
+end

data/rails/init.rb

@@ -0,0 +1,2 @@
+require 'mysql_big_table_migration'
+

data/tasks/mysql_big_table_migration_tasks.rake

@@ -0,0 +1,4 @@
+# desc "Explaining what the task does"
+# task :mysql_big_table_migration do
+#   # Task goes here
+# end

data/test/database.yml

@@ -0,0 +1,17 @@
+basics: &basics
+  username: travis
+  database: mysql_big_table_migration_test
+  socket: <%= [
+      '/var/lib/mysql/mysql.sock',
+      '/var/run/mysqld/mysqld.sock',
+      '/tmp/mysqld.sock',
+      '/tmp/mysql.sock',
+    ].detect { |socket| File.exist?(socket) } %>
+
+mysql:
+  <<: *basics
+  adapter: mysql
+
+mysql2:
+  <<: *basics
+  adapter: mysql2

data/test/mysql_big_table_migration_test.rb

@@ -0,0 +1,262 @@
+require_relative 'test_helper'
+
+class MysqlBigTableMigrationTest < Minitest::Test
+  extend DatabaseTest
+
+  test_against_all_configs :methods_are_added_to_migration do
+    if ActiveRecord::VERSION::STRING < "3.0"
+      method_target = ActiveRecord::Migration
+    else
+      method_target = ActiveRecord::Migration.new
+    end
+
+    MySQLBigTableMigration.instance_methods(false).each do |method|
+      assert_respond_to method_target, method
+    end
+  end
+
+  test_against_all_configs :with_tmp_table_creates_tmp_table do
+    silence_stream($stdout) do
+      ActiveRecord::Migration.send(:with_tmp_table, :test_table) {}
+    end
+    assert_match "CREATE TABLE tmp_new_test_table LIKE test_table", read_log_file
+  end
+
+  class SmallBatchMigration < ActiveRecord::Migration
+    def mysql_big_table_migration_bach_size
+      4
+    end
+  end
+
+  test_against_all_configs :with_tmp_table_copies_all_rows do
+    silence_stream($stdout) do
+      SmallBatchMigration.new.send(:with_tmp_table, :test_table) {}
+    end
+
+    assert_equal 5, test_table_rows.length
+  end
+
+
+  test_against_all_configs :with_exactly_one_row, fixture_row_count: 1 do
+    silence_stream($stdout) do
+      ActiveRecord::Migration.send(:with_tmp_table, :test_table) {}
+    end
+
+    assert_equal 1, test_table_rows.length
+  end
+
+  test_against_all_configs :add_column_using_tmp_table do
+    silence_stream($stdout) do
+      ActiveRecord::Migration.add_column_using_tmp_table(:test_table, :baz, :string)
+    end
+
+    fields = test_table_fields
+    assert_equal 4, fields.length
+    assert_equal "baz", fields[3]["Field"]
+    assert_equal "varchar(255)", fields[3]["Type"]
+
+    results = test_table_rows
+    assert_equal 5, results.length
+    assert_equal "foo2", results[2]["foo"]
+    assert_equal "bar3", results[3]["bar"]
+    assert_equal nil, results[4]["baz"]
+  end
+
+  test_against_all_configs :remove_column_using_tmp_table do
+    silence_stream($stdout) do
+      ActiveRecord::Migration.remove_column_using_tmp_table(:test_table, :bar)
+    end
+
+    fields = test_table_fields
+    assert_equal 2, fields.length
+    assert_equal "id", fields[0]["Field"]
+    assert_equal "foo", fields[1]["Field"]
+
+    results = test_table_rows
+    assert_equal 5, results.length
+    assert_equal "foo2", results[2]["foo"]
+    assert !results[3].has_key?("bar")
+  end
+
+  test_against_all_configs :rename_column_using_tmp_table do
+    silence_stream($stdout) do
+      ActiveRecord::Migration.rename_column_using_tmp_table(:test_table, :foo, :baz)
+    end
+
+    fields = test_table_fields
+    assert_equal 3, fields.length
+    assert_equal "id", fields[0]["Field"]
+    assert_equal "int(11)", fields[0]["Type"]
+    assert_equal "baz", fields[1]["Field"]
+    assert_equal "varchar(255)", fields[1]["Type"]
+    assert_equal "bar", fields[2]["Field"]
+    assert_equal "varchar(255)", fields[2]["Type"]
+
+    results = test_table_rows
+    assert_equal 5, results.length
+    5.times do |i|
+      assert_equal "foo#{i}", results[i]["baz"]
+      assert_equal "bar#{i}", results[i]["bar"]
+    end
+  end
+
+  test_against_all_configs :change_column_using_tmp_table do
+    silence_stream($stdout) do
+      ActiveRecord::Migration.change_column_using_tmp_table(:test_table, :bar, :text)
+    end
+
+    fields = test_table_fields
+    assert_equal 3, fields.length
+    assert_equal "id", fields[0]["Field"]
+    assert_equal "foo", fields[1]["Field"]
+    assert_equal "bar", fields[2]["Field"]
+    assert_equal "text", fields[2]["Type"]
+
+    results = test_table_rows
+    assert_equal 5, results.length
+    assert_equal "foo2", results[2]["foo"]
+    assert_equal "bar3", results[3]["bar"]
+  end
+
+  test_against_all_configs :add_index_using_tmp_table do
+    silence_stream($stdout) do
+      ActiveRecord::Migration.add_index_using_tmp_table(:test_table, :bar)
+    end
+
+    indexes = result_hashes("SHOW INDEX FROM test_table")
+    assert_equal 3, indexes.length
+    assert_equal "id", indexes[0]["Column_name"]
+    assert_equal "foo", indexes[1]["Column_name"]
+    assert_equal "bar", indexes[2]["Column_name"]
+  end
+
+  test_against_all_configs :remove_index_using_tmp_table do
+    silence_stream($stdout) do
+      ActiveRecord::Migration.remove_index_using_tmp_table(:test_table, :foo)
+    end
+
+    indexes = result_hashes("SHOW INDEX FROM test_table")
+    assert_equal 1, indexes.length
+    assert_equal "id", indexes[0]["Column_name"]
+  end
+
+  test_against_all_configs :rename_with_remove do
+    silence_stream($stdout) do
+      ActiveRecord::Migration.with_tmp_table(:test_table) do |tmp_table_name|
+        ActiveRecord::Migration.rename_column tmp_table_name, :bar, :baz
+        ActiveRecord::Migration.remove_column tmp_table_name, :foo
+      end
+    end
+
+    fields = test_table_fields
+    assert_equal 2, fields.length
+    assert_equal "id", fields[0]["Field"]
+    assert_equal "baz", fields[1]["Field"]
+
+    results = test_table_rows
+    assert_equal 5, results.length
+    5.times do |i|
+      assert_equal "bar#{i}", results[i]["baz"]
+    end
+  end
+
+  test_against_all_configs :rename_with_add do
+    silence_stream($stdout) do
+      ActiveRecord::Migration.with_tmp_table(:test_table) do |tmp_table_name|
+        ActiveRecord::Migration.rename_column tmp_table_name, :bar, :baz
+        ActiveRecord::Migration.add_column tmp_table_name, :dummy, :integer
+      end
+    end
+
+    fields = test_table_fields
+    assert_equal 4, fields.length
+    assert_equal "id", fields[0]["Field"]
+    assert_equal "foo", fields[1]["Field"]
+    assert_equal "baz", fields[2]["Field"]
+    assert_equal "dummy", fields[3]["Field"]
+
+    results = test_table_rows
+    assert_equal 5, results.length
+    5.times do |i|
+      assert_equal "foo#{i}", results[i]["foo"]
+      assert_equal "bar#{i}", results[i]["baz"]
+      assert_equal nil, results[i]["dummy"]
+    end
+  end
+
+  test_against_all_configs :rename_with_change do
+    silence_stream($stdout) do
+      ActiveRecord::Migration.with_tmp_table(:test_table) do |tmp_table_name|
+        ActiveRecord::Migration.rename_column tmp_table_name, :bar, :baz
+        # MySQL can't properly change string data to integer ("Incorrect integer value: 'foo0'")
+        # Also, limit: 3 will fail ("Data too long") w/ sql_mode = 'STRICT_ALL_TABLES'
+        ActiveRecord::Migration.change_column tmp_table_name, :foo, :string, limit: 10
+      end
+    end
+
+    fields = test_table_fields
+    assert_equal 3, fields.length
+    assert_equal "id", fields[0]["Field"]
+    assert_equal "foo", fields[1]["Field"]
+    assert_equal "varchar(10)", fields[1]["Type"]
+    assert_equal "baz", fields[2]["Field"]
+
+    results = test_table_rows
+    assert_equal 5, results.length
+    5.times do |i|
+      assert_equal "foo#{i}", results[i]["foo"]
+      assert_equal "bar#{i}", results[i]["baz"]
+    end
+  end
+
+  test_against_all_configs :rename_with_rename do
+    silence_stream($stdout) do
+      ActiveRecord::Migration.with_tmp_table(:test_table) do |tmp_table_name|
+        ActiveRecord::Migration.rename_column tmp_table_name, :bar, :baz
+        ActiveRecord::Migration.rename_column tmp_table_name, :foo, :dummy
+      end
+    end
+
+    fields = test_table_fields
+    assert_equal 3, fields.length
+    assert_equal "id", fields[0]["Field"]
+    assert_equal "dummy", fields[1]["Field"]
+    assert_equal "baz", fields[2]["Field"]
+
+    results = test_table_rows
+    assert_equal 5, results.length
+    5.times do |i|
+      assert_equal "foo#{i}", results[i]["dummy"]
+      assert_equal "bar#{i}", results[i]["baz"]
+    end
+  end
+
+  test_against_all_configs :rename_column do
+    silence_stream($stdout) do
+      ActiveRecord::Migration.rename_column :test_table, :bar, :baz
+    end
+
+    fields = test_table_fields
+    assert_equal 3, fields.length
+    assert_equal "id", fields[0]["Field"]
+    assert_equal "foo", fields[1]["Field"]
+    assert_equal "baz", fields[2]["Field"]
+
+    results = test_table_rows
+    assert_equal 5, results.length
+    5.times do |i|
+      assert_equal "foo#{i}", results[i]["foo"]
+      assert_equal "bar#{i}", results[i]["baz"]
+    end
+  end
+
+  private
+
+  def test_table_fields
+    result_hashes("DESCRIBE test_table")
+  end
+
+  def test_table_rows
+    result_hashes("SELECT * FROM test_table")
+  end
+end

data/test/schema.rb

@@ -0,0 +1,8 @@
+ActiveRecord::Schema.define(:version => 0) do
+  create_table :test_table, :force => true do |t|
+    t.string :foo
+    t.string :bar
+  end
+
+  add_index :test_table, [:foo]
+end

data/test/test_helper.rb

@@ -0,0 +1,111 @@
+require 'rubygems'
+require 'fileutils'
+require 'minitest/autorun'
+require 'active_record'
+require 'active_record/connection_adapters/mysql_adapter'
+require 'active_support'
+require 'active_support/core_ext'
+require 'mysql'
+require 'mysql2'
+require 'logger'
+require 'yaml'
+require File.dirname(__FILE__) + "/../lib/mysql_big_table_migration"
+
+TEST_CONFIGS = ["mysql", "mysql2"]
+
+Mysql::Result.class_eval do
+  unless respond_to?(:all_hashes)
+    def all_hashes
+      rows = []
+      each_hash do |row|
+        rows << row
+      end
+      rows
+    end
+  end
+end
+
+def read_log_file
+  @log.string
+end
+
+def load_schema(adapter)
+  config = YAML.load(ERB.new(File.read(File.join(File.dirname(__FILE__), 'database.yml'))).result)
+  @log = StringIO.new
+  ActiveRecord::Base.logger = Logger.new(@log)
+  ActiveRecord::Base.establish_connection(config[adapter.to_s])
+  load(File.join(File.dirname(__FILE__), "schema.rb"))
+end
+
+def load_fixtures(options = {})
+  connection = ActiveRecord::Base.connection
+
+  connection.execute("DELETE FROM test_table;")
+  (options[:fixture_row_count] || 5).times do |i|
+    connection.execute("INSERT INTO test_table (foo, bar) VALUES ('foo#{i}', 'bar#{i}');")
+  end
+end
+
+def assert_valid_database_setup(options = {})
+  fields = result_hashes("DESCRIBE test_table")
+  assert_equal 3, fields.length
+  assert_equal "id", fields[0]["Field"]
+  assert_equal "int(11)", fields[0]["Type"]
+  assert_equal "foo", fields[1]["Field"]
+  assert_equal "varchar(255)", fields[1]["Type"]
+  assert_equal "bar", fields[2]["Field"]
+  assert_equal "varchar(255)", fields[2]["Type"]
+
+  indexes = result_hashes("SHOW INDEX FROM test_table")
+  assert_equal 2, indexes.length
+  assert_equal "id", indexes[0]["Column_name"]
+  assert_equal "foo", indexes[1]["Column_name"]
+
+  results = result_hashes("SELECT * FROM test_table")
+  assert_equal options[:fixture_row_count] || 5, results.length
+end
+
+def result_hashes(query)
+  result = connection.execute(query)
+  case connection
+  when ActiveRecord::ConnectionAdapters::MysqlAdapter
+    result.all_hashes
+  when ActiveRecord::ConnectionAdapters::Mysql2Adapter
+    result.each(:as => :hash)
+  else
+    raise "Unknown adapter"
+  end
+end
+
+# Proc#bind removed from ActiveSupport 4+
+class Proc
+  def bind(object)
+    block, time = self, Time.now
+    object.class_eval do
+      method_name = "__bind_#{time.to_i}_#{time.usec}"
+      define_method(method_name, &block)
+      method = instance_method(method_name)
+      remove_method(method_name)
+      method
+    end.bind(object)
+  end
+end
+
+module DatabaseTest
+  def test_against_all_configs(name, options = {}, &block)
+    TEST_CONFIGS.each do |config|
+      self.send(:define_method, :"test_#{name.to_s}_with_#{config}") do
+          silence_stream($stdout) do
+          load_schema(config)
+          load_fixtures options
+        end
+        assert_valid_database_setup options
+        block.bind(self).call
+      end
+    end
+  end
+end
+
+def connection
+  ActiveRecord::Base.connection
+end

data/uninstall.rb

@@ -0,0 +1 @@
+# Uninstall hook code here

metadata

@@ -0,0 +1,191 @@
+--- !ruby/object:Gem::Specification
+name: mysql_big_table_migration
+version: !ruby/object:Gem::Version
+  version: 0.1.2
+platform: ruby
+authors:
+- Mark Woods
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-04-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  type: :runtime
+  name: activerecord
+  prerelease: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  type: :runtime
+  name: activesupport
+  prerelease: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  type: :development
+  name: bundler
+  prerelease: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  type: :development
+  name: fileutils
+  prerelease: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  type: :development
+  name: logger
+  prerelease: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  type: :development
+  name: minitest
+  prerelease: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  type: :development
+  name: mysql
+  prerelease: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  type: :development
+  name: mysql2
+  prerelease: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  type: :development
+  name: rake
+  prerelease: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: |2
+    A Rails plugin that adds methods to ActiveRecord::Migration to allow columns
+    and indexes to be added to and removed from large tables with millions of
+    rows in MySQL, without leaving processes seemingly stalled in state "copy
+    to tmp table".
+email: sai@makeyourlaws.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- CHANGELOG
+- Gemfile
+- MIT-LICENSE
+- README.md
+- Rakefile
+- init.rb
+- install.rb
+- lib/mysql_big_table_migration.rb
+- lib/mysql_big_table_migration/version.rb
+- mysql_big_table_migration.gemspec
+- rails/init.rb
+- tasks/mysql_big_table_migration_tasks.rake
+- test/database.yml
+- test/mysql_big_table_migration_test.rb
+- test/schema.rb
+- test/test_helper.rb
+- uninstall.rb
+homepage: http://github.com/MakeYourLaws/mysql_big_table_migration
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.6
+signing_key: 
+specification_version: 4
+summary: allow columns and indexes to be added to and removed from large tables
+test_files: []