zdm 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c4711654e1f0b53c22c8bf17f27c8d89b9667112
+  data.tar.gz: 43df95e14936ae972174b70852cdaad748fb34e2
+SHA512:
+  metadata.gz: 167fdd1568bc0227c89f12f49ce956da2b7cd990b87458a13c1bd5cd96722c8d0e811fad86092f2a3ed7def58ee56ca59e39eb5abaae370924681435c77d364b
+  data.tar.gz: 866bb805124b34d25d104dd6325ba8ade8668569f08c01588bf4a91d9c7a245aac79849e1c7ad4113385fe00a0d025d1a2b6315f751407b152badcd85df2be68

data/.gitignore

@@ -0,0 +1,27 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Environment normalization:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc
+
+*.sh

data/.travis.yml

@@ -0,0 +1,19 @@
+language: ruby
+
+rvm:
+  - 2.3.0
+  - 2.4.0
+
+sudo: false
+
+gemfile:
+  - gemfiles/4.1.gemfile
+  - gemfiles/4.2.gemfile
+  - gemfiles/5.0.gemfile
+
+services:
+  - mysql
+before_install:
+  - mysql -e 'CREATE DATABASE zdm_test DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;'
+
+script: 'bundle exec rake'

data/Appraisals

@@ -0,0 +1,9 @@
+appraise '4.1' do
+  gem 'activerecord', '4.1.15'
+end
+appraise '4.2' do
+  gem 'activerecord', '4.2.8'
+end
+appraise '5.0' do
+  gem 'activerecord', '5.0.1'
+end

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 ITRP
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,18 @@
+# Zero Downtime Migrator
+
+Minimal code to migrate big tables in mysql, mariadb or aurora with zero downtime of the systems.
+Only works with tables that have an auto increment primary key column named `id`.
+
+Read [Facebook's OCS](https://www.facebook.com/note.php?note_id=430801045932) commentary.
+Instead of using outfiles we follow [lhm](https://github.com/soundcloud/lhm)'s approach.
+
+The code is the readme. If you donot grok the code then you really should not use this.
+
+Install
+=======
+
+```
+gem install zdm
+```
+
+[![Build Status](https://travis-ci.org/itrp/zdm.png)](https://travis-ci.org/itrp/zdm)

data/Rakefile

@@ -0,0 +1,11 @@
+require 'rubygems'
+require 'bundler/setup'
+
+require 'bundler/gem_tasks'
+
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+desc 'Run the specs.'
+task default: :spec

data/gemfiles/4.1.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activerecord", "4.1.15"
+
+gemspec :path => "../"

data/gemfiles/4.1.gemfile.lock

@@ -0,0 +1,66 @@
+PATH
+  remote: ..
+  specs:
+    zdm (1.0.0)
+      activerecord (>= 4.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (4.1.15)
+      activesupport (= 4.1.15)
+      builder (~> 3.1)
+    activerecord (4.1.15)
+      activemodel (= 4.1.15)
+      activesupport (= 4.1.15)
+      arel (~> 5.0.0)
+    activesupport (4.1.15)
+      i18n (~> 0.6, >= 0.6.9)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.1)
+      tzinfo (~> 1.1)
+    appraisal (2.1.0)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    arel (5.0.1.20140414130214)
+    builder (3.2.3)
+    diff-lcs (1.3)
+    i18n (0.8.1)
+    json (1.8.6)
+    minitest (5.10.1)
+    mysql2 (0.3.20)
+    rake (12.0.0)
+    rspec (3.5.0)
+      rspec-core (~> 3.5.0)
+      rspec-expectations (~> 3.5.0)
+      rspec-mocks (~> 3.5.0)
+    rspec-core (3.5.4)
+      rspec-support (~> 3.5.0)
+    rspec-expectations (3.5.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.5.0)
+    rspec-mocks (3.5.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.5.0)
+    rspec-support (3.5.0)
+    thor (0.19.4)
+    thread_safe (0.3.6)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activerecord (= 4.1.15)
+  appraisal
+  bundler (~> 1)
+  mysql2
+  rake
+  rspec
+  zdm!
+
+BUNDLED WITH
+   1.14.5

data/gemfiles/4.2.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activerecord", "4.2.8"
+
+gemspec :path => "../"

data/gemfiles/4.2.gemfile.lock

@@ -0,0 +1,64 @@
+PATH
+  remote: ..
+  specs:
+    zdm (1.0.0)
+      activerecord (>= 4.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (4.2.8)
+      activesupport (= 4.2.8)
+      builder (~> 3.1)
+    activerecord (4.2.8)
+      activemodel (= 4.2.8)
+      activesupport (= 4.2.8)
+      arel (~> 6.0)
+    activesupport (4.2.8)
+      i18n (~> 0.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
+    appraisal (2.1.0)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    arel (6.0.4)
+    builder (3.2.3)
+    diff-lcs (1.3)
+    i18n (0.8.1)
+    minitest (5.10.1)
+    mysql2 (0.4.5)
+    rake (12.0.0)
+    rspec (3.5.0)
+      rspec-core (~> 3.5.0)
+      rspec-expectations (~> 3.5.0)
+      rspec-mocks (~> 3.5.0)
+    rspec-core (3.5.4)
+      rspec-support (~> 3.5.0)
+    rspec-expectations (3.5.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.5.0)
+    rspec-mocks (3.5.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.5.0)
+    rspec-support (3.5.0)
+    thor (0.19.4)
+    thread_safe (0.3.6)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activerecord (= 4.2.8)
+  appraisal
+  bundler (~> 1)
+  mysql2
+  rake
+  rspec
+  zdm!
+
+BUNDLED WITH
+   1.14.5

data/gemfiles/5.0.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activerecord", "5.0.1"
+
+gemspec :path => "../"

data/gemfiles/5.0.gemfile.lock

@@ -0,0 +1,63 @@
+PATH
+  remote: ..
+  specs:
+    zdm (1.0.0)
+      activerecord (>= 4.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (5.0.1)
+      activesupport (= 5.0.1)
+    activerecord (5.0.1)
+      activemodel (= 5.0.1)
+      activesupport (= 5.0.1)
+      arel (~> 7.0)
+    activesupport (5.0.1)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (~> 0.7)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+    appraisal (2.1.0)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    arel (7.1.4)
+    concurrent-ruby (1.0.4)
+    diff-lcs (1.3)
+    i18n (0.8.1)
+    minitest (5.10.1)
+    mysql2 (0.4.5)
+    rake (12.0.0)
+    rspec (3.5.0)
+      rspec-core (~> 3.5.0)
+      rspec-expectations (~> 3.5.0)
+      rspec-mocks (~> 3.5.0)
+    rspec-core (3.5.4)
+      rspec-support (~> 3.5.0)
+    rspec-expectations (3.5.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.5.0)
+    rspec-mocks (3.5.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.5.0)
+    rspec-support (3.5.0)
+    thor (0.19.4)
+    thread_safe (0.3.6)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activerecord (= 5.0.1)
+  appraisal
+  bundler (~> 1)
+  mysql2
+  rake
+  rspec
+  zdm!
+
+BUNDLED WITH
+   1.14.5

data/lib/version.rb

@@ -0,0 +1,3 @@
+module Zdm
+  VERSION = '1.0.0'
+end

data/lib/zdm.rb

@@ -0,0 +1,268 @@
+module Zdm
+  require 'version'
+
+  class << self
+    attr_accessor :io
+
+    def change_table(name, &block)
+      table = Table.new(name)
+      yield table
+      Migrator.new(table).migrate!
+      cleanup if defined?(Rails) && Rails.env.development?
+    end
+
+    def cleanup(before: nil)
+      conn = ActiveRecord::Base.connection
+      zdm_tables = conn.send(tables_method).select { |name| name.starts_with?('zdm_') }
+      zdm_tables.each { |name| Migrator.new(Table.new(name.sub(/^zdm_/, ''))).cleanup }
+
+      zdm_archive_tables = conn.send(tables_method).select { |name| name.starts_with?('zdma_') }
+      if before
+        zdm_archive_tables.select! { |table|
+          Time.strptime(table, 'zdma_%Y%m%d_%H%M%S%N') <= before
+        }
+      end
+      zdm_archive_tables.each { |name| conn.execute('DROP TABLE `%s`' % name) }
+    end
+
+    def tables_method
+      ActiveRecord.version.to_s =~ /^5/ ? :data_sources : :tables
+    end
+  end
+
+  class Table
+    attr_reader :origin, :copy, :archive, :statements
+
+    def initialize(name)
+      @origin = name
+      @copy = "zdm_#{name}"
+      @archive = "zdma_#{Time.now.strftime("%Y%m%d_%H%M%S%N")}_#{name}"[0..64]
+      @statements = []
+    end
+
+    def ddl(statement)
+      @statements << statement
+    end
+
+    def alter(definition)
+      ddl('ALTER TABLE `%s` %s' % [@copy, definition])
+    end
+
+    def add_column(name, definition)
+      ddl('ALTER TABLE `%s` ADD COLUMN `%s` %s' % [@copy, name, definition])
+    end
+
+    def change_column(name, definition)
+      ddl('ALTER TABLE `%s` MODIFY COLUMN `%s` %s' % [@copy, name, definition])
+    end
+
+    def remove_column(name)
+      ddl('ALTER TABLE `%s` DROP `%s`' % [@copy, name])
+    end
+
+    def rename_column(old_name, new_name)
+      raise "Unsupported: you must first run a migration adding the column `#{new_name}`, deploy the code live, then run another migration at a later time to remove the column `#{old_name}`"
+    end
+  end
+
+  class Migrator
+    attr_reader :table
+
+    def initialize(table)
+      @table = table
+    end
+
+    def migrate!
+      validate
+      set_session_lock_wait_timeouts
+      cleanup
+      create_destination_table
+      drop_destination_indexes
+      apply_ddl_statements
+      create_triggers
+      batched_copy
+      create_destination_indexes
+      atomic_switcharoo!
+    ensure
+      cleanup
+    end
+
+    def cleanup
+      drop_triggers
+      execute('DROP TABLE IF EXISTS `%s`' % table.copy)
+    end
+
+    private
+
+      def connection
+        ActiveRecord::Base.connection
+      end
+
+      def execute(stmt)
+        connection.execute(stmt)
+      end
+
+      def columns(table)
+        connection.columns(table).map(&:name)
+      end
+
+      def common_columns
+        @common_columns ||= (columns(table.origin) & columns(table.copy))
+      end
+
+      def validate
+        unless connection.columns(table.origin).detect {|c| c.name == 'id'}&.extra == 'auto_increment'
+          raise 'Cannot migrate table `%s`, missing auto increment primary key `id`' % table.origin
+        end
+      end
+
+      LOCK_WAIT_TIMEOUT_DELTA = -2  # seconds
+      def set_session_lock_wait_timeouts
+        timeout = connection.select_one("SHOW GLOBAL VARIABLES LIKE 'innodb_lock_wait_timeout'")
+        if timeout
+          execute('SET SESSION innodb_lock_wait_timeout=%d' % (timeout['Value'].to_i + LOCK_WAIT_TIMEOUT_DELTA))
+        end
+      end
+
+      def create_destination_table
+        execute('CREATE TABLE `%s` LIKE `%s`' % [table.copy, table.origin])
+      end
+
+      def apply_ddl_statements
+        table.statements.each { |statement| execute(statement) }
+      end
+
+      def atomic_switcharoo!
+        execute('RENAME TABLE `%s` to `%s`, `%s` to `%s`' % [table.origin, table.archive, table.copy, table.origin])
+      end
+
+      def create_triggers
+        create_delete_trigger
+        create_insert_trigger
+        create_update_trigger
+      end
+
+      def create_delete_trigger
+        execute(<<-SQL.squish)
+          CREATE TRIGGER `#{trigger_name(:del)}`
+          AFTER DELETE ON `#{table.origin}` FOR EACH ROW
+          DELETE IGNORE FROM `#{table.copy}` WHERE `#{table.copy}`.`id` = `OLD`.`id`
+        SQL
+      end
+
+      def create_insert_trigger
+        execute(<<-SQL.squish)
+          CREATE TRIGGER `#{trigger_name(:ins)}`
+          AFTER INSERT ON `#{table.origin}` FOR EACH ROW
+          REPLACE INTO `#{table.copy}` SET #{trigger_column_setters}
+        SQL
+      end
+
+      def create_update_trigger
+        execute(<<-SQL.squish)
+          CREATE TRIGGER `#{trigger_name(:upd)}`
+          AFTER UPDATE ON `#{table.origin}` FOR EACH ROW
+          REPLACE INTO `#{table.copy}` SET #{trigger_column_setters}
+        SQL
+      end
+
+      def trigger_column_setters
+        common_columns.map { |name| "`#{name}`=`NEW`.`#{name}`"}.join(', ')
+      end
+
+      def drop_triggers
+        execute('DROP TRIGGER IF EXISTS `%s`' % trigger_name(:del))
+        execute('DROP TRIGGER IF EXISTS `%s`' % trigger_name(:ins))
+        execute('DROP TRIGGER IF EXISTS `%s`' % trigger_name(:upd))
+      end
+
+      def trigger_name(trigger_type)
+        "zdmt_#{trigger_type}_#{table.origin}"[0...64]
+      end
+
+      # Drop indexes to speed up batched_copy
+      def drop_destination_indexes
+        @indexes = connection.indexes(table.copy).reject(&:unique)
+        @indexes.each do |index_def|
+          execute('ALTER TABLE `%s` DROP INDEX `%s`' % [table.copy, index_def.name])
+        end
+      end
+
+      # Recreate the indexes previously dropped
+      def create_destination_indexes
+        @indexes.each do |index_def|
+          opts = { name: index_def.name, using: index_def.using }
+          if index_def.lengths.compact.any?
+            opts[:length] = Hash[index_def.columns.map.with_index { |col, idx| [col, index_def.lengths[idx]] }]
+          end
+          connection.add_index(table.copy, index_def.columns, opts)
+        end
+      end
+
+      BATCH_SIZE = 40_000
+      DECREASE_THROTTLER = 4  # seconds
+      DECREASE_SIZE = 5_000
+      MIN_BATCH_SIZE = 10_000
+      PROGRESS_EVERY = 30  # seconds
+      def batched_copy
+        min = connection.select_value('SELECT MIN(`id`) FROM %s' % table.origin)
+        return unless min
+
+        max = connection.select_value('SELECT MAX(`id`) FROM %s' % table.origin)
+        todo = max - min + 1
+
+        insert_columns = common_columns.map {|c| "`#{c}`"}.join(', ')
+        select_columns = common_columns.map {|c| "`#{table.origin}`.`#{c}`"}.join(', ')
+
+        batch_size = BATCH_SIZE
+        batch_end = min - 1
+        start_time = last_progress = Time.now
+        while true
+          batch_start = batch_end + 1
+          batch_end = [batch_start + batch_size - 1, max].min
+          start_batch_time = Time.now
+
+          execute(<<-SQL.squish)
+            INSERT IGNORE INTO `#{table.copy}` (#{insert_columns})
+            SELECT #{select_columns}
+            FROM `#{table.origin}`
+            WHERE `#{table.origin}`.`id` BETWEEN #{batch_start} AND #{batch_end}
+          SQL
+
+          if $exit
+            write('Received SIGTERM, exiting...')
+            cleanup
+            exit 1
+          end
+
+          # The end!
+          break if batch_end >= max
+
+          # Throttle
+          current_time = Time.now
+          if (current_time - start_batch_time) > DECREASE_THROTTLER
+            batch_size = [(batch_size - DECREASE_SIZE).to_i, MIN_BATCH_SIZE].max
+          end
+
+          # Periodically show progress
+          if (current_time - last_progress) >= PROGRESS_EVERY
+            last_progress = current_time
+            done = batch_end - min + 1
+            write("%.2f%% (#{done}/#{todo})" % (done.to_f / todo * 100.0))
+          end
+        end
+
+        duration = Time.now - start_time
+        duration = (duration < 2*60) ? "#{duration.to_i} secs" : "#{(duration / 60).to_i} mins"
+        write("Completed (#{duration})")
+      end
+
+      def write(msg)
+        return if Zdm.io == false
+        io = Zdm.io || $stderr
+        io.puts("#{table.origin}: #{msg}")
+        io.flush
+      end
+  end
+end
+trap('TERM') { $exit = true }

data/spec/database.yml

@@ -0,0 +1,21 @@
+test:
+  adapter: mysql2
+  encoding: utf8mb4
+  charset: utf8mb4
+  collation: utf8mb4_unicode_ci
+  host: <%= ENV['DB_HOST'] %>
+  port: <%= ENV['DB_PORT'] %>
+  username: <%= ENV['DB_USERNAME'] || 'travis' %>
+  password: <%= ENV['DB_PASSWORD'] %>
+  database: <%= ENV['DB_DATABASE'] || 'zdm_test' %>
+  sslkey: <%= ENV['DB_SSLKEY'] %>
+  sslcert: <%= ENV['DB_SSLCERT'] %>
+  sslca: <%= ENV['DB_SSLCA'] %>
+  sslcapath: <%= ENV['DB_SSLCAPATH'] %>
+  sslcipher: <%= ENV['DB_SSLCIPHER'] %>
+  sslverify: <%= ENV['DB_SSLVERIFY'] || false %>
+  strict: false
+  variables:
+    sql_mode: 'NO_ENGINE_SUBSTITUTION'
+    character_set_connection: utf8mb4
+    collation_connection: utf8mb4_unicode_ci

data/spec/spec_helper.rb

@@ -0,0 +1,38 @@
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/../lib'
+$LOAD_PATH.unshift File.dirname(__FILE__)
+
+$stderr.puts("Running Specs using Ruby v#{RUBY_VERSION}")
+
+require 'rspec'
+require 'logger'
+require 'zdm'
+require 'active_record'
+require 'yaml'
+require 'erb'
+
+# require 'rspec/support'
+# RSpec::Support.require_rspec_support "object_formatter"
+# RSpec::Support::ObjectFormatter.default_instance.max_formatted_output_length = nil
+
+config = YAML::load(ERB.new(IO.read(File.dirname(__FILE__) + '/database.yml')).result)
+ActiveRecord::Base.establish_connection(config['test'])
+
+ActiveRecord::Schema.define version: 0 do
+  create_table :people, force: true do |t|
+    t.integer :account_id
+    t.string :name, limit: 30
+    t.string :code
+    t.datetime :created_at
+  end
+  add_index(:people, :name, unique: true)
+  add_index(:people, [:account_id, :code], length: {account_id: nil, code: 191})
+
+  create_table :people_teams, id: false, force: true do |t|
+    t.integer :team_id, null: false
+    t.integer :person_id, null: false
+  end
+end
+
+ActiveRecord::Base.connection.execute(%[INSERT INTO people(account_id, name, code, created_at) VALUES (10,'foo','bar','2017-03-01 23:59:59')])
+ActiveRecord::Base.connection.execute(%[INSERT INTO people(account_id, name, code, created_at) VALUES (20,'foo2','bar2','2017-03-02 23:59:59')])
+

data/spec/zdm_spec.rb

@@ -0,0 +1,88 @@
+require 'spec_helper'
+
+describe Zdm do
+
+  before(:example) {
+    Zdm.io = false
+    Zdm.cleanup
+  }
+
+  it 'requires an autoincrement primary key `id` field' do
+    expect{Zdm.change_table(:people_teams) {}}.to raise_error('Cannot migrate table `people_teams`, missing auto increment primary key `id`')
+  end
+
+  it 'sends output to stderr' do
+    Zdm.io = nil
+    filename = "test_stderr.#{$$}.log"
+    at_exit { File.unlink(filename) rescue nil }
+    orig_err = STDERR.dup
+    STDERR.reopen(filename, 'a')
+    STDERR.sync = true
+    begin
+      Zdm.change_table(:people) {}
+      expect(File.read(filename).strip).to eq('people: Completed (0 secs)')
+    ensure
+      STDERR.reopen(orig_err)
+    end
+  end
+
+  it 'migrates live tables' do
+    Zdm.change_table(:people) do |m|
+      m.alter("DEFAULT CHARACTER SET utf8 COLLATE utf8_unicode_ci")
+      m.add_column('test', "varchar(32) DEFAULT 'foo'")
+      m.change_column('name', 'varchar(99) NOT NULL')
+    end
+
+    conn = ActiveRecord::Base.connection
+    stmt = conn.select_rows('show create table people')[0][1]
+    expect(stmt.squish).to eq(<<-EOS.squish)
+      CREATE TABLE `people` (
+        `id` int(11) NOT NULL AUTO_INCREMENT,
+        `account_id` int(11) DEFAULT NULL,
+        `name` varchar(99) COLLATE utf8_unicode_ci NOT NULL,
+        `code` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+        `created_at` datetime DEFAULT NULL,
+        `test` varchar(32) COLLATE utf8_unicode_ci DEFAULT 'foo',
+        PRIMARY KEY (`id`), UNIQUE KEY `index_people_on_name` (`name`),
+        KEY `index_people_on_account_id_and_code` (`account_id`,`code`(191)) USING BTREE
+      ) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+    EOS
+
+    archive_tables = conn.send(Zdm.tables_method).select { |name| name.starts_with?('zdma_') }
+    expect(archive_tables.length).to eq(1)
+    rows = conn.select_rows("SELECT * FROM #{archive_tables[0]}")
+    expect(rows).to eq([
+      [1, 10, 'foo', 'bar', '2017-03-01 23:59:59 UTC'],
+      [2, 20, 'foo2', 'bar2', '2017-03-02 23:59:59 UTC']
+    ])
+
+    rows = conn.select_rows("SELECT * FROM `people`")
+    expect(rows).to eq([
+      [1, 10, 'foo', 'bar', '2017-03-01 23:59:59 UTC', 'foo'],
+      [2, 20, 'foo2', 'bar2', '2017-03-02 23:59:59 UTC', 'foo']
+    ])
+
+    Zdm.change_table(:people) do |m|
+      m.alter("DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci")
+      m.remove_column('test')
+      m.change_column('name', 'varchar(30)')
+    end
+
+    stmt = conn.select_rows('show create table people')[0][1]
+    expect(stmt.squish).to eq(<<-EOS.squish)
+        CREATE TABLE `people` (
+          `id` int(11) NOT NULL AUTO_INCREMENT,
+          `account_id` int(11) DEFAULT NULL,
+          `name` varchar(30) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+          `code` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+          `created_at` datetime DEFAULT NULL, PRIMARY KEY (`id`),
+          UNIQUE KEY `index_people_on_name` (`name`),
+          KEY `index_people_on_account_id_and_code` (`account_id`,`code`(191)) USING BTREE
+        ) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
+      EOS
+
+    archive_tables = conn.send(Zdm.tables_method).select { |name| name.starts_with?('zdma_') }
+    expect(archive_tables.length).to eq(2)
+  end
+
+end

data/zdm.gemspec

@@ -0,0 +1,26 @@
+$LOAD_PATH.push File.expand_path('../lib', __FILE__)
+require 'version'
+
+Gem::Specification.new do |s|
+  s.name          = 'zdm'
+  s.version       = Zdm::VERSION
+  s.authors       = ['ITRP Institute, Inc.']
+  s.email         = ['support@itrp.com']
+  s.description   = %q{Zero Downtime Migrator of mysql compatible databases}
+  s.summary       = %q{Zero Downtime Migrator for mysql in ruby}
+  s.homepage      = 'https://github.com/itrp/zdm'
+  s.license       = 'MIT'
+
+  s.files         = `git ls-files`.split($/)
+  s.executables   = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  s.test_files    = s.files.grep(%r{^(test|spec|features)/})
+  s.require_paths = ['lib']
+
+  s.add_dependency 'activerecord', '>= 4.0'
+
+  s.add_development_dependency 'bundler', '~> 1'
+  s.add_development_dependency 'rake'
+  s.add_development_dependency 'rspec'
+  s.add_development_dependency 'mysql2'
+  s.add_development_dependency 'appraisal'
+end

metadata

@@ -0,0 +1,150 @@
+--- !ruby/object:Gem::Specification
+name: zdm
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- ITRP Institute, Inc.
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-03-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: appraisal
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Zero Downtime Migrator of mysql compatible databases
+email:
+- support@itrp.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Appraisals
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- gemfiles/4.1.gemfile
+- gemfiles/4.1.gemfile.lock
+- gemfiles/4.2.gemfile
+- gemfiles/4.2.gemfile.lock
+- gemfiles/5.0.gemfile
+- gemfiles/5.0.gemfile.lock
+- lib/version.rb
+- lib/zdm.rb
+- spec/database.yml
+- spec/spec_helper.rb
+- spec/zdm_spec.rb
+- zdm.gemspec
+homepage: https://github.com/itrp/zdm
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.8
+signing_key: 
+specification_version: 4
+summary: Zero Downtime Migrator for mysql in ruby
+test_files:
+- spec/database.yml
+- spec/spec_helper.rb
+- spec/zdm_spec.rb