lhm-teak 3.6.0

data/spec/integration/chunk_insert_spec.rb

@@ -0,0 +1,30 @@
+require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
+require 'lhm/migration'
+
+describe Lhm::ChunkInsert do
+  include IntegrationHelper
+
+  describe 'insert_and_return_count_of_rows_created' do
+    before(:each) do
+      connect_master!
+      @origin = table_create(:origin)
+      @destination = table_create(:destination)
+      @migration = Lhm::Migration.new(@origin, @destination)
+      execute("insert into origin set id = 1001")
+      @connection = Lhm::Connection.new(connection: connection)
+      @instance = Lhm::ChunkInsert.new(@migration, @connection, 1001, 1001)
+    end
+
+    it "returns the count" do
+      assert_equal 1, @instance.insert_and_return_count_of_rows_created
+    end
+
+    it "inserts the record into the slave" do
+      @instance.insert_and_return_count_of_rows_created
+
+      slave do
+        value(count_all(@destination.name)).must_equal(1)
+      end
+    end
+  end
+end

data/spec/integration/chunker_spec.rb

@@ -0,0 +1,269 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
+require 'lhm/table'
+require 'lhm/migration'
+
+describe Lhm::Chunker do
+  include IntegrationHelper
+
+  before(:each) { connect_master! }
+
+  describe 'copying' do
+    before(:each) do
+      @origin = table_create(:origin)
+      @destination = table_create(:destination)
+      @migration = Lhm::Migration.new(@origin, @destination)
+      @logs = StringIO.new
+      Lhm.logger = Logger.new(@logs)
+    end
+
+    def log_messages
+      @logs.string.split("\n")
+    end
+
+    it 'should copy 1 row from origin to destination even if the id of the single row does not start at 1' do
+      execute("insert into origin set id = 1001 ")
+
+      Lhm::Chunker.new(@migration, connection, {throttler: throttler, printer: printer} ).run
+
+      slave do
+        value(count_all(@destination.name)).must_equal(1)
+      end
+
+    end
+
+    it 'should copy and ignore duplicate primary key' do
+      execute("insert into origin set id = 1001 ")
+      execute("insert into origin set id = 1002 ")
+      execute("insert into destination set id = 1002 ")
+
+      Lhm::Chunker.new(@migration, connection, {throttler: throttler, printer: printer} ).run
+
+      slave do
+        value(count_all(@destination.name)).must_equal(2)
+      end
+    end
+
+    it 'should copy and ignore duplicate composite primary key' do
+      origin = table_create(:composite_primary_key)
+      destination = table_create(:composite_primary_key_dest)
+      migration = Lhm::Migration.new(origin, destination)
+
+      execute("insert into composite_primary_key set id = 1001, shop_id = 1")
+      execute("insert into composite_primary_key set id = 1002, shop_id = 1")
+      execute("insert into composite_primary_key_dest set id = 1002, shop_id = 1")
+
+      Lhm::Chunker.new(migration, connection, {throttler: throttler, printer: printer} ).run
+
+      slave do
+        value(count_all(destination.name)).must_equal(2)
+      end
+    end
+
+    it 'should copy and raise on unexpected warnings' do
+      origin = table_create(:custom_primary_key)
+      destination = table_create(:custom_primary_key_dest)
+      migration = Lhm::Migration.new(origin, destination)
+
+      execute("insert into custom_primary_key set id = 1001, pk = 1")
+      execute("insert into custom_primary_key_dest set id = 1001, pk = 2")
+
+      exception = assert_raises(Lhm::Error) do
+        Lhm::Chunker.new(migration, connection, {raise_on_warnings: true, throttler: throttler, printer: printer} ).run
+      end
+
+      assert_match "Unexpected warning found for inserted row: Duplicate entry '1001' for key 'index_custom_primary_key_on_id'", exception.message
+    end
+
+    it 'should copy and warn on unexpected warnings by default' do
+      origin = table_create(:custom_primary_key)
+      destination = table_create(:custom_primary_key_dest)
+      migration = Lhm::Migration.new(origin, destination)
+
+      execute("insert into custom_primary_key set id = 1001, pk = 1")
+      execute("insert into custom_primary_key_dest set id = 1001, pk = 2")
+
+      Lhm::Chunker.new(migration, connection, {throttler: throttler, printer: printer} ).run
+
+      assert_equal 2, log_messages.length
+      assert log_messages[1].include?("Unexpected warning found for inserted row: Duplicate entry '1001' for key 'index_custom_primary_key_on_id'"), log_messages
+    end
+
+    it 'should log two times for two unexpected warnings' do
+      origin = table_create(:custom_primary_key)
+      destination = table_create(:custom_primary_key_dest)
+      migration = Lhm::Migration.new(origin, destination)
+
+      execute("insert into custom_primary_key set id = 1001, pk = 1")
+      execute("insert into custom_primary_key set id = 1002, pk = 2")
+      execute("insert into custom_primary_key_dest set id = 1001, pk = 3")
+      execute("insert into custom_primary_key_dest set id = 1002, pk = 4")
+
+      Lhm::Chunker.new(migration, connection, {throttler: throttler, printer: printer} ).run
+
+      assert_equal 3, log_messages.length
+      assert log_messages[1].include?("Unexpected warning found for inserted row: Duplicate entry '1001' for key 'index_custom_primary_key_on_id'"), log_messages
+      assert log_messages[2].include?("Unexpected warning found for inserted row: Duplicate entry '1002' for key 'index_custom_primary_key_on_id'"), log_messages
+    end
+
+    it 'should copy and warn on unexpected warnings' do
+      origin = table_create(:custom_primary_key)
+      destination = table_create(:custom_primary_key_dest)
+      migration = Lhm::Migration.new(origin, destination)
+
+      execute("insert into custom_primary_key set id = 1001, pk = 1")
+      execute("insert into custom_primary_key_dest set id = 1001, pk = 2")
+
+      Lhm::Chunker.new(migration, connection, {raise_on_warnings: false, throttler: throttler, printer: printer} ).run
+
+      assert_equal 2, log_messages.length
+      assert log_messages[1].include?("Unexpected warning found for inserted row: Duplicate entry '1001' for key 'index_custom_primary_key_on_id'"), log_messages
+    end
+
+    it 'should create the modified destination, even if the source is empty' do
+      execute("truncate origin ")
+
+      Lhm::Chunker.new(@migration, connection, {throttler: throttler, printer: printer} ).run
+
+      slave do
+        value(count_all(@destination.name)).must_equal(0)
+      end
+
+    end
+
+    it 'should copy 23 rows from origin to destination in one shot, regardless of the value of the id' do
+      23.times { |n| execute("insert into origin set id = '#{ n * n + 23 }'") }
+
+      printer = MiniTest::Mock.new
+      printer.expect(:notify, :return_value, [Integer, Integer])
+      printer.expect(:end, :return_value, [])
+
+      Lhm::Chunker.new(
+        @migration, connection, { throttler: throttler, printer: printer }
+      ).run
+
+      slave do
+        value(count_all(@destination.name)).must_equal(23)
+      end
+
+      printer.verify
+
+    end
+
+    it 'should copy all the records of a table, even if the last chunk starts with the last record of it.' do
+      11.times { |n| execute("insert into origin set id = '#{ n + 1 }'") }
+
+
+      Lhm::Chunker.new(
+        @migration, connection, { throttler: Lhm::Throttler::Time.new(stride: 10), printer: printer }
+      ).run
+
+      slave do
+        value(count_all(@destination.name)).must_equal(11)
+      end
+
+    end
+
+    it 'should copy 23 rows from origin to destination in one shot with slave lag based throttler, regardless of the value of the id' do
+      23.times { |n| execute("insert into origin set id = '#{ 100000 + n * n + 23 }'") }
+
+      printer = MiniTest::Mock.new
+      printer.expect(:notify, :return_value, [Integer, Integer])
+      printer.expect(:end, :return_value, [])
+
+      Lhm::Throttler::Slave.any_instance.stubs(:slave_hosts).returns(['127.0.0.1'])
+      Lhm::Throttler::SlaveLag.any_instance.stubs(:master_slave_hosts).returns(['127.0.0.1'])
+
+      Lhm::Chunker.new(
+        @migration, connection, { throttler: Lhm::Throttler::SlaveLag.new(stride: 100), printer: printer }
+      ).run
+
+      slave do
+        value(count_all(@destination.name)).must_equal(23)
+      end
+
+      printer.verify
+    end
+
+    it 'should throttle work stride based on slave lag' do
+      15.times { |n| execute("insert into origin set id = '#{ (n * n) + 1 }'") }
+
+      printer = mock()
+      printer.expects(:notify).with(instance_of(Integer), instance_of(Integer)).twice
+      printer.expects(:end)
+
+      throttler = Lhm::Throttler::SlaveLag.new(stride: 10, allowed_lag: 0)
+      def throttler.max_current_slave_lag
+        1
+      end
+
+      Lhm::Chunker.new(
+        @migration, connection, { throttler: throttler, printer: printer }
+      ).run
+
+      assert_equal(Lhm::Throttler::SlaveLag::INITIAL_TIMEOUT * 2 * 2, throttler.timeout_seconds)
+
+      slave do
+        value(count_all(@destination.name)).must_equal(15)
+      end
+    end
+
+    it 'should detect a single slave with no lag in the default configuration' do
+      15.times { |n| execute("insert into origin set id = '#{ (n * n) + 1 }'") }
+
+      printer = mock()
+      printer.expects(:notify).with(instance_of(Integer), instance_of(Integer)).twice
+      printer.expects(:verify)
+      printer.expects(:end)
+
+      Lhm::Throttler::Slave.any_instance.stubs(:slave_hosts).returns(['127.0.0.1'])
+      Lhm::Throttler::SlaveLag.any_instance.stubs(:master_slave_hosts).returns(['127.0.0.1'])
+
+      throttler = Lhm::Throttler::SlaveLag.new(stride: 10, allowed_lag: 0)
+
+      if master_slave_mode?
+        def throttler.slave_connection(slave)
+          config = ActiveRecord::Base.connection_pool.db_config.configuration_hash.dup
+          config[:host] = slave
+          config[:port] = 33007
+          ActiveRecord::Base.send('mysql2_connection', config)
+        end
+      end
+
+      Lhm::Chunker.new(
+        @migration, connection, { throttler: throttler, printer: printer }
+      ).run
+
+      assert_equal(Lhm::Throttler::SlaveLag::INITIAL_TIMEOUT, throttler.timeout_seconds)
+      assert_equal(0, throttler.send(:max_current_slave_lag))
+
+      slave do
+        value(count_all(@destination.name)).must_equal(15)
+      end
+
+      printer.verify
+    end
+
+    it 'should abort early if the triggers are removed' do
+      15.times { |n| execute("insert into origin set id = '#{ (n * n) + 1 }'") }
+
+      printer = mock()
+
+      failer = Proc.new { false }
+
+      exception = assert_raises do
+        Lhm::Chunker.new(
+          @migration, connection, { verifier: failer, printer: printer, throttler: throttler }
+        ).run
+      end
+
+      assert_match "Verification failed, aborting early", exception.message
+
+      slave do
+        value(count_all(@destination.name)).must_equal(0)
+      end
+    end
+  end
+end

data/spec/integration/cleanup_spec.rb

@@ -0,0 +1,147 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
+
+describe Lhm, 'cleanup' do
+  include IntegrationHelper
+  before(:each) { connect_master! }
+
+  describe 'changes' do
+    before(:each) do
+      table_create(:users)
+      table_create(:permissions)
+      simulate_failed_migration do
+        Lhm.change_table(:users, :atomic_switch => false) do |t|
+          t.add_column(:logins, "INT(12) DEFAULT '0'")
+          t.add_index(:logins)
+        end
+      end
+      simulate_failed_migration do
+        Lhm.change_table(:permissions, :atomic_switch => false) do |t|
+          t.add_column(:user_id, "INT(12) DEFAULT '0'")
+          t.add_index(:user_id)
+        end
+      end
+    end
+
+    after(:each) do
+      Lhm.cleanup(true)
+    end
+
+    describe 'cleanup' do
+      it 'should show temporary tables' do
+        output = capture_stdout do |logger|
+          Lhm.logger = logger
+          Lhm.cleanup
+        end
+        value(output).must_include('Would drop LHM backup tables')
+        value(output).must_match(/lhma_[0-9_]*_users/)
+        value(output).must_match(/lhma_[0-9_]*_permissions/)
+      end
+
+      it 'should show temporary tables within range' do
+        table = OpenStruct.new(:name => 'users')
+        table_name = Lhm::Migration.new(table, nil, nil, {}, Time.now - 172800).archive_name
+        table_rename(:users, table_name)
+
+        table2 = OpenStruct.new(:name => 'permissions')
+        table_name2 = Lhm::Migration.new(table2, nil, nil, {}, Time.now - 172800).archive_name
+        table_rename(:permissions, table_name2)
+
+        output = capture_stdout do |logger|
+          Lhm.logger = logger
+          Lhm.cleanup false, { :until => Time.now - 86400 }
+        end
+        value(output).must_include('Would drop LHM backup tables')
+        value(output).must_match(/lhma_[0-9_]*_users/)
+        value(output).must_match(/lhma_[0-9_]*_permissions/)
+      end
+
+      it 'should exclude temporary tables outside range' do
+        table = OpenStruct.new(:name => 'users')
+        table_name = Lhm::Migration.new(table, nil, nil, {}, Time.now).archive_name
+        table_rename(:users, table_name)
+
+        table2 = OpenStruct.new(:name => 'permissions')
+        table_name2 = Lhm::Migration.new(table2, nil, nil, {}, Time.now).archive_name
+        table_rename(:permissions, table_name2)
+
+        output = capture_stdout do |logger|
+          Lhm.logger = logger
+          Lhm.cleanup false, { :until => Time.now - 172800 }
+        end
+        value(output).must_include('Would drop LHM backup tables')
+        value(output).wont_match(/lhma_[0-9_]*_users/)
+        value(output).wont_match(/lhma_[0-9_]*_permissions/)
+      end
+
+      it 'should show temporary triggers' do
+        output = capture_stdout do |logger|
+          Lhm.logger = logger
+          Lhm.cleanup
+        end
+        value(output).must_include('Would drop LHM triggers')
+        value(output).must_include('lhmt_ins_users')
+        value(output).must_include('lhmt_del_users')
+        value(output).must_include('lhmt_upd_users')
+        value(output).must_include('lhmt_ins_permissions')
+        value(output).must_include('lhmt_del_permissions')
+        value(output).must_include('lhmt_upd_permissions')
+      end
+
+      it 'should delete temporary tables' do
+        value(Lhm.cleanup(true)).must_equal(true)
+        value(Lhm.cleanup).must_equal(true)
+      end
+
+      it 'outputs deleted tables and triggers' do
+        output = capture_stdout do |logger|
+          Lhm.logger = logger
+          Lhm.cleanup(true)
+        end
+        value(output).must_include('Dropped triggers lhmt_ins_users, lhmt_upd_users, lhmt_del_users, lhmt_ins_permissions, lhmt_upd_permissions, lhmt_del_permissions')
+      end
+    end
+
+    describe 'cleanup_current_run' do
+      it 'should show lhmn table for the specified table only' do
+        table_create(:permissions)
+        table_rename(:permissions, 'lhmn_permissions')
+        output = capture_stdout do |logger|
+          Lhm.logger = logger
+          Lhm.cleanup_current_run(false, 'permissions')
+        end
+
+        value(output).must_include("The following DDLs would be executed:")
+        value(output).must_include("drop trigger if exists lhmt_ins_permissions")
+        value(output).must_include("drop trigger if exists lhmt_upd_permissions")
+        value(output).must_include("drop trigger if exists lhmt_del_permissions")
+      end
+
+      it 'should show temporary triggers for the specified table only' do
+        output = capture_stdout do |logger|
+          Lhm.logger = logger
+          Lhm.cleanup_current_run(false, 'permissions')
+        end
+        value(output).must_include("The following DDLs would be executed:")
+        value(output).must_include("drop trigger if exists lhmt_ins_permissions")
+        value(output).must_include("drop trigger if exists lhmt_upd_permissions")
+        value(output).must_include("drop trigger if exists lhmt_del_permissions")
+      end
+
+      it 'should delete temporary tables and triggers for the specified table only' do
+        assert Lhm.cleanup_current_run(true, 'permissions')
+
+        all_tables = Lhm.connection.select_values('show tables')
+        all_triggers = Lhm.connection.select_values('show triggers')
+
+        refute all_tables.include?('lhmn_permissions')
+        assert all_tables.find { |t| t =~ /lhma_(.*)_users/}
+
+        refute all_triggers.find { |t| t =~ /lhmt_(.*)_permissions/}
+        assert all_triggers.find { |t| t =~ /lhmt_(.*)_users/}
+      end
+    end
+  end
+end

data/spec/integration/database.yml

@@ -0,0 +1,25 @@
+master:
+  host: mysql-1
+  user: root
+  password: password
+  port: 33006
+slave:
+  host: mysql-2
+  user: root
+  password: password
+  port: 33007
+proxysql:
+  host: proxysql
+  user: root
+  password: password
+  port: 33005
+master_toxic:
+  host: toxiproxy
+  user: root
+  password: password
+  port: 22220
+proxysql_toxic:
+  host: toxiproxy
+  user: root
+  password: password
+  port: 22222

data/spec/integration/entangler_spec.rb

@@ -0,0 +1,68 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
+
+require 'lhm/table'
+require 'lhm/migration'
+require 'lhm/entangler'
+require 'lhm/connection'
+
+describe Lhm::Entangler do
+  include IntegrationHelper
+
+  before(:each) { connect_master! }
+
+  describe 'entanglement' do
+    before(:each) do
+      @origin = table_create('origin')
+      @destination = table_create('destination')
+      @migration = Lhm::Migration.new(@origin, @destination)
+      @connection = Lhm::Connection.new(connection: connection)
+      @entangler = Lhm::Entangler.new(@migration, @connection)
+    end
+
+    it 'should replay inserts from origin into destination' do
+      @entangler.run do
+        execute("insert into origin (common) values ('inserted')")
+      end
+
+      slave do
+        value(count(:destination, 'common', 'inserted')).must_equal(1)
+      end
+    end
+
+    it 'should replay deletes from origin into destination' do
+      execute("insert into origin (common) values ('inserted')")
+
+      @entangler.run do
+        execute("delete from origin where common = 'inserted'")
+      end
+
+      slave do
+        value(count(:destination, 'common', 'inserted')).must_equal(0)
+      end
+    end
+
+    it 'should replay updates from origin into destination' do
+      @entangler.run do
+        execute("insert into origin (common) values ('inserted')")
+        execute("update origin set common = 'updated'")
+      end
+
+      slave do
+        value(count(:destination, 'common', 'updated')).must_equal(1)
+      end
+    end
+
+    it 'should remove entanglement' do
+      @entangler.run {}
+
+      execute("insert into origin (common) values ('inserted')")
+
+      slave do
+        value(count(:destination, 'common', 'inserted')).must_equal(0)
+      end
+    end
+  end
+end