lhm-teak 3.6.0

data/spec/test_helper.rb

@@ -0,0 +1,69 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+if ENV['COV']
+  require 'simplecov'
+  SimpleCov.start
+end
+
+require 'minitest/autorun'
+require 'minitest/spec'
+require 'minitest/mock'
+require 'mocha/minitest'
+require 'after_do'
+require 'byebug'
+require 'pathname'
+require 'lhm'
+
+$project = Pathname.new(File.dirname(__FILE__) + '/..').cleanpath
+$spec = $project.join('spec')
+$fixtures = $spec.join('fixtures')
+
+$db_name = 'test'
+
+require 'active_record'
+require 'mysql2'
+
+logger = Logger.new STDOUT
+logger.level = Logger::WARN
+Lhm.logger = logger
+
+# Want test to be efficient without having to wait the normal value of 120s
+Lhm::SqlRetry::RECONNECT_RETRY_MAX_ITERATION = 4
+
+def without_verbose(&block)
+  old_verbose, $VERBOSE = $VERBOSE, nil
+  yield
+ensure
+  $VERBOSE = old_verbose
+end
+
+def printer
+  printer = Lhm::Printer::Base.new
+
+  def printer.notify(*) ;end
+  def printer.end(*) [] ;end
+
+  printer
+end
+
+def throttler
+  Lhm::Throttler::Time.new(:stride => 100)
+end
+
+def init_test_db
+  db_config = YAML.load_file(File.expand_path(File.dirname(__FILE__)) + '/integration/database.yml')
+  conn = Mysql2::Client.new(
+    :host => '127.0.0.1',
+    :username => db_config['master']['user'],
+    :password => db_config['master']['password'],
+    :port => db_config['master']['port']
+  )
+
+  conn.query("DROP DATABASE IF EXISTS #{$db_name}")
+  conn.query("CREATE DATABASE #{$db_name}")
+end
+
+init_test_db
+
+

data/spec/unit/atomic_switcher_spec.rb

@@ -0,0 +1,29 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require File.expand_path(File.dirname(__FILE__)) + '/unit_helper'
+
+require 'lhm/table'
+require 'lhm/migration'
+require 'lhm/atomic_switcher'
+
+describe Lhm::AtomicSwitcher do
+  include UnitHelper
+
+  before(:each) do
+    @start       = Time.now
+    @origin      = Lhm::Table.new('origin')
+    @destination = Lhm::Table.new('destination')
+    @migration   = Lhm::Migration.new(@origin, @destination, @start)
+    @switcher    = Lhm::AtomicSwitcher.new(@migration, nil)
+  end
+
+  describe 'atomic switch' do
+    it 'should perform a single atomic rename' do
+      value(@switcher.atomic_switch).must_equal(
+        "rename table `origin` to `#{ @migration.archive_name }`, " \
+        '`destination` to `origin`'
+      )
+    end
+  end
+end

data/spec/unit/chunk_finder_spec.rb

@@ -0,0 +1,73 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require File.expand_path(File.dirname(__FILE__)) + '/unit_helper'
+
+describe Lhm::ChunkFinder do
+  before(:each) do
+    @origin = Lhm::Table.new('foo')
+    @destination = Lhm::Table.new('bar')
+    @migration = Lhm::Migration.new(@origin, @destination)
+    @connection = mock()
+  end
+
+  describe '#validate' do
+    describe 'when start is greater than limit' do
+      it 'raises' do
+        assert_raises { Lhm::ChunkFinder.new(@connection, @migration, {start: 2, limit: 1}).validate }
+      end
+    end
+
+    describe 'when start is greater than limit' do
+      it 'does not raise' do
+        Lhm::ChunkFinder.new(@connection, @migration, {start: 1, limit: 2}).validate # does not raise
+      end
+    end
+  end
+
+  describe '#start' do
+    describe 'when initialized with 5' do
+      before(:each) do
+        @instance = Lhm::ChunkFinder.new(@connection, @migration, {start: 5, limit: 6})
+      end
+
+      it 'returns 5' do
+        assert_equal @instance.send(:start), 5
+      end
+    end
+
+    describe 'when initialized with nil and the min(id) is 22' do
+      before(:each) do
+        @connection.expects(:select_value).returns(22)
+        @instance = Lhm::ChunkFinder.new(@migration, @connection, {limit: 6})
+      end
+
+      it 'returns 22' do
+        assert_equal @instance.send(:start), 22
+      end
+    end
+  end
+
+  describe '#limit' do
+    describe 'when initialized with 6' do
+      before(:each) do
+        @instance = Lhm::ChunkFinder.new(@connection, @migration, {start: 5, limit: 6})
+      end
+
+      it 'returns 6' do
+        assert_equal @instance.send(:limit), 6
+      end
+    end
+
+    describe 'when initialized with nil and the max(id) is 33' do
+      before(:each) do
+        @connection.expects(:select_value).returns(33)
+        @instance = Lhm::ChunkFinder.new(@migration, @connection, {start: 5})
+      end
+
+      it 'returns 33' do
+        assert_equal @instance.send(:limit), 33
+      end
+    end
+  end
+end

data/spec/unit/chunk_insert_spec.rb

@@ -0,0 +1,67 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require File.expand_path(File.dirname(__FILE__)) + '/unit_helper'
+
+require 'lhm/chunk_insert'
+require 'lhm/connection'
+
+describe Lhm::ChunkInsert do
+  before(:each) do
+    ar_connection = mock()
+    ar_connection.stubs(:execute).returns([["dummy"]])
+    @connection = Lhm::Connection.new(connection: ar_connection, options: {reconnect_with_consistent_host: false})
+    @origin = Lhm::Table.new('foo')
+    @destination = Lhm::Table.new('bar')
+  end
+
+  describe "#sql" do
+    describe "when migration has no conditions" do
+      before do
+        @migration = Lhm::Migration.new(@origin, @destination)
+      end
+
+      it "uses a simple where clause" do
+        assert_equal(
+          Lhm::ChunkInsert.new(@migration, @connection, 1, 2).send(:sql),
+          "insert ignore into `bar` () select  from `foo` where `foo`.`id` between 1 and 2"
+        )
+      end
+    end
+
+    describe "when migration has a WHERE condition" do
+      before do
+        @migration = Lhm::Migration.new(
+          @origin,
+          @destination,
+          "where foo.created_at > '2013-07-10' or foo.baz = 'quux'"
+        )
+      end
+
+      it "combines the clause with the chunking WHERE condition" do
+        assert_equal(
+          Lhm::ChunkInsert.new(@migration, @connection, 1, 2).send(:sql),
+          "insert ignore into `bar` () select  from `foo` where (foo.created_at > '2013-07-10' or foo.baz = 'quux') and `foo`.`id` between 1 and 2"
+        )
+      end
+    end
+
+    describe "when migration has a WHERE as a proc" do
+      before do
+        @date = Date.today.to_s
+        @migration = Lhm::Migration.new(
+          @origin,
+          @destination,
+          -> { "where foo.created_at > '#{@date}' or foo.baz = 'quux'" }
+        )
+      end
+
+      it "combines the clause with the chunking WHERE condition" do
+        assert_equal(
+          Lhm::ChunkInsert.new(@migration, @connection, 1, 2).send(:sql),
+          "insert ignore into `bar` () select  from `foo` where (foo.created_at > '#{@date}' or foo.baz = 'quux') and `foo`.`id` between 1 and 2"
+        )
+      end
+    end
+  end
+end

data/spec/unit/chunker_spec.rb

@@ -0,0 +1,176 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require File.expand_path(File.dirname(__FILE__)) + '/unit_helper'
+
+require 'lhm/table'
+require 'lhm/migration'
+require 'lhm/chunker'
+require 'lhm/throttler'
+require 'lhm/connection'
+
+describe Lhm::Chunker do
+  include UnitHelper
+
+  EXPECTED_RETRY_FLAGS_CHUNKER = {:should_retry => true, :log_prefix => "Chunker"}
+  EXPECTED_RETRY_FLAGS_CHUNK_INSERT = {:should_retry => true, :log_prefix => "ChunkInsert"}
+
+  before(:each) do
+    @origin = Lhm::Table.new('foo')
+    @destination = Lhm::Table.new('bar')
+    @migration = Lhm::Migration.new(@origin, @destination)
+    @connection = mock()
+    @connection.stubs(:execute).returns([["dummy"]])
+    # This is a poor man's stub
+    @throttler = Object.new
+    def @throttler.run
+      # noop
+    end
+    def @throttler.stride
+      1
+    end
+
+    @chunker = Lhm::Chunker.new(@migration, @connection, :throttler => @throttler,
+                                                         :start     => 1,
+                                                         :limit     => 10)
+  end
+
+  describe '#run' do
+
+    it 'detects the max id to use in the chunk using the stride and use it if it is lower than the limit' do
+      def @throttler.stride
+        5
+      end
+
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 4/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(7)
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 4/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(21)
+      @connection.expects(:update).with(regexp_matches(/between 1 and 7/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
+      @connection.expects(:update).with(regexp_matches(/between 8 and 10/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
+      @connection.expects(:execute).twice.with(regexp_matches(/show warnings/),EXPECTED_RETRY_FLAGS_CHUNKER).returns([])
+
+      @chunker.run
+    end
+
+
+    it 'chunks the result set according to the stride size' do
+      def @throttler.stride
+        2
+      end
+
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(2)
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(4)
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 5 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(6)
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 7 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(8)
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(10)
+
+      @connection.expects(:update).with(regexp_matches(/between 1 and 2/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
+      @connection.expects(:update).with(regexp_matches(/between 3 and 4/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
+      @connection.expects(:update).with(regexp_matches(/between 5 and 6/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
+      @connection.expects(:update).with(regexp_matches(/between 7 and 8/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
+      @connection.expects(:update).with(regexp_matches(/between 9 and 10/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
+
+      @chunker.run
+    end
+
+    it 'handles stride changes during execution' do
+      # roll our own stubbing
+      def @throttler.stride
+        @run_count ||= 0
+        @run_count = @run_count + 1
+        if @run_count > 1
+          3
+        else
+          2
+        end
+      end
+
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(2)
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 2/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(5)
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 2/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(8)
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 2/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(nil)
+
+      @connection.expects(:update).with(regexp_matches(/between 1 and 2/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
+      @connection.expects(:update).with(regexp_matches(/between 3 and 5/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
+      @connection.expects(:update).with(regexp_matches(/between 6 and 8/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
+      @connection.expects(:update).with(regexp_matches(/between 9 and 10/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
+
+      @connection.expects(:execute).twice.with(regexp_matches(/show warnings/),EXPECTED_RETRY_FLAGS_CHUNKER).returns([])
+
+      @chunker.run
+    end
+
+    it 'correctly copies single record tables' do
+      @chunker = Lhm::Chunker.new(@migration, @connection, :throttler => @throttler,
+                                                           :start     => 1,
+                                                           :limit     => 1)
+
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 0/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(nil)
+      @connection.expects(:update).with(regexp_matches(/between 1 and 1/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(1)
+
+      @chunker.run
+    end
+
+    it 'copies the last record of a table, even it is the start of the last chunk' do
+      @chunker = Lhm::Chunker.new(@migration, @connection, :throttler => @throttler,
+                                                           :start     => 2,
+                                                           :limit     => 10)
+      def @throttler.stride
+        2
+      end
+
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 2 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(3)
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 4 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(5)
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(7)
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(9)
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 10 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(nil)
+
+      @connection.expects(:update).with(regexp_matches(/between 2 and 3/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
+      @connection.expects(:update).with(regexp_matches(/between 4 and 5/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
+      @connection.expects(:update).with(regexp_matches(/between 6 and 7/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
+      @connection.expects(:update).with(regexp_matches(/between 8 and 9/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
+      @connection.expects(:update).with(regexp_matches(/between 10 and 10/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(1)
+
+      @chunker.run
+    end
+
+
+    it 'separates filter conditions from chunking conditions' do
+      @chunker = Lhm::Chunker.new(@migration, @connection, :throttler => @throttler,
+                                                           :start     => 1,
+                                                           :limit     => 2)
+      def @throttler.stride
+        2
+      end
+
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(2)
+      @connection.expects(:update).with(regexp_matches(/where \(foo.created_at > '2013-07-10' or foo.baz = 'quux'\) and `foo`/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(1)
+      @connection.expects(:execute).with(regexp_matches(/show warnings/),EXPECTED_RETRY_FLAGS_CHUNKER).returns([])
+
+      def @migration.conditions
+        "where foo.created_at > '2013-07-10' or foo.baz = 'quux'"
+      end
+
+      @chunker.run
+    end
+
+    it "doesn't mess with inner join filters" do
+      @chunker = Lhm::Chunker.new(@migration, @connection, :throttler => @throttler,
+                                                           :start     => 1,
+                                                           :limit     => 2)
+
+      def @throttler.stride
+        2
+      end
+
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(2)
+      @connection.expects(:update).with(regexp_matches(/inner join bar on foo.id = bar.foo_id and/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(1)
+      @connection.expects(:execute).with(regexp_matches(/show warnings/),EXPECTED_RETRY_FLAGS_CHUNKER).returns([])
+
+      def @migration.conditions
+        'inner join bar on foo.id = bar.foo_id'
+      end
+
+      @chunker.run
+    end
+  end
+end

data/spec/unit/connection_spec.rb

@@ -0,0 +1,111 @@
+require 'lhm/connection'
+require 'lhm/proxysql_helper'
+
+describe Lhm::Connection do
+
+  LOCK_WAIT = ActiveRecord::StatementInvalid.new('Lock wait timeout exceeded; try restarting transaction.')
+
+  before(:each) do
+    @logs = StringIO.new
+    Lhm.logger = Logger.new(@logs)
+  end
+
+  it "Should find use calling file as prefix" do
+    ar_connection = mock()
+    ar_connection.stubs(:execute).raises(LOCK_WAIT).then.returns(true)
+    ar_connection.stubs(:active?).returns(true)
+
+    connection = Lhm::Connection.new(connection: ar_connection, options: {
+      retriable: {
+        base_interval: 0
+      }
+    })
+
+    connection.execute("SHOW TABLES", should_retry: true)
+
+    log_messages = @logs.string.split("\n")
+    assert_equal(1, log_messages.length)
+    assert log_messages.first.include?("[ConnectionSpec]")
+  end
+
+  it "#execute should be retried" do
+    ar_connection = mock()
+    ar_connection.stubs(:execute).raises(LOCK_WAIT)
+                 .then.raises(LOCK_WAIT)
+                 .then.returns(true)
+    ar_connection.stubs(:active?).returns(true)
+
+    connection = Lhm::Connection.new(connection: ar_connection, options: {
+      retriable: {
+        base_interval: 0,
+        tries: 3
+      }
+    })
+
+    connection.execute("SHOW TABLES", should_retry: true)
+
+    log_messages = @logs.string.split("\n")
+    assert_equal(2, log_messages.length)
+  end
+
+  it "#update should be retried" do
+    ar_connection = mock()
+    ar_connection.stubs(:update).raises(LOCK_WAIT)
+                 .then.raises(LOCK_WAIT)
+                 .then.returns(1)
+    ar_connection.stubs(:active?).returns(true)
+
+    connection = Lhm::Connection.new(connection: ar_connection, options: {
+      retriable: {
+        base_interval: 0,
+        tries: 3
+      }
+    })
+
+    val = connection.update("SHOW TABLES", should_retry: true)
+
+    log_messages = @logs.string.split("\n")
+    assert_equal val, 1
+    assert_equal(2, log_messages.length)
+  end
+
+  it "#select_value should be retried" do
+    ar_connection = mock()
+    ar_connection.stubs(:select_value).raises(LOCK_WAIT)
+                 .then.raises(LOCK_WAIT)
+                 .then.returns("dummy")
+    ar_connection.stubs(:active?).returns(true)
+
+    connection = Lhm::Connection.new(connection: ar_connection, options: {
+      retriable: {
+        base_interval: 0,
+        tries: 3
+      }
+    })
+
+    val = connection.select_value("SHOW TABLES", should_retry: true)
+
+    log_messages = @logs.string.split("\n")
+    assert_equal val, "dummy"
+    assert_equal(2, log_messages.length)
+  end
+
+  it "Queries should be tagged with ProxySQL tag if reconnect_with_consistent_host is enabled" do
+    ar_connection = mock()
+    ar_connection.expects(:public_send).with(:select_value, "SHOW TABLES #{Lhm::ProxySQLHelper::ANNOTATION}").returns("dummy")
+    ar_connection.stubs(:execute).times(4).returns([["dummy"]])
+    ar_connection.stubs(:active?).returns(true)
+
+    connection = Lhm::Connection.new(connection: ar_connection, options: {
+      reconnect_with_consistent_host: true,
+      retriable: {
+        base_interval: 0,
+        tries: 3
+      }
+    })
+
+    val = connection.select_value("SHOW TABLES", should_retry: true)
+
+    assert_equal val, "dummy"
+  end
+end