lhm-teak 3.6.0

data/spec/integration/locked_switcher_spec.rb

@@ -0,0 +1,50 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
+
+require 'lhm/table'
+require 'lhm/migration'
+require 'lhm/locked_switcher'
+
+describe Lhm::LockedSwitcher do
+  include IntegrationHelper
+
+  before(:each) do
+    connect_master!
+    @old_logger = Lhm.logger
+    Lhm.logger = Logger.new('/dev/null')
+  end
+
+  after(:each) do
+    Lhm.logger = @old_logger
+  end
+
+  describe 'switching' do
+    before(:each) do
+      @origin = table_create('origin')
+      @destination = table_create('destination')
+      @migration = Lhm::Migration.new(@origin, @destination)
+    end
+
+    it 'rename origin to archive' do
+      switcher = Lhm::LockedSwitcher.new(@migration, connection)
+      switcher.run
+
+      slave do
+        value(data_source_exists?(@origin)).must_equal true
+        value(table_read(@migration.archive_name).columns.keys).must_include 'origin'
+      end
+    end
+
+    it 'rename destination to origin' do
+      switcher = Lhm::LockedSwitcher.new(@migration, connection)
+      switcher.run
+
+      slave do
+        value(data_source_exists?(@destination)).must_equal false
+        value(table_read(@origin.name).columns.keys).must_include 'destination'
+      end
+    end
+  end
+end

data/spec/integration/proxysql_spec.rb

@@ -0,0 +1,34 @@
+describe "ProxySQL integration" do
+  it "Should contact the writer" do
+    conn = Mysql2::Client.new(
+      host: '127.0.0.1',
+      username: "writer",
+      password: "password",
+      port: "33005",
+      )
+
+    assert_equal conn.query("SELECT @@global.hostname as host").each.first["host"], "mysql-1"
+  end
+
+  it "Should contact the reader" do
+    conn = Mysql2::Client.new(
+      host: '127.0.0.1',
+      username: "reader",
+      password: "password",
+      port: "33005",
+      )
+
+    assert_equal conn.query("SELECT @@global.hostname as host").each.first["host"], "mysql-2"
+  end
+
+  it "Should override default hostgroup from user if rule matches" do
+    conn = Mysql2::Client.new(
+      host: '127.0.0.1',
+      username: "reader",
+      password: "password",
+      port: "33005",
+      )
+
+    assert_equal conn.query("SELECT @@global.hostname as host #{Lhm::ProxySQLHelper::ANNOTATION}").each.first["host"], "mysql-1"
+  end
+end

data/spec/integration/sql_retry/db_connection_helper.rb

@@ -0,0 +1,52 @@
+require 'yaml'
+require 'mysql2'
+
+class DBConnectionHelper
+
+  DATABASE_CONFIG_FILE = "database.yml"
+
+  class << self
+    def db_config
+      @db_config ||= YAML.load_file(File.expand_path(File.dirname(__FILE__)) + "/../#{DATABASE_CONFIG_FILE}")
+    end
+
+    def new_mysql_connection(role = :master, with_data = false, toxic = false)
+
+      key = role.to_s + toxic_postfix(toxic)
+
+      conn = ActiveRecord::Base.establish_connection(
+        :host => '127.0.0.1',
+        :adapter => "mysql2",
+        :username => db_config[key]['user'],
+        :password => db_config[key]['password'],
+        :database => test_db_name,
+        :port => db_config[key]['port']
+      )
+      conn = conn.connection
+      init_with_dummy_data(conn) if with_data
+      conn
+    end
+
+    def toxic_postfix(toxic)
+      toxic ? "_toxic" : ""
+    end
+
+    def test_db_name
+      @test_db_name ||= "test"
+    end
+
+    def test_table_name
+      @test_table_name ||= "test"
+    end
+
+    def init_with_dummy_data(conn)
+      conn.execute("DROP TABLE IF EXISTS #{test_table_name} ")
+      conn.execute("CREATE TABLE #{test_table_name} (id int)")
+
+      1.upto(9) do |i|
+        query = "INSERT INTO #{test_table_name} (id) VALUE (#{i})"
+        conn.execute(query)
+      end
+    end
+  end
+end

data/spec/integration/sql_retry/lock_wait_spec.rb

@@ -0,0 +1,127 @@
+require 'minitest/autorun'
+require 'mysql2'
+require 'integration/sql_retry/lock_wait_timeout_test_helper'
+require 'lhm'
+
+describe Lhm::SqlRetry do
+  before(:each) do
+    @old_logger = Lhm.logger
+    @logger = StringIO.new
+    Lhm.logger = Logger.new(@logger)
+
+    @helper = LockWaitTimeoutTestHelper.new(
+      lock_duration: 5,
+      innodb_lock_wait_timeout: 2
+    )
+
+    @helper.create_table_to_lock
+
+    # Start a thread to hold a lock on the table
+    @locked_record_id = @helper.hold_lock
+
+    # Assert our pre-conditions
+    assert_equal 2, @helper.record_count
+
+    Mysql2::Client.any_instance.stubs(:active?).returns(true)
+  end
+
+  after(:each) do
+    # Restore default logger
+    Lhm.logger = @old_logger
+  end
+
+  # This is the control test case. It shows that when Lhm::SqlRetry is not used,
+  # a lock wait timeout exceeded error is raised.
+  it "does nothing to prevent exceptions, when not used" do
+    puts ""
+    puts "***The output you see below is OK so long as the test passes.***"
+    puts "*" * 64
+    # Start a thread to retry, once the lock is held, execute the block
+    @helper.with_waiting_lock do |waiting_connection|
+      @helper.insert_records_at_ids(waiting_connection, [@locked_record_id])
+    end
+
+    exception = assert_raises { @helper.trigger_wait_lock }
+
+    assert_match /Lock wait timeout exceeded; try restarting transaction/, exception.message
+    assert_equal Mysql2::Error::TimeoutError, exception.class
+
+    assert_equal 2, @helper.record_count # no records inserted
+    puts "*" * 64
+  end
+
+  # This is test demonstrating the happy path: a well configured retry
+  # tuned to the locks it encounters.
+  it "successfully executes the SQL despite the errors encountered" do
+    # Start a thread to retry, once the lock is held, execute the block
+    @helper.with_waiting_lock do |waiting_connection|
+      sql_retry = Lhm::SqlRetry.new(waiting_connection, retry_options: {
+        base_interval: 0.2, # first retry after 200ms
+        multiplier: 1, # subsequent retries wait 1x longer than first retry (no change)
+        tries: 3, # we only need 3 tries (including the first) for the scenario described below
+        rand_factor: 0 # do not introduce randomness to wait timer
+      }, reconnect_with_consistent_host: false)
+
+      # RetryTestHelper is configured to hold lock for 5 seconds and timeout after 2 seconds.
+      # Therefore the sequence of events will be:
+      # 0s:   first insert query is started while lock is held
+      # 2s:   first timeout error will occur, SqlRetry is configured to wait 200ms after this
+      # 2.2s: second insert query is started while lock is held
+      # 4.2s: second timeout error will occur, SqlRetry is configured to wait 200ms after this
+      # 4.4s: third insert query is started while lock is held
+      # 5s:   lock is released, insert successful no further retries needed
+      sql_retry.with_retries do |retriable_connection|
+        @helper.insert_records_at_ids(retriable_connection, [@locked_record_id])
+      end
+    end
+
+    @helper.trigger_wait_lock
+
+    assert_equal 3, @helper.record_count # records inserted successfully despite lock
+
+    logs = @logger.string.split("\n")
+    assert_equal 2, logs.length
+
+    assert logs.first.include?("Mysql2::Error::TimeoutError: 'Lock wait timeout exceeded; try restarting transaction' - 1 tries")
+    assert logs.first.include?("0.2 seconds until the next try")
+
+    assert logs.last.include?("Mysql2::Error::TimeoutError: 'Lock wait timeout exceeded; try restarting transaction' - 2 tries")
+    assert logs.last.include?("0.2 seconds until the next try")
+  end
+
+  # This is test demonstrating the sad configuration path: it shows
+  # that when the retries are not tuned to the locks encountered,
+  # retries are not effective.
+  it "fails to retry enough to overcome the timeout" do
+    puts ""
+    puts "***The output you see below is OK so long as the test passes.***"
+    puts "*" * 64
+    # Start a thread to retry, once the lock is held, execute the block
+    @helper.with_waiting_lock do |waiting_connection|
+      sql_retry = Lhm::SqlRetry.new(waiting_connection, retry_options: {
+        base_interval: 0.2, # first retry after 200ms
+        multiplier: 1, # subsequent retries wait 1x longer than first retry (no change)
+        tries: 2, # we need 3 tries (including the first) for the scenario described below, but we only get two...we will fail
+        rand_factor: 0 # do not introduce randomness to wait timer
+      }, reconnect_with_consistent_host: false)
+
+      # RetryTestHelper is configured to hold lock for 5 seconds and timeout after 2 seconds.
+      # Therefore the sequence of events will be:
+      # 0s:   first insert query is started while lock is held
+      # 2s:   first timeout error will occur, SqlRetry is configured to wait 200ms after this
+      # 2.2s: second insert query is started while lock is held
+      # 4.2s: second timeout error will occur, SqlRetry is configured to only try twice, so we fail here
+      sql_retry.with_retries do |retriable_connection|
+        @helper.insert_records_at_ids(retriable_connection, [@locked_record_id])
+      end
+    end
+
+    exception = assert_raises { @helper.trigger_wait_lock }
+
+    assert_match /Lock wait timeout exceeded; try restarting transaction/, exception.message
+    assert_equal Mysql2::Error::TimeoutError, exception.class
+
+    assert_equal 2, @helper.record_count # no records inserted
+    puts "*" * 64
+  end
+end

data/spec/integration/sql_retry/lock_wait_timeout_test_helper.rb

@@ -0,0 +1,114 @@
+require 'integration/integration_helper'
+
+class LockWaitTimeoutTestHelper
+
+  def initialize(lock_duration:, innodb_lock_wait_timeout:)
+    # This connection will be used exclusively to setup the test,
+    # assert pre-conditions and assert post-conditions.
+    # We choose to use a `Mysql2::Client` connection instead of
+    # `ActiveRecord::Base.establish_connection` because of AR's connection
+    # pool which forces thread syncronization. In this test,
+    # we want to intentionally create a lock to test retries,
+    # so that is an anti-feature.
+    @main_conn = new_mysql_connection
+
+    @lock_duration = lock_duration
+
+    # While implementing this, I discovered that MySQL seems to have an off-by-one
+    # bug with the innodb_lock_wait_timeout. If you ask it to wait 2 seconds, it will wait 3.
+    # In order to avoid surprisingly the user, let's account for that here, but also
+    # guard against a case where we go below 1, the minimum value.
+    raise ArgumentError, "innodb_lock_wait_timeout must be greater than or equal to 2" unless innodb_lock_wait_timeout >= 2
+    raise ArgumentError, "innodb_lock_wait_timeout must be an integer" if innodb_lock_wait_timeout.class != Integer
+    @innodb_lock_wait_timeout = innodb_lock_wait_timeout - 1
+
+    @threads = []
+    @queue = Queue.new
+  end
+
+  def create_table_to_lock(connection = main_conn)
+    connection.query("DROP TABLE IF EXISTS #{test_table_name};")
+    connection.query("CREATE TABLE #{test_table_name} (id INT, PRIMARY KEY (id)) ENGINE=InnoDB;")
+  end
+
+  def hold_lock(seconds = lock_duration, queue = @queue)
+    # We are intentionally choosing to create a gap in the between the IDs to
+    # create a gap lock.
+    insert_records_at_ids(main_conn, [1001,1003])
+    locked_id = 1002
+
+    # This is the locking thread. It creates gap lock. It must be created first.
+    @threads << Thread.new do
+      conn = new_mysql_connection
+      conn.query("START TRANSACTION;")
+      conn.query("DELETE FROM #{test_table_name} WHERE id=#{locked_id}") # we now have the lock
+      queue.push(true) # this will signal the waiting thread to unblock, now that the lock is held
+      sleep seconds # hold the lock, while the waiting thread is waiting/retrying
+      conn.query("ROLLBACK;") # release the lock
+    end
+
+    return locked_id
+  end
+
+  def record_count(connection = main_conn)
+    response = connection.query("SELECT COUNT(id) FROM #{test_table_name}")
+    response.first.values.first
+  end
+
+  def with_waiting_lock(lock_time = @lock_duration, queue = @queue)
+    @threads << Thread.new do
+      conn = new_mysql_connection
+      conn.query("SET SESSION innodb_lock_wait_timeout = #{innodb_lock_wait_timeout}") # set timeout to be less than lock_time, so the timeout will happen
+      queue.pop # this will block until the lock thread establishes lock
+      yield(conn) # invoke the code that should retry while lock is held
+    end
+  end
+
+  def trigger_wait_lock
+    @threads.each(&:join)
+  end
+
+  def insert_records_at_ids(connection, ids)
+    ids.each do |id|
+      mysql_exec(connection, "INSERT INTO #{test_table_name} (id) VALUES (#{id})")
+    end
+  end
+
+  private
+
+  attr_reader :main_conn, :lock_duration, :innodb_lock_wait_timeout
+
+  def new_mysql_connection
+    Mysql2::Client.new(
+      host: '127.0.0.1',
+      username: db_config['master']['user'],
+      password: db_config['master']['password'],
+      port: db_config['master']['port'],
+      database: test_db_name
+    )
+  end
+
+  def test_db_name
+    @test_db_name ||= "test"
+  end
+
+  def db_config
+    @db_config ||= YAML.load_file(File.expand_path(File.dirname(__FILE__)) + '/../database.yml')
+  end
+
+  def test_table_name
+    @test_table_name ||= "lock_wait"
+  end
+
+  private
+
+  def mysql_exec(connection, statement)
+    if connection.class == Mysql2::Client
+      connection.query(statement)
+    elsif connection.class.to_s.include?("ActiveRecord")
+      connection.execute(statement)
+    else
+      raise StandardError.new("Unrecognized MySQL client")
+    end
+  end
+end

data/spec/integration/sql_retry/proxysql_helper.rb

@@ -0,0 +1,22 @@
+class ProxySQLHelper
+  class << self
+    # Flips the destination hostgroup for /maintenance:lhm/ from 0 (i.e. writer) to 1 (i.e. reader)
+    def with_lhm_hostgroup_flip
+      conn = Mysql2::Client.new(
+        host: '127.0.0.1',
+        username: "remote-admin",
+        password: "password",
+        port: "6032",
+      )
+
+      begin
+        conn.query("UPDATE mysql_query_rules SET destination_hostgroup=1 WHERE match_pattern=\"maintenance:lhm\"")
+        conn.query("LOAD MYSQL QUERY RULES TO RUNTIME;")
+        yield
+      ensure
+        conn.query("UPDATE mysql_query_rules SET destination_hostgroup=0 WHERE match_pattern=\"maintenance:lhm\"")
+        conn.query("LOAD MYSQL QUERY RULES TO RUNTIME;")
+      end
+    end
+  end
+end

data/spec/integration/sql_retry/retry_with_proxysql_spec.rb

@@ -0,0 +1,109 @@
+require 'minitest/autorun'
+require 'mysql2'
+require 'lhm'
+require 'toxiproxy'
+
+require 'integration/sql_retry/lock_wait_timeout_test_helper'
+require 'integration/sql_retry/db_connection_helper'
+require 'integration/sql_retry/proxysql_helper'
+require 'integration/toxiproxy_helper'
+
+describe Lhm::SqlRetry, "ProxiSQL tests for LHM retry" do
+  include ToxiproxyHelper
+
+  before(:each) do
+    @old_logger = Lhm.logger
+    @logger = StringIO.new
+    Lhm.logger = Logger.new(@logger)
+
+    @connection = DBConnectionHelper::new_mysql_connection(:proxysql, true, true)
+
+    @lhm_retry = Lhm::SqlRetry.new(@connection, retry_options: {},
+                                   reconnect_with_consistent_host: true)
+  end
+
+  after(:each) do
+    # Restore default logger
+    Lhm.logger = @old_logger
+  end
+
+  it "Will abort if service is down" do
+
+    e = assert_raises Lhm::Error do
+      #Service down
+      Toxiproxy[:mysql_proxysql].down do
+        @lhm_retry.with_retries do |retriable_connection|
+          retriable_connection.execute("INSERT INTO #{DBConnectionHelper.test_table_name} (id) VALUES (2000)")
+        end
+      end
+    end
+    assert_equal Lhm::Error, e.class
+    assert_match(/LHM tried the reconnection procedure but failed. Aborting/, e.message)
+  end
+
+  it "Will retry until connection is achieved" do
+
+    #Creating a network blip
+    ToxiproxyHelper.with_kill_and_restart(:mysql_proxysql, 2.seconds) do
+      @lhm_retry.with_retries do |retriable_connection|
+        retriable_connection.execute("INSERT INTO #{DBConnectionHelper.test_table_name} (id) VALUES (2000)")
+      end
+    end
+
+    assert_equal @connection.execute("Select * from #{DBConnectionHelper.test_table_name} WHERE id=2000").to_a.first.first, 2000
+
+    logs = @logger.string.split("\n")
+
+    assert logs.first.include?("Lost connection to MySQL, will retry to connect to same host")
+    assert logs.last.include?("LHM successfully reconnected to initial host")
+  end
+
+  it "Will abort if new writer is not same host" do
+    # The hostname will be constant before the blip
+    Lhm::SqlRetry.any_instance.stubs(:hostname).returns("mysql-1").then.returns("mysql-2")
+    Lhm::SqlRetry.any_instance.stubs(:server_id).returns(1).then.returns(2)
+
+    # Need new instance for stub to take into effect
+    lhm_retry = Lhm::SqlRetry.new(@connection, retry_options: {},
+                                  reconnect_with_consistent_host: true)
+
+    e = assert_raises Lhm::Error do
+      #Creating a network blip
+      ToxiproxyHelper.with_kill_and_restart(:mysql_proxysql, 2.seconds) do
+        lhm_retry.with_retries do |retriable_connection|
+          retriable_connection.execute("INSERT INTO #{DBConnectionHelper.test_table_name} (id) VALUES (2000)")
+        end
+      end
+    end
+
+    assert_equal e.class, Lhm::Error
+    assert_match(/LHM tried the reconnection procedure but failed. Aborting/, e.message)
+
+    logs = @logger.string.split("\n")
+
+    assert logs.first.include?("Lost connection to MySQL, will retry to connect to same host")
+    assert logs.last.include?("Reconnected to wrong host. Started migration on: mysql-1 (server_id: 1), but reconnected to: mysql-2 (server_id: 2).")
+  end
+
+  it "Will abort if failover happens (mimicked with proxySQL)" do
+    e = assert_raises Lhm::Error do
+      #Creates a failover by switching the target hostgroup for the #hostname
+      ProxySQLHelper.with_lhm_hostgroup_flip do
+        #Creating a network blip
+        ToxiproxyHelper.with_kill_and_restart(:mysql_proxysql, 2.seconds) do
+          @lhm_retry.with_retries do |retriable_connection|
+            retriable_connection.execute("INSERT INTO #{DBConnectionHelper.test_table_name} (id) VALUES (2000)")
+          end
+        end
+      end
+    end
+
+    assert_equal e.class, Lhm::Error
+    assert_match(/LHM tried the reconnection procedure but failed. Aborting/, e.message)
+
+    logs = @logger.string.split("\n")
+
+    assert logs.first.include?("Lost connection to MySQL, will retry to connect to same host")
+    assert logs.last.include?("Reconnected to wrong host. Started migration on: mysql-1 (server_id: 1), but reconnected to: mysql-2 (server_id: 2).")
+  end
+end

data/spec/integration/table_spec.rb

@@ -0,0 +1,83 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
+require 'lhm/table'
+
+describe Lhm::Table do
+  include IntegrationHelper
+
+  describe 'id numeric column requirement' do
+    describe 'when met' do
+      before(:each) do
+        connect_master!
+        @table = table_create(:custom_primary_key)
+      end
+
+      it 'should parse primary key' do
+        value(@table.pk).must_equal('pk')
+      end
+
+      it 'should parse indices' do
+        value(@table.indices['index_custom_primary_key_on_id']).must_equal(['id'])
+      end
+
+      it 'should parse columns' do
+        value(@table.columns['id'][:type]).must_match(/(bigint|int)\(\d+\)/)
+      end
+
+      it 'should return true for method that should be renamed' do
+        value(@table.satisfies_id_column_requirement?).must_equal true
+      end
+
+      it 'should support bigint tables' do
+        @table = table_create(:bigint_table)
+        value(@table.satisfies_id_column_requirement?).must_equal true
+      end
+    end
+
+    describe 'when not met' do
+      before(:each) do
+        connect_master!
+      end
+
+      it 'should return false for a non-int id column' do
+        @table = table_create(:wo_id_int_column)
+        value(@table.satisfies_id_column_requirement?).must_equal false
+      end
+    end
+  end
+
+  describe Lhm::Table::Parser do
+    describe 'create table parsing' do
+      before(:each) do
+        connect_master!
+        @table = table_create(:users)
+      end
+
+      it 'should parse table name in show create table' do
+        value(@table.name).must_equal('users')
+      end
+
+      it 'should parse primary key' do
+        value(@table.pk).must_equal('id')
+      end
+
+      it 'should parse column type in show create table' do
+        value(@table.columns['username'][:type]).must_equal('varchar(255)')
+      end
+
+      it 'should parse column metadata' do
+        assert_nil @table.columns['username'][:column_default]
+      end
+
+      it 'should parse indices' do
+        value(@table.indices['index_users_on_username_and_created_at']).must_equal(['username', 'created_at'])
+      end
+
+      it 'should parse index' do
+        value(@table.indices['index_users_on_reference']).must_equal(['reference'])
+      end
+    end
+  end
+end

data/spec/integration/toxiproxy_helper.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+require 'toxiproxy'
+
+module ToxiproxyHelper
+  class << self
+
+    def included(base)
+      Toxiproxy.reset
+
+      # listen on localhost, but toxiproxy is in a container itself, thus the upstream uses the Docker-Compose DNS
+      Toxiproxy.populate(
+        [
+          {
+            name: 'mysql_master',
+            listen: '0.0.0.0:22220',
+            upstream: 'mysql-1:3306'
+          },
+          {
+            name: 'mysql_proxysql',
+            listen: '0.0.0.0:22222',
+            upstream: 'proxysql:3306'
+          }
+        ])
+    end
+
+    def with_kill_and_restart(target, restart_after)
+      thread = Thread.new do
+        sleep(restart_after) unless restart_after.nil?
+        Toxiproxy[target].enable
+      end
+
+      Toxiproxy[target].disable
+
+      yield
+
+    ensure
+      thread.join
+    end
+  end
+end