lhm-shopify 3.5.0 → 3.5.1

data/spec/integration/lhm_spec.rb

@@ -2,6 +2,7 @@
 # Schmidt
 
 require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
+require 'integration/toxiproxy_helper'
 
 describe Lhm do
   include IntegrationHelper
@@ -581,5 +582,79 @@ describe Lhm do
         end
       end
     end
+
+    describe 'connection' do
+      include ToxiproxyHelper
+
+      before(:each) do
+        @logs = StringIO.new
+        Lhm.logger = Logger.new(@logs)
+      end
+
+      it " should not try to reconnect if reconnect_with_consistent_host is not provided" do
+        connect_master_with_toxiproxy!(with_retry: false)
+
+        table_create(:users)
+        100.times { |n| execute("insert into users set reference = '#{ n }'") }
+
+        assert_raises ActiveRecord::StatementInvalid do
+          Toxiproxy[:mysql_master].down do
+            Lhm.change_table(:users, :atomic_switch => false) do |t|
+              t.ddl("ALTER TABLE #{t.name} CHANGE id id bigint (20) NOT NULL")
+              t.ddl("ALTER TABLE #{t.name} DROP PRIMARY KEY, ADD PRIMARY KEY (username, id)")
+              t.ddl("ALTER TABLE #{t.name} ADD INDEX (id)")
+              t.ddl("ALTER TABLE #{t.name} CHANGE id id bigint (20) NOT NULL AUTO_INCREMENT")
+            end
+          end
+        end
+      end
+
+      it "should reconnect if reconnect_with_consistent_host is true" do
+        connect_master_with_toxiproxy!(with_retry: true)
+        mysql_disabled = false
+
+        table_create(:users)
+        100.times { |n| execute("insert into users set reference = '#{ n }'") }
+
+        # Redeclare Lhm::ChunkInsert to use Hook to disable MySQL writer for 3 seconds before first insert
+        Lhm::ChunkInsert.class_eval do
+          extend AfterDo
+
+          before(:insert_and_return_count_of_rows_created) do
+            unless mysql_disabled
+              mysql_disabled = true
+              Thread.new do
+                Toxiproxy[:mysql_master].down do
+                  sleep 3
+                end
+              end
+            end
+          end
+
+          # Need to call `#method_added` manually to have the hooks take into effect
+          method_added(:insert_and_return_count_of_rows_created)
+        end
+
+        Lhm.change_table(:users, :atomic_switch => false) do |t|
+          t.ddl("ALTER TABLE #{t.name} CHANGE id id bigint (20) NOT NULL")
+          t.ddl("ALTER TABLE #{t.name} DROP PRIMARY KEY, ADD PRIMARY KEY (username, id)")
+          t.ddl("ALTER TABLE #{t.name} ADD INDEX (id)")
+          t.ddl("ALTER TABLE #{t.name} CHANGE id id bigint (20) NOT NULL AUTO_INCREMENT")
+        end
+
+        log_lines = @logs.string.split("\n")
+
+        assert log_lines.one?{ |line| line.include?("Lost connection to MySQL, will retry to connect to same host")}
+        assert log_lines.any?{ |line| line.include?("Lost connection to MySQL server at 'reading initial communication packet'")}
+        assert log_lines.one?{ |line| line.include?("LHM successfully reconnected to initial host")}
+        assert log_lines.one?{ |line| line.include?("100% complete")}
+
+        Lhm::ChunkInsert.remove_all_callbacks
+
+        slave do
+          value(count_all(:users)).must_equal(100)
+        end
+      end
+    end
   end
 end

data/spec/integration/proxysql_spec.rb

@@ -0,0 +1,34 @@
+describe "ProxySQL integration" do
+  it "Should contact the writer" do
+    conn = Mysql2::Client.new(
+      host: '127.0.0.1',
+      username: "writer",
+      password: "password",
+      port: "33005",
+      )
+
+    assert_equal conn.query("SELECT @@global.hostname as host").each.first["host"], "mysql-1"
+  end
+
+  it "Should contact the reader" do
+    conn = Mysql2::Client.new(
+      host: '127.0.0.1',
+      username: "reader",
+      password: "password",
+      port: "33005",
+      )
+
+    assert_equal conn.query("SELECT @@global.hostname as host").each.first["host"], "mysql-2"
+  end
+
+  it "Should override default hostgroup from user if rule matches" do
+    conn = Mysql2::Client.new(
+      host: '127.0.0.1',
+      username: "reader",
+      password: "password",
+      port: "33005",
+      )
+
+    assert_equal conn.query("/*maintenance:lhm*/SELECT @@global.hostname as host").each.first["host"], "mysql-1"
+  end
+end

data/spec/integration/sql_retry/db_connection_helper.rb

@@ -0,0 +1,52 @@
+require 'yaml'
+require 'mysql2'
+
+class DBConnectionHelper
+
+  DATABASE_CONFIG_FILE = "database.yml"
+
+  class << self
+    def db_config
+      @db_config ||= YAML.load_file(File.expand_path(File.dirname(__FILE__)) + "/../#{DATABASE_CONFIG_FILE}")
+    end
+
+    def new_mysql_connection(role = :master, with_data = false, toxic = false)
+
+      key = role.to_s + toxic_postfix(toxic)
+
+      conn = ActiveRecord::Base.establish_connection(
+        :host => '127.0.0.1',
+        :adapter => "mysql2",
+        :username => db_config[key]['user'],
+        :password => db_config[key]['password'],
+        :database => test_db_name,
+        :port => db_config[key]['port']
+      )
+      conn = conn.connection
+      init_with_dummy_data(conn) if with_data
+      conn
+    end
+
+    def toxic_postfix(toxic)
+      toxic ? "_toxic" : ""
+    end
+
+    def test_db_name
+      @test_db_name ||= "test"
+    end
+
+    def test_table_name
+      @test_table_name ||= "test"
+    end
+
+    def init_with_dummy_data(conn)
+      conn.execute("DROP TABLE IF EXISTS #{test_table_name} ")
+      conn.execute("CREATE TABLE #{test_table_name} (id int)")
+
+      1.upto(9) do |i|
+        query = "INSERT INTO #{test_table_name} (id) VALUE (#{i})"
+        conn.execute(query)
+      end
+    end
+  end
+end

data/spec/integration/sql_retry/lock_wait_spec.rb

@@ -21,6 +21,8 @@ describe Lhm::SqlRetry do
 
     # Assert our pre-conditions
     assert_equal 2, @helper.record_count
+
+    Mysql2::Client.any_instance.stubs(:active?).returns(true)
   end
 
   after(:each) do
@@ -41,7 +43,7 @@ describe Lhm::SqlRetry do
 
     exception = assert_raises { @helper.trigger_wait_lock }
 
-    assert_equal "Lock wait timeout exceeded; try restarting transaction", exception.message
+    assert_match /Lock wait timeout exceeded; try restarting transaction/, exception.message
     assert_equal Mysql2::Error::TimeoutError, exception.class
 
     assert_equal 2, @helper.record_count # no records inserted
@@ -53,12 +55,12 @@ describe Lhm::SqlRetry do
   it "successfully executes the SQL despite the errors encountered" do
     # Start a thread to retry, once the lock is held, execute the block
     @helper.with_waiting_lock do |waiting_connection|
-      sql_retry = Lhm::SqlRetry.new(waiting_connection, {
+      sql_retry = Lhm::SqlRetry.new(waiting_connection, options: {
         base_interval: 0.2, # first retry after 200ms
         multiplier: 1, # subsequent retries wait 1x longer than first retry (no change)
         tries: 3, # we only need 3 tries (including the first) for the scenario described below
         rand_factor: 0 # do not introduce randomness to wait timer
-      })
+      }, reconnect_with_consistent_host: false)
 
       # RetryTestHelper is configured to hold lock for 5 seconds and timeout after 2 seconds.
       # Therefore the sequence of events will be:
@@ -96,12 +98,12 @@ describe Lhm::SqlRetry do
     puts "*" * 64
     # Start a thread to retry, once the lock is held, execute the block
     @helper.with_waiting_lock do |waiting_connection|
-      sql_retry = Lhm::SqlRetry.new(waiting_connection, {
+      sql_retry = Lhm::SqlRetry.new(waiting_connection, options: {
         base_interval: 0.2, # first retry after 200ms
         multiplier: 1, # subsequent retries wait 1x longer than first retry (no change)
         tries: 2, # we need 3 tries (including the first) for the scenario described below, but we only get two...we will fail
         rand_factor: 0 # do not introduce randomness to wait timer
-      })
+      }, reconnect_with_consistent_host: false)
 
       # RetryTestHelper is configured to hold lock for 5 seconds and timeout after 2 seconds.
       # Therefore the sequence of events will be:
@@ -116,7 +118,7 @@ describe Lhm::SqlRetry do
 
     exception = assert_raises { @helper.trigger_wait_lock }
 
-    assert_equal "Lock wait timeout exceeded; try restarting transaction", exception.message
+    assert_match /Lock wait timeout exceeded; try restarting transaction/, exception.message
     assert_equal Mysql2::Error::TimeoutError, exception.class
 
     assert_equal 2, @helper.record_count # no records inserted

data/spec/integration/sql_retry/lock_wait_timeout_test_helper.rb

@@ -1,5 +1,7 @@
-require 'yaml'
+require 'integration/integration_helper'
+
 class LockWaitTimeoutTestHelper
+
   def initialize(lock_duration:, innodb_lock_wait_timeout:)
     # This connection will be used exclusively to setup the test,
     # assert pre-conditions and assert post-conditions.
@@ -68,7 +70,7 @@ class LockWaitTimeoutTestHelper
 
   def insert_records_at_ids(connection, ids)
     ids.each do |id|
-      connection.query "INSERT INTO #{test_table_name} (id) VALUES (#{id})"
+      mysql_exec(connection, "INSERT INTO #{test_table_name} (id) VALUES (#{id})")
     end
   end
 
@@ -77,17 +79,13 @@ class LockWaitTimeoutTestHelper
   attr_reader :main_conn, :lock_duration, :innodb_lock_wait_timeout
 
   def new_mysql_connection
-    client = Mysql2::Client.new(
+    Mysql2::Client.new(
       host: '127.0.0.1',
       username: db_config['master']['user'],
       password: db_config['master']['password'],
-      port: db_config['master']['port']
+      port: db_config['master']['port'],
+      database: test_db_name
     )
-
-    # For some reasons sometimes the database does not exist
-    client.query("CREATE DATABASE IF NOT EXISTS #{test_db_name}")
-    client.select_db(test_db_name)
-    client
   end
 
   def test_db_name
@@ -101,4 +99,16 @@ class LockWaitTimeoutTestHelper
   def test_table_name
     @test_table_name ||= "lock_wait"
   end
+
+  private
+
+  def mysql_exec(connection, statement)
+    if connection.class == Mysql2::Client
+      connection.query(statement)
+    elsif connection.class.to_s.include?("ActiveRecord")
+      connection.execute(statement)
+    else
+      raise StandardError.new("Unrecognized MySQL client")
+    end
+  end
 end

data/spec/integration/sql_retry/proxysql_helper.rb

@@ -0,0 +1,22 @@
+class ProxySQLHelper
+  class << self
+    # Flips the destination hostgroup for /maintenance:lhm/ from 0 (i.e. writer) to 1 (i.e. reader)
+    def with_lhm_hostgroup_flip
+      conn = Mysql2::Client.new(
+        host: '127.0.0.1',
+        username: "remote-admin",
+        password: "password",
+        port: "6032",
+      )
+
+      begin
+        conn.query("UPDATE mysql_query_rules SET destination_hostgroup=1 WHERE match_pattern=\"maintenance:lhm\"")
+        conn.query("LOAD MYSQL QUERY RULES TO RUNTIME;")
+        yield
+      ensure
+        conn.query("UPDATE mysql_query_rules SET destination_hostgroup=0 WHERE match_pattern=\"maintenance:lhm\"")
+        conn.query("LOAD MYSQL QUERY RULES TO RUNTIME;")
+      end
+    end
+  end
+end

data/spec/integration/sql_retry/retry_with_proxysql_spec.rb

@@ -0,0 +1,108 @@
+require 'minitest/autorun'
+require 'mysql2'
+require 'lhm'
+require 'toxiproxy'
+
+require 'integration/sql_retry/lock_wait_timeout_test_helper'
+require 'integration/sql_retry/db_connection_helper'
+require 'integration/sql_retry/proxysql_helper'
+require 'integration/toxiproxy_helper'
+
+describe Lhm::SqlRetry, "ProxiSQL tests for LHM retry" do
+  include ToxiproxyHelper
+
+  before(:each) do
+    @old_logger = Lhm.logger
+    @logger = StringIO.new
+    Lhm.logger = Logger.new(@logger)
+
+    @connection = DBConnectionHelper::new_mysql_connection(:proxysql, true, true)
+
+    @lhm_retry = Lhm::SqlRetry.new(@connection, options: {},
+                                   reconnect_with_consistent_host: true)
+  end
+
+  after(:each) do
+    # Restore default logger
+    Lhm.logger = @old_logger
+  end
+
+  it "Will abort if service is down" do
+
+    e = assert_raises Lhm::Error do
+      #Service down
+      Toxiproxy[:mysql_proxysql].down do
+        @lhm_retry.with_retries do |retriable_connection|
+          retriable_connection.execute("INSERT INTO #{DBConnectionHelper.test_table_name} (id) VALUES (2000)")
+        end
+      end
+    end
+    assert_equal Lhm::Error, e.class
+    assert_match(/LHM tried the reconnection procedure but failed. Latest error:/, e.message)
+  end
+
+  it "Will retry until connection is achieved" do
+
+    #Creating a network blip
+    ToxiproxyHelper.with_kill_and_restart(:mysql_proxysql, 2.seconds) do
+      @lhm_retry.with_retries do |retriable_connection|
+        retriable_connection.execute("INSERT INTO #{DBConnectionHelper.test_table_name} (id) VALUES (2000)")
+      end
+    end
+
+    assert_equal @connection.execute("Select * from #{DBConnectionHelper.test_table_name} WHERE id=2000").to_a.first.first, 2000
+
+    logs = @logger.string.split("\n")
+
+    assert logs.first.include?("Lost connection to MySQL, will retry to connect to same host")
+    assert logs.last.include?("LHM successfully reconnected to initial host")
+  end
+
+  it "Will abort if new writer is not same host" do
+    # The hostname will be constant before the blip
+    Lhm::SqlRetry.any_instance.stubs(:hostname).returns("mysql-1").then.returns("mysql-2")
+
+    # Need new instance for stub to take into effect
+    lhm_retry = Lhm::SqlRetry.new(@connection, options: {},
+                                  reconnect_with_consistent_host: true)
+
+    e = assert_raises Lhm::Error do
+      #Creating a network blip
+      ToxiproxyHelper.with_kill_and_restart(:mysql_proxysql, 2.seconds) do
+        lhm_retry.with_retries do |retriable_connection|
+          retriable_connection.execute("INSERT INTO #{DBConnectionHelper.test_table_name} (id) VALUES (2000)")
+        end
+      end
+    end
+
+    assert_equal e.class, Lhm::Error
+    assert_match(/LHM tried the reconnection procedure but failed. Latest error: Reconnected to wrong host/, e.message)
+
+    logs = @logger.string.split("\n")
+
+    assert logs.first.include?("Lost connection to MySQL, will retry to connect to same host")
+    assert logs.last.include?("Lost connection to MySQL server at 'reading initial communication packet")
+  end
+
+  it "Will abort if failover happens (mimicked with proxySQL)" do
+    e = assert_raises Lhm::Error do
+      #Creates a failover by switching the target hostgroup for the #hostname
+      ProxySQLHelper.with_lhm_hostgroup_flip do
+        #Creating a network blip
+        ToxiproxyHelper.with_kill_and_restart(:mysql_proxysql, 2.seconds) do
+          @lhm_retry.with_retries do |retriable_connection|
+            retriable_connection.execute("INSERT INTO #{DBConnectionHelper.test_table_name} (id) VALUES (2000)")
+          end
+        end
+      end
+    end
+
+    assert_equal e.class, Lhm::Error
+    assert_match(/LHM tried the reconnection procedure but failed. Latest error: Reconnected to wrong host/, e.message)
+
+    logs = @logger.string.split("\n")
+
+    assert logs.first.include?("Lost connection to MySQL, will retry to connect to same host")
+    assert logs.last.include?("Lost connection to MySQL server at 'reading initial communication packet")
+  end
+end

data/spec/integration/toxiproxy_helper.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+require 'toxiproxy'
+
+module ToxiproxyHelper
+  class << self
+
+    def included(base)
+      Toxiproxy.reset
+
+      # listen on localhost, but toxiproxy is in a container itself, thus the upstream uses the Docker-Compose DNS
+      Toxiproxy.populate(
+        [
+          {
+            name: 'mysql_master',
+            listen: '0.0.0.0:22220',
+            upstream: 'mysql-1:3306'
+          },
+          {
+            name: 'mysql_proxysql',
+            listen: '0.0.0.0:22222',
+            upstream: 'proxysql:3306'
+          }
+        ])
+    end
+
+    def with_kill_and_restart(target, restart_after)
+      thread = Thread.new do
+        sleep(restart_after) unless restart_after.nil?
+        Toxiproxy[target].enable
+      end
+
+      Toxiproxy[target].disable
+
+      yield
+
+    ensure
+      thread.join
+    end
+  end
+end

data/spec/test_helper.rb

@@ -10,6 +10,8 @@ require 'minitest/autorun'
 require 'minitest/spec'
 require 'minitest/mock'
 require 'mocha/minitest'
+require 'after_do'
+require 'byebug'
 require 'pathname'
 require 'lhm'
 
@@ -17,6 +19,8 @@ $project = Pathname.new(File.dirname(__FILE__) + '/..').cleanpath
 $spec = $project.join('spec')
 $fixtures = $spec.join('fixtures')
 
+$db_name = 'test'
+
 require 'active_record'
 require 'mysql2'
 
@@ -43,3 +47,20 @@ end
 def throttler
   Lhm::Throttler::Time.new(:stride => 100)
 end
+
+def init_test_db
+  db_config = YAML.load_file(File.expand_path(File.dirname(__FILE__)) + '/integration/database.yml')
+  conn = Mysql2::Client.new(
+    :host => '127.0.0.1',
+    :username => db_config['master']['user'],
+    :password => db_config['master']['password'],
+    :port => db_config['master']['port']
+  )
+
+  conn.query("DROP DATABASE IF EXISTS #{$db_name}")
+  conn.query("CREATE DATABASE #{$db_name}")
+end
+
+init_test_db
+
+

data/spec/unit/chunk_insert_spec.rb

@@ -4,17 +4,22 @@
 require File.expand_path(File.dirname(__FILE__)) + '/unit_helper'
 
 require 'lhm/chunk_insert'
+require 'lhm/connection'
 
 describe Lhm::ChunkInsert do
   before(:each) do
-    @connection = stub(:connection)
+    ar_connection = mock()
+    ar_connection.stubs(:execute).returns([["dummy"]])
+    @connection = Lhm::Connection.new(connection: ar_connection, options: {reconnect_with_consistent_host: false})
     @origin = Lhm::Table.new('foo')
     @destination = Lhm::Table.new('bar')
   end
 
   describe "#sql" do
     describe "when migration has no conditions" do
-      before { @migration = Lhm::Migration.new(@origin, @destination) }
+      before do
+        @migration = Lhm::Migration.new(@origin, @destination)
+      end
 
       it "uses a simple where clause" do
         assert_equal(