lhm-shopify 3.5.0 → 3.5.4

data/spec/integration/atomic_switcher_spec.rb

@@ -16,9 +16,9 @@ describe Lhm::AtomicSwitcher do
   describe 'switching' do
     before(:each) do
       Thread.abort_on_exception = true
-      @origin      = table_create('origin')
+      @origin = table_create('origin')
       @destination = table_create('destination')
-      @migration   = Lhm::Migration.new(@origin, @destination)
+      @migration = Lhm::Migration.new(@origin, @destination)
       @logs = StringIO.new
       Lhm.logger = Logger.new(@logs)
       @connection.execute('SET GLOBAL innodb_lock_wait_timeout=3')
@@ -32,11 +32,22 @@ describe Lhm::AtomicSwitcher do
     it 'should retry and log on lock wait timeouts' do
       ar_connection = mock()
       ar_connection.stubs(:data_source_exists?).returns(true)
-      ar_connection.stubs(:execute).raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.').then.returns(true)
+      ar_connection.stubs(:active?).returns(true)
+      ar_connection.stubs(:execute).returns([["dummy"]], [["dummy"]], [["dummy"]])
+                   .then
+                   .raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
+                   .then
+                   .returns([["dummy"]]) # Matches initial host -> triggers retry
+
+      connection = Lhm::Connection.new(connection: ar_connection, options: {
+        reconnect_with_consistent_host: true,
+        retriable: {
+          tries: 3,
+          base_interval: 0
+        }
+      })
 
-      connection = Lhm::Connection.new(connection: ar_connection)
-
-      switcher = Lhm::AtomicSwitcher.new(@migration, connection, retriable: {base_interval: 0})
+      switcher = Lhm::AtomicSwitcher.new(@migration, connection)
 
       assert switcher.run
 
@@ -50,11 +61,28 @@ describe Lhm::AtomicSwitcher do
     it 'should give up on lock wait timeouts after a configured number of tries' do
       ar_connection = mock()
       ar_connection.stubs(:data_source_exists?).returns(true)
-      ar_connection.stubs(:execute).twice.raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
+      ar_connection.stubs(:active?).returns(true)
+      ar_connection.stubs(:execute).returns([["dummy"]], [["dummy"]], [["dummy"]])
+                   .then
+                   .raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
+                   .then
+                   .returns([["dummy"]]) # triggers retry 1
+                   .then
+                   .raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
+                   .then
+                   .returns([["dummy"]]) # triggers retry 2
+                   .then
+                   .raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.') # triggers retry 2
+
+      connection = Lhm::Connection.new(connection: ar_connection, options: {
+        reconnect_with_consistent_host: true,
+        retriable: {
+          tries: 2,
+          base_interval: 0
+        }
+      })
 
-      connection = Lhm::Connection.new(connection: ar_connection)
-
-      switcher = Lhm::AtomicSwitcher.new(@migration, connection, retriable: {tries: 2, base_interval: 0})
+      switcher = Lhm::AtomicSwitcher.new(@migration, connection)
 
       assert_raises(ActiveRecord::StatementInvalid) { switcher.run }
     end

data/spec/integration/chunk_insert_spec.rb

@@ -11,7 +11,8 @@ describe Lhm::ChunkInsert do
       @destination = table_create(:destination)
       @migration = Lhm::Migration.new(@origin, @destination)
       execute("insert into origin set id = 1001")
-      @instance = Lhm::ChunkInsert.new(@migration, connection, 1001, 1001)
+      @connection = Lhm::Connection.new(connection: connection)
+      @instance = Lhm::ChunkInsert.new(@migration, @connection, 1001, 1001)
     end
 
     it "returns the count" do

data/spec/integration/chunker_spec.rb

@@ -173,6 +173,9 @@ describe Lhm::Chunker do
       printer.expect(:notify, :return_value, [Integer, Integer])
       printer.expect(:end, :return_value, [])
 
+      Lhm::Throttler::Slave.any_instance.stubs(:slave_hosts).returns(['127.0.0.1'])
+      Lhm::Throttler::SlaveLag.any_instance.stubs(:master_slave_hosts).returns(['127.0.0.1'])
+
       Lhm::Chunker.new(
         @migration, connection, { throttler: Lhm::Throttler::SlaveLag.new(stride: 100), printer: printer }
       ).run
@@ -215,17 +218,16 @@ describe Lhm::Chunker do
       printer.expects(:verify)
       printer.expects(:end)
 
-      throttler = Lhm::Throttler::SlaveLag.new(stride: 10, allowed_lag: 0)
+      Lhm::Throttler::Slave.any_instance.stubs(:slave_hosts).returns(['127.0.0.1'])
+      Lhm::Throttler::SlaveLag.any_instance.stubs(:master_slave_hosts).returns(['127.0.0.1'])
 
-      def throttler.slave_hosts
-        ['127.0.0.1']
-      end
+      throttler = Lhm::Throttler::SlaveLag.new(stride: 10, allowed_lag: 0)
 
       if master_slave_mode?
         def throttler.slave_connection(slave)
-          config = ActiveRecord::Base.connection_pool.spec.config.dup
+          config = ActiveRecord::Base.connection_pool.db_config.configuration_hash.dup
           config[:host] = slave
-          config[:port] = 3307
+          config[:port] = 33007
           ActiveRecord::Base.send('mysql2_connection', config)
         end
       end

data/spec/integration/database.yml

@@ -13,3 +13,13 @@ proxysql:
   user: root
   password: password
   port: 33005
+master_toxic:
+  host: toxiproxy
+  user: root
+  password: password
+  port: 22220
+proxysql_toxic:
+  host: toxiproxy
+  user: root
+  password: password
+  port: 22222

data/spec/integration/entangler_spec.rb

@@ -6,6 +6,7 @@ require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
 require 'lhm/table'
 require 'lhm/migration'
 require 'lhm/entangler'
+require 'lhm/connection'
 
 describe Lhm::Entangler do
   include IntegrationHelper
@@ -17,7 +18,8 @@ describe Lhm::Entangler do
       @origin = table_create('origin')
       @destination = table_create('destination')
       @migration = Lhm::Migration.new(@origin, @destination)
-      @entangler = Lhm::Entangler.new(@migration, connection)
+      @connection = Lhm::Connection.new(connection: connection)
+      @entangler = Lhm::Entangler.new(@migration, @connection)
     end
 
     it 'should replay inserts from origin into destination' do

data/spec/integration/integration_helper.rb

@@ -35,6 +35,15 @@ module IntegrationHelper
     @connection
   end
 
+  def connect_proxysql!
+    connect!(
+      '127.0.0.1',
+      $db_config['proxysql']['port'],
+      $db_config['proxysql']['user'],
+      $db_config['proxysql']['password'],
+    )
+  end
+
   def connect_master!
     connect!(
       '127.0.0.1',
@@ -53,14 +62,21 @@ module IntegrationHelper
     )
   end
 
+  def connect_master_with_toxiproxy!
+    connect!(
+      '127.0.0.1',
+      $db_config['master_toxic']['port'],
+      $db_config['master_toxic']['user'],
+      $db_config['master_toxic']['password'])
+  end
+
   def connect!(hostname, port, user, password)
-    adapter = Lhm::Connection.new(connection: ar_conn(hostname, port, user, password))
-    Lhm.setup(adapter)
+    Lhm.setup(ar_conn(hostname, port, user, password))
     unless defined?(@@cleaned_up)
       Lhm.cleanup(true)
       @@cleaned_up  = true
     end
-    @connection = adapter
+    @connection = Lhm.connection
   end
 
   def ar_conn(host, port, user, password)
@@ -119,7 +135,7 @@ module IntegrationHelper
   # Helps testing behaviour when another client locks the db
   def start_locking_thread(lock_for, queue, locking_query)
     Thread.new do
-      conn = Mysql2::Client.new(host: '127.0.0.1', database: $db_name, user: 'root', port: 3306)
+      conn = new_mysql_connection
       conn.query('BEGIN')
       conn.query(locking_query)
       queue.push(true)

data/spec/integration/lhm_spec.rb

@@ -2,6 +2,7 @@
 # Schmidt
 
 require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
+require 'integration/toxiproxy_helper'
 
 describe Lhm do
   include IntegrationHelper
@@ -581,5 +582,79 @@ describe Lhm do
         end
       end
     end
+
+    describe 'connection' do
+      include ToxiproxyHelper
+
+      before(:each) do
+        @logs = StringIO.new
+        Lhm.logger = Logger.new(@logs)
+      end
+
+      it " should not try to reconnect if reconnect_with_consistent_host is not provided" do
+        connect_master_with_toxiproxy!
+
+        table_create(:users)
+        100.times { |n| execute("insert into users set reference = '#{ n }'") }
+
+        assert_raises ActiveRecord::StatementInvalid do
+          Toxiproxy[:mysql_master].down do
+            Lhm.change_table(:users, :atomic_switch => false) do |t|
+              t.ddl("ALTER TABLE #{t.name} CHANGE id id bigint (20) NOT NULL")
+              t.ddl("ALTER TABLE #{t.name} DROP PRIMARY KEY, ADD PRIMARY KEY (username, id)")
+              t.ddl("ALTER TABLE #{t.name} ADD INDEX (id)")
+              t.ddl("ALTER TABLE #{t.name} CHANGE id id bigint (20) NOT NULL AUTO_INCREMENT")
+            end
+          end
+        end
+      end
+
+      it "should reconnect if reconnect_with_consistent_host is true" do
+        connect_master_with_toxiproxy!
+        mysql_disabled = false
+
+        table_create(:users)
+        100.times { |n| execute("insert into users set reference = '#{ n }'") }
+
+        # Redeclare Lhm::ChunkInsert to use Hook to disable MySQL writer for 3 seconds before first insert
+        Lhm::ChunkInsert.class_eval do
+          extend AfterDo
+
+          before(:insert_and_return_count_of_rows_created) do
+            unless mysql_disabled
+              mysql_disabled = true
+              Thread.new do
+                Toxiproxy[:mysql_master].down do
+                  sleep 3
+                end
+              end
+            end
+          end
+
+          # Need to call `#method_added` manually to have the hooks take into effect
+          method_added(:insert_and_return_count_of_rows_created)
+        end
+
+        Lhm.change_table(:users, atomic_switch: false, reconnect_with_consistent_host: true) do |t|
+          t.ddl("ALTER TABLE #{t.name} CHANGE id id bigint (20) NOT NULL")
+          t.ddl("ALTER TABLE #{t.name} DROP PRIMARY KEY, ADD PRIMARY KEY (username, id)")
+          t.ddl("ALTER TABLE #{t.name} ADD INDEX (id)")
+          t.ddl("ALTER TABLE #{t.name} CHANGE id id bigint (20) NOT NULL AUTO_INCREMENT")
+        end
+
+        log_lines = @logs.string.split("\n")
+
+        assert log_lines.one?{ |line| line.include?("Lost connection to MySQL, will retry to connect to same host")}
+        assert log_lines.any?{ |line| line.include?("Lost connection to MySQL server at 'reading initial communication packet'")}
+        assert log_lines.one?{ |line| line.include?("LHM successfully reconnected to initial host")}
+        assert log_lines.one?{ |line| line.include?("100% complete")}
+
+        Lhm::ChunkInsert.remove_all_callbacks
+
+        slave do
+          value(count_all(:users)).must_equal(100)
+        end
+      end
+    end
   end
 end

data/spec/integration/proxysql_spec.rb

@@ -0,0 +1,34 @@
+describe "ProxySQL integration" do
+  it "Should contact the writer" do
+    conn = Mysql2::Client.new(
+      host: '127.0.0.1',
+      username: "writer",
+      password: "password",
+      port: "33005",
+      )
+
+    assert_equal conn.query("SELECT @@global.hostname as host").each.first["host"], "mysql-1"
+  end
+
+  it "Should contact the reader" do
+    conn = Mysql2::Client.new(
+      host: '127.0.0.1',
+      username: "reader",
+      password: "password",
+      port: "33005",
+      )
+
+    assert_equal conn.query("SELECT @@global.hostname as host").each.first["host"], "mysql-2"
+  end
+
+  it "Should override default hostgroup from user if rule matches" do
+    conn = Mysql2::Client.new(
+      host: '127.0.0.1',
+      username: "reader",
+      password: "password",
+      port: "33005",
+      )
+
+    assert_equal conn.query("SELECT @@global.hostname as host #{Lhm::ProxySQLHelper::ANNOTATION}").each.first["host"], "mysql-1"
+  end
+end

data/spec/integration/sql_retry/db_connection_helper.rb

@@ -0,0 +1,52 @@
+require 'yaml'
+require 'mysql2'
+
+class DBConnectionHelper
+
+  DATABASE_CONFIG_FILE = "database.yml"
+
+  class << self
+    def db_config
+      @db_config ||= YAML.load_file(File.expand_path(File.dirname(__FILE__)) + "/../#{DATABASE_CONFIG_FILE}")
+    end
+
+    def new_mysql_connection(role = :master, with_data = false, toxic = false)
+
+      key = role.to_s + toxic_postfix(toxic)
+
+      conn = ActiveRecord::Base.establish_connection(
+        :host => '127.0.0.1',
+        :adapter => "mysql2",
+        :username => db_config[key]['user'],
+        :password => db_config[key]['password'],
+        :database => test_db_name,
+        :port => db_config[key]['port']
+      )
+      conn = conn.connection
+      init_with_dummy_data(conn) if with_data
+      conn
+    end
+
+    def toxic_postfix(toxic)
+      toxic ? "_toxic" : ""
+    end
+
+    def test_db_name
+      @test_db_name ||= "test"
+    end
+
+    def test_table_name
+      @test_table_name ||= "test"
+    end
+
+    def init_with_dummy_data(conn)
+      conn.execute("DROP TABLE IF EXISTS #{test_table_name} ")
+      conn.execute("CREATE TABLE #{test_table_name} (id int)")
+
+      1.upto(9) do |i|
+        query = "INSERT INTO #{test_table_name} (id) VALUE (#{i})"
+        conn.execute(query)
+      end
+    end
+  end
+end

data/spec/integration/sql_retry/lock_wait_spec.rb

@@ -21,6 +21,8 @@ describe Lhm::SqlRetry do
 
     # Assert our pre-conditions
     assert_equal 2, @helper.record_count
+
+    Mysql2::Client.any_instance.stubs(:active?).returns(true)
   end
 
   after(:each) do
@@ -41,7 +43,7 @@ describe Lhm::SqlRetry do
 
     exception = assert_raises { @helper.trigger_wait_lock }
 
-    assert_equal "Lock wait timeout exceeded; try restarting transaction", exception.message
+    assert_match /Lock wait timeout exceeded; try restarting transaction/, exception.message
     assert_equal Mysql2::Error::TimeoutError, exception.class
 
     assert_equal 2, @helper.record_count # no records inserted
@@ -53,12 +55,12 @@ describe Lhm::SqlRetry do
   it "successfully executes the SQL despite the errors encountered" do
     # Start a thread to retry, once the lock is held, execute the block
     @helper.with_waiting_lock do |waiting_connection|
-      sql_retry = Lhm::SqlRetry.new(waiting_connection, {
+      sql_retry = Lhm::SqlRetry.new(waiting_connection, retry_options: {
         base_interval: 0.2, # first retry after 200ms
         multiplier: 1, # subsequent retries wait 1x longer than first retry (no change)
         tries: 3, # we only need 3 tries (including the first) for the scenario described below
         rand_factor: 0 # do not introduce randomness to wait timer
-      })
+      }, reconnect_with_consistent_host: false)
 
       # RetryTestHelper is configured to hold lock for 5 seconds and timeout after 2 seconds.
       # Therefore the sequence of events will be:
@@ -96,12 +98,12 @@ describe Lhm::SqlRetry do
     puts "*" * 64
     # Start a thread to retry, once the lock is held, execute the block
     @helper.with_waiting_lock do |waiting_connection|
-      sql_retry = Lhm::SqlRetry.new(waiting_connection, {
+      sql_retry = Lhm::SqlRetry.new(waiting_connection, retry_options: {
         base_interval: 0.2, # first retry after 200ms
         multiplier: 1, # subsequent retries wait 1x longer than first retry (no change)
         tries: 2, # we need 3 tries (including the first) for the scenario described below, but we only get two...we will fail
         rand_factor: 0 # do not introduce randomness to wait timer
-      })
+      }, reconnect_with_consistent_host: false)
 
       # RetryTestHelper is configured to hold lock for 5 seconds and timeout after 2 seconds.
       # Therefore the sequence of events will be:
@@ -116,7 +118,7 @@ describe Lhm::SqlRetry do
 
     exception = assert_raises { @helper.trigger_wait_lock }
 
-    assert_equal "Lock wait timeout exceeded; try restarting transaction", exception.message
+    assert_match /Lock wait timeout exceeded; try restarting transaction/, exception.message
     assert_equal Mysql2::Error::TimeoutError, exception.class
 
     assert_equal 2, @helper.record_count # no records inserted

data/spec/integration/sql_retry/lock_wait_timeout_test_helper.rb

@@ -1,5 +1,7 @@
-require 'yaml'
+require 'integration/integration_helper'
+
 class LockWaitTimeoutTestHelper
+
   def initialize(lock_duration:, innodb_lock_wait_timeout:)
     # This connection will be used exclusively to setup the test,
     # assert pre-conditions and assert post-conditions.
@@ -68,7 +70,7 @@ class LockWaitTimeoutTestHelper
 
   def insert_records_at_ids(connection, ids)
     ids.each do |id|
-      connection.query "INSERT INTO #{test_table_name} (id) VALUES (#{id})"
+      mysql_exec(connection, "INSERT INTO #{test_table_name} (id) VALUES (#{id})")
     end
   end
 
@@ -77,17 +79,13 @@ class LockWaitTimeoutTestHelper
   attr_reader :main_conn, :lock_duration, :innodb_lock_wait_timeout
 
   def new_mysql_connection
-    client = Mysql2::Client.new(
+    Mysql2::Client.new(
       host: '127.0.0.1',
       username: db_config['master']['user'],
       password: db_config['master']['password'],
-      port: db_config['master']['port']
+      port: db_config['master']['port'],
+      database: test_db_name
     )
-
-    # For some reasons sometimes the database does not exist
-    client.query("CREATE DATABASE IF NOT EXISTS #{test_db_name}")
-    client.select_db(test_db_name)
-    client
   end
 
   def test_db_name
@@ -101,4 +99,16 @@ class LockWaitTimeoutTestHelper
   def test_table_name
     @test_table_name ||= "lock_wait"
   end
+
+  private
+
+  def mysql_exec(connection, statement)
+    if connection.class == Mysql2::Client
+      connection.query(statement)
+    elsif connection.class.to_s.include?("ActiveRecord")
+      connection.execute(statement)
+    else
+      raise StandardError.new("Unrecognized MySQL client")
+    end
+  end
 end

data/spec/integration/sql_retry/proxysql_helper.rb

@@ -0,0 +1,22 @@
+class ProxySQLHelper
+  class << self
+    # Flips the destination hostgroup for /maintenance:lhm/ from 0 (i.e. writer) to 1 (i.e. reader)
+    def with_lhm_hostgroup_flip
+      conn = Mysql2::Client.new(
+        host: '127.0.0.1',
+        username: "remote-admin",
+        password: "password",
+        port: "6032",
+      )
+
+      begin
+        conn.query("UPDATE mysql_query_rules SET destination_hostgroup=1 WHERE match_pattern=\"maintenance:lhm\"")
+        conn.query("LOAD MYSQL QUERY RULES TO RUNTIME;")
+        yield
+      ensure
+        conn.query("UPDATE mysql_query_rules SET destination_hostgroup=0 WHERE match_pattern=\"maintenance:lhm\"")
+        conn.query("LOAD MYSQL QUERY RULES TO RUNTIME;")
+      end
+    end
+  end
+end

data/spec/integration/sql_retry/retry_with_proxysql_spec.rb

@@ -0,0 +1,108 @@
+require 'minitest/autorun'
+require 'mysql2'
+require 'lhm'
+require 'toxiproxy'
+
+require 'integration/sql_retry/lock_wait_timeout_test_helper'
+require 'integration/sql_retry/db_connection_helper'
+require 'integration/sql_retry/proxysql_helper'
+require 'integration/toxiproxy_helper'
+
+describe Lhm::SqlRetry, "ProxiSQL tests for LHM retry" do
+  include ToxiproxyHelper
+
+  before(:each) do
+    @old_logger = Lhm.logger
+    @logger = StringIO.new
+    Lhm.logger = Logger.new(@logger)
+
+    @connection = DBConnectionHelper::new_mysql_connection(:proxysql, true, true)
+
+    @lhm_retry = Lhm::SqlRetry.new(@connection, retry_options: {},
+                                   reconnect_with_consistent_host: true)
+  end
+
+  after(:each) do
+    # Restore default logger
+    Lhm.logger = @old_logger
+  end
+
+  it "Will abort if service is down" do
+
+    e = assert_raises Lhm::Error do
+      #Service down
+      Toxiproxy[:mysql_proxysql].down do
+        @lhm_retry.with_retries do |retriable_connection|
+          retriable_connection.execute("INSERT INTO #{DBConnectionHelper.test_table_name} (id) VALUES (2000)")
+        end
+      end
+    end
+    assert_equal Lhm::Error, e.class
+    assert_match(/LHM tried the reconnection procedure but failed. Latest error:/, e.message)
+  end
+
+  it "Will retry until connection is achieved" do
+
+    #Creating a network blip
+    ToxiproxyHelper.with_kill_and_restart(:mysql_proxysql, 2.seconds) do
+      @lhm_retry.with_retries do |retriable_connection|
+        retriable_connection.execute("INSERT INTO #{DBConnectionHelper.test_table_name} (id) VALUES (2000)")
+      end
+    end
+
+    assert_equal @connection.execute("Select * from #{DBConnectionHelper.test_table_name} WHERE id=2000").to_a.first.first, 2000
+
+    logs = @logger.string.split("\n")
+
+    assert logs.first.include?("Lost connection to MySQL, will retry to connect to same host")
+    assert logs.last.include?("LHM successfully reconnected to initial host")
+  end
+
+  it "Will abort if new writer is not same host" do
+    # The hostname will be constant before the blip
+    Lhm::SqlRetry.any_instance.stubs(:hostname).returns("mysql-1").then.returns("mysql-2")
+
+    # Need new instance for stub to take into effect
+    lhm_retry = Lhm::SqlRetry.new(@connection, retry_options: {},
+                                  reconnect_with_consistent_host: true)
+
+    e = assert_raises Lhm::Error do
+      #Creating a network blip
+      ToxiproxyHelper.with_kill_and_restart(:mysql_proxysql, 2.seconds) do
+        lhm_retry.with_retries do |retriable_connection|
+          retriable_connection.execute("INSERT INTO #{DBConnectionHelper.test_table_name} (id) VALUES (2000)")
+        end
+      end
+    end
+
+    assert_equal e.class, Lhm::Error
+    assert_match(/LHM tried the reconnection procedure but failed. Latest error: Reconnected to wrong host/, e.message)
+
+    logs = @logger.string.split("\n")
+
+    assert logs.first.include?("Lost connection to MySQL, will retry to connect to same host")
+    assert logs.last.include?("Lost connection to MySQL server at 'reading initial communication packet")
+  end
+
+  it "Will abort if failover happens (mimicked with proxySQL)" do
+    e = assert_raises Lhm::Error do
+      #Creates a failover by switching the target hostgroup for the #hostname
+      ProxySQLHelper.with_lhm_hostgroup_flip do
+        #Creating a network blip
+        ToxiproxyHelper.with_kill_and_restart(:mysql_proxysql, 2.seconds) do
+          @lhm_retry.with_retries do |retriable_connection|
+            retriable_connection.execute("INSERT INTO #{DBConnectionHelper.test_table_name} (id) VALUES (2000)")
+          end
+        end
+      end
+    end
+
+    assert_equal e.class, Lhm::Error
+    assert_match(/LHM tried the reconnection procedure but failed. Latest error: Reconnected to wrong host/, e.message)
+
+    logs = @logger.string.split("\n")
+
+    assert logs.first.include?("Lost connection to MySQL, will retry to connect to same host")
+    assert logs.last.include?("Lost connection to MySQL server at 'reading initial communication packet")
+  end
+end

data/spec/integration/toxiproxy_helper.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+require 'toxiproxy'
+
+module ToxiproxyHelper
+  class << self
+
+    def included(base)
+      Toxiproxy.reset
+
+      # listen on localhost, but toxiproxy is in a container itself, thus the upstream uses the Docker-Compose DNS
+      Toxiproxy.populate(
+        [
+          {
+            name: 'mysql_master',
+            listen: '0.0.0.0:22220',
+            upstream: 'mysql-1:3306'
+          },
+          {
+            name: 'mysql_proxysql',
+            listen: '0.0.0.0:22222',
+            upstream: 'proxysql:3306'
+          }
+        ])
+    end
+
+    def with_kill_and_restart(target, restart_after)
+      thread = Thread.new do
+        sleep(restart_after) unless restart_after.nil?
+        Toxiproxy[target].enable
+      end
+
+      Toxiproxy[target].disable
+
+      yield
+
+    ensure
+      thread.join
+    end
+  end
+end