lhm-shopify 3.4.2 → 3.5.1

data/scripts/proxysql/proxysql.cnf

@@ -0,0 +1,117 @@
+#file proxysql.cfg
+
+datadir="/var/lib/proxysql"
+restart_on_missing_heartbeats=999999
+query_parser_token_delimiters=","
+query_parser_key_value_delimiters=":"
+unit_of_work_identifiers="consistent_read_id"
+
+admin_variables=
+{
+  mysql_ifaces="0.0.0.0:6032"
+  admin_credentials="admin:password;remote-admin:password"
+}
+
+mysql_servers =
+(
+  {
+    address="mysql-1"
+    port=3306
+    hostgroup=0
+    max_connections=200
+  },
+  {
+    address="mysql-2"
+    port=3306
+    hostgroup=1
+    max_connections=200
+  }
+)
+
+mysql_variables=
+{
+  session_idle_ms=1
+    auto_increment_delay_multiplex=0
+
+    threads=8
+    max_connections=100000
+    interfaces="0.0.0.0:3306"
+    server_version="5.7.18-proxysql"
+    connect_timeout_server=10000
+    connect_timeout_server_max=10000
+    connect_retries_on_failure=0
+    default_charset="utf8mb4"
+    free_connections_pct=100
+    connection_warming=true
+    max_allowed_packet=16777216
+    monitor_enabled=false
+    query_retries_on_failure=0
+    shun_on_failures=999999
+    shun_recovery_time_sec=0
+    kill_backend_connection_when_disconnect=false
+    stats_time_backend_query=false
+    stats_time_query_processor=false
+    max_stmts_per_connection=5
+    default_max_latency_ms=999999
+    wait_timeout=1800000
+    eventslog_format=3
+    log_multiplexing_disabled=true
+    log_unhealthy_connections=false
+}
+
+# defines all the MySQL users
+mysql_users:
+(
+  {
+    username = "root"
+    password = "password"
+    default_hostgroup = 0
+    max_connections=1000
+    active = 1
+  },
+  {
+    username = "writer"
+    password = "password"
+    default_hostgroup = 0
+    max_connections=50000
+    active = 1
+    transaction_persistent=1
+  },
+  {
+    username = "reader"
+    password = "password"
+    default_hostgroup = 1
+    max_connections=50000
+    active = 1
+    transaction_persistent=1
+  }
+)
+
+#defines MySQL Query Rules
+mysql_query_rules:
+(
+  {
+      rule_id = 1
+      active = 1
+      match_digest = "@@SESSION"
+      multiplex = 2
+    },
+    {
+      rule_id = 2
+      active = 1
+      match_digest = "@@global\.server_id"
+      multiplex = 2
+    },
+    {
+      rule_id = 3
+      active = 1
+      match_digest = "@@global\.hostname"
+      multiplex = 2
+    },
+    {
+      rule_id = 4
+      active = 1
+      match_pattern = "maintenance:lhm"
+      destination_hostgroup = 0
+    }
+)

data/spec/integration/atomic_switcher_spec.rb

@@ -6,6 +6,7 @@ require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
 require 'lhm/table'
 require 'lhm/migration'
 require 'lhm/atomic_switcher'
+require 'lhm/connection'
 
 describe Lhm::AtomicSwitcher do
   include IntegrationHelper
@@ -15,9 +16,9 @@ describe Lhm::AtomicSwitcher do
   describe 'switching' do
     before(:each) do
       Thread.abort_on_exception = true
-      @origin      = table_create('origin')
+      @origin = table_create('origin')
       @destination = table_create('destination')
-      @migration   = Lhm::Migration.new(@origin, @destination)
+      @migration = Lhm::Migration.new(@origin, @destination)
       @logs = StringIO.new
       Lhm.logger = Logger.new(@logs)
       @connection.execute('SET GLOBAL innodb_lock_wait_timeout=3')
@@ -29,26 +30,48 @@ describe Lhm::AtomicSwitcher do
     end
 
     it 'should retry and log on lock wait timeouts' do
-      connection = mock()
-      connection.stubs(:data_source_exists?).returns(true)
-      connection.stubs(:execute).raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.').then.returns(true)
+      ar_connection = mock()
+      ar_connection.stubs(:data_source_exists?).returns(true)
+      ar_connection.stubs(:active?).returns(true)
+      ar_connection.stubs(:execute).returns([["dummy"]], [["dummy"]], [["dummy"]])
+                   .then
+                   .raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
+                   .then
+                   .returns([["dummy"]]) # Matches initial host -> triggers retry
 
-      switcher = Lhm::AtomicSwitcher.new(@migration, connection, retriable: {base_interval: 0})
+      connection = Lhm::Connection.new(connection: ar_connection, options: {reconnect_with_consistent_host: true})
+
+
+      switcher = Lhm::AtomicSwitcher.new(@migration, connection, retriable: { tries: 3, base_interval: 0 })
 
       assert switcher.run
 
       log_messages = @logs.string.split("\n")
       assert_equal(2, log_messages.length)
       assert log_messages[0].include? "Starting run of class=Lhm::AtomicSwitcher"
+      # On failure of this assertion, check for Lhm::Connection#file
       assert log_messages[1].include? "[AtomicSwitcher] ActiveRecord::StatementInvalid: 'Lock wait timeout exceeded; try restarting transaction.' - 1 tries"
     end
 
     it 'should give up on lock wait timeouts after a configured number of tries' do
-      connection = mock()
-      connection.stubs(:data_source_exists?).returns(true)
-      connection.stubs(:execute).twice.raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
-
-      switcher = Lhm::AtomicSwitcher.new(@migration, connection, retriable: {tries: 2, base_interval: 0})
+      ar_connection = mock()
+      ar_connection.stubs(:data_source_exists?).returns(true)
+      ar_connection.stubs(:active?).returns(true)
+      ar_connection.stubs(:execute).returns([["dummy"]], [["dummy"]], [["dummy"]])
+                .then
+                .raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
+                .then
+                .returns([["dummy"]]) # triggers retry 1
+                .then
+                .raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
+                .then
+                .returns([["dummy"]]) # triggers retry 2
+                .then
+                .raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.') # triggers retry 2
+
+      connection = Lhm::Connection.new(connection: ar_connection, options: {reconnect_with_consistent_host: true})
+
+      switcher = Lhm::AtomicSwitcher.new(@migration, connection, retriable: { tries: 2, base_interval: 0 })
 
       assert_raises(ActiveRecord::StatementInvalid) { switcher.run }
     end
@@ -62,8 +85,10 @@ describe Lhm::AtomicSwitcher do
     end
 
     it "should raise when destination doesn't exist" do
-      connection = mock()
-      connection.stubs(:data_source_exists?).returns(false)
+      ar_connection = mock()
+      ar_connection.stubs(:data_source_exists?).returns(false)
+
+      connection = Lhm::Connection.new(connection: ar_connection)
 
       switcher = Lhm::AtomicSwitcher.new(@migration, connection)
 

data/spec/integration/chunk_insert_spec.rb

@@ -11,7 +11,8 @@ describe Lhm::ChunkInsert do
       @destination = table_create(:destination)
       @migration = Lhm::Migration.new(@origin, @destination)
       execute("insert into origin set id = 1001")
-      @instance = Lhm::ChunkInsert.new(@migration, connection, 1001, 1001)
+      @connection = Lhm::Connection.new(connection: connection)
+      @instance = Lhm::ChunkInsert.new(@migration, @connection, 1001, 1001)
     end
 
     it "returns the count" do

data/spec/integration/chunker_spec.rb

@@ -223,7 +223,7 @@ describe Lhm::Chunker do
 
       if master_slave_mode?
         def throttler.slave_connection(slave)
-          config = ActiveRecord::Base.connection_pool.spec.config.dup
+          config = ActiveRecord::Base.connection_pool.db_config.configuration_hash.dup
           config[:host] = slave
           config[:port] = 3307
           ActiveRecord::Base.send('mysql2_connection', config)

data/spec/integration/database.yml

@@ -0,0 +1,25 @@
+master:
+  host: mysql-1
+  user: root
+  password: password
+  port: 33006
+slave:
+  host: mysql-2
+  user: root
+  password: password
+  port: 33007
+proxysql:
+  host: proxysql
+  user: root
+  password: password
+  port: 33005
+master_toxic:
+  host: toxiproxy
+  user: root
+  password: password
+  port: 22220
+proxysql_toxic:
+  host: toxiproxy
+  user: root
+  password: password
+  port: 22222

data/spec/integration/entangler_spec.rb

@@ -6,6 +6,7 @@ require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
 require 'lhm/table'
 require 'lhm/migration'
 require 'lhm/entangler'
+require 'lhm/connection'
 
 describe Lhm::Entangler do
   include IntegrationHelper
@@ -17,7 +18,8 @@ describe Lhm::Entangler do
       @origin = table_create('origin')
       @destination = table_create('destination')
       @migration = Lhm::Migration.new(@origin, @destination)
-      @entangler = Lhm::Entangler.new(@migration, connection)
+      @connection = Lhm::Connection.new(connection: connection)
+      @entangler = Lhm::Entangler.new(@migration, @connection)
     end
 
     it 'should replay inserts from origin into destination' do

data/spec/integration/integration_helper.rb

@@ -35,13 +35,21 @@ module IntegrationHelper
     @connection
   end
 
+  def connect_proxysql!
+    connect!(
+      '127.0.0.1',
+      $db_config['proxysql']['port'],
+      $db_config['proxysql']['user'],
+      $db_config['proxysql']['password'],
+    )
+  end
+
   def connect_master!
     connect!(
       '127.0.0.1',
       $db_config['master']['port'],
       $db_config['master']['user'],
       $db_config['master']['password'],
-      $db_config['master']['socket']
     )
   end
 
@@ -51,28 +59,35 @@ module IntegrationHelper
       $db_config['slave']['port'],
       $db_config['slave']['user'],
       $db_config['slave']['password'],
-      $db_config['slave']['socket']
     )
   end
 
-  def connect!(hostname, port, user, password, socket)
-    adapter = ar_conn(hostname, port, user, password, socket)
-    Lhm.setup(adapter)
+  def connect_master_with_toxiproxy!(with_retry: false)
+    connect!(
+      '127.0.0.1',
+      $db_config['master_toxic']['port'],
+      $db_config['master_toxic']['user'],
+      $db_config['master_toxic']['password'],
+      with_retry)
+  end
+
+  def connect!(hostname, port, user, password, with_retry = false)
+    adapter = ar_conn(hostname, port, user, password)
+    Lhm.setup(adapter,{reconnect_with_consistent_host: with_retry})
     unless defined?(@@cleaned_up)
       Lhm.cleanup(true)
       @@cleaned_up  = true
     end
-    @connection = adapter
+    @connection = Lhm.connection
   end
 
-  def ar_conn(host, port, user, password, socket)
+  def ar_conn(host, port, user, password)
     ActiveRecord::Base.establish_connection(
       :adapter  => 'mysql2',
       :host     => host,
       :username => user,
       :port     => port,
       :password => password,
-      :socket   => socket,
       :database => $db_name
     )
     ActiveRecord::Base.connection
@@ -122,7 +137,7 @@ module IntegrationHelper
   # Helps testing behaviour when another client locks the db
   def start_locking_thread(lock_for, queue, locking_query)
     Thread.new do
-      conn = Mysql2::Client.new(host: '127.0.0.1', database: $db_name, user: 'root', port: 3306)
+      conn = new_mysql_connection
       conn.query('BEGIN')
       conn.query(locking_query)
       queue.push(true)

data/spec/integration/lhm_spec.rb

@@ -2,6 +2,7 @@
 # Schmidt
 
 require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
+require 'integration/toxiproxy_helper'
 
 describe Lhm do
   include IntegrationHelper
@@ -581,5 +582,79 @@ describe Lhm do
         end
       end
     end
+
+    describe 'connection' do
+      include ToxiproxyHelper
+
+      before(:each) do
+        @logs = StringIO.new
+        Lhm.logger = Logger.new(@logs)
+      end
+
+      it " should not try to reconnect if reconnect_with_consistent_host is not provided" do
+        connect_master_with_toxiproxy!(with_retry: false)
+
+        table_create(:users)
+        100.times { |n| execute("insert into users set reference = '#{ n }'") }
+
+        assert_raises ActiveRecord::StatementInvalid do
+          Toxiproxy[:mysql_master].down do
+            Lhm.change_table(:users, :atomic_switch => false) do |t|
+              t.ddl("ALTER TABLE #{t.name} CHANGE id id bigint (20) NOT NULL")
+              t.ddl("ALTER TABLE #{t.name} DROP PRIMARY KEY, ADD PRIMARY KEY (username, id)")
+              t.ddl("ALTER TABLE #{t.name} ADD INDEX (id)")
+              t.ddl("ALTER TABLE #{t.name} CHANGE id id bigint (20) NOT NULL AUTO_INCREMENT")
+            end
+          end
+        end
+      end
+
+      it "should reconnect if reconnect_with_consistent_host is true" do
+        connect_master_with_toxiproxy!(with_retry: true)
+        mysql_disabled = false
+
+        table_create(:users)
+        100.times { |n| execute("insert into users set reference = '#{ n }'") }
+
+        # Redeclare Lhm::ChunkInsert to use Hook to disable MySQL writer for 3 seconds before first insert
+        Lhm::ChunkInsert.class_eval do
+          extend AfterDo
+
+          before(:insert_and_return_count_of_rows_created) do
+            unless mysql_disabled
+              mysql_disabled = true
+              Thread.new do
+                Toxiproxy[:mysql_master].down do
+                  sleep 3
+                end
+              end
+            end
+          end
+
+          # Need to call `#method_added` manually to have the hooks take into effect
+          method_added(:insert_and_return_count_of_rows_created)
+        end
+
+        Lhm.change_table(:users, :atomic_switch => false) do |t|
+          t.ddl("ALTER TABLE #{t.name} CHANGE id id bigint (20) NOT NULL")
+          t.ddl("ALTER TABLE #{t.name} DROP PRIMARY KEY, ADD PRIMARY KEY (username, id)")
+          t.ddl("ALTER TABLE #{t.name} ADD INDEX (id)")
+          t.ddl("ALTER TABLE #{t.name} CHANGE id id bigint (20) NOT NULL AUTO_INCREMENT")
+        end
+
+        log_lines = @logs.string.split("\n")
+
+        assert log_lines.one?{ |line| line.include?("Lost connection to MySQL, will retry to connect to same host")}
+        assert log_lines.any?{ |line| line.include?("Lost connection to MySQL server at 'reading initial communication packet'")}
+        assert log_lines.one?{ |line| line.include?("LHM successfully reconnected to initial host")}
+        assert log_lines.one?{ |line| line.include?("100% complete")}
+
+        Lhm::ChunkInsert.remove_all_callbacks
+
+        slave do
+          value(count_all(:users)).must_equal(100)
+        end
+      end
+    end
   end
 end

data/spec/integration/proxysql_spec.rb

@@ -0,0 +1,34 @@
+describe "ProxySQL integration" do
+  it "Should contact the writer" do
+    conn = Mysql2::Client.new(
+      host: '127.0.0.1',
+      username: "writer",
+      password: "password",
+      port: "33005",
+      )
+
+    assert_equal conn.query("SELECT @@global.hostname as host").each.first["host"], "mysql-1"
+  end
+
+  it "Should contact the reader" do
+    conn = Mysql2::Client.new(
+      host: '127.0.0.1',
+      username: "reader",
+      password: "password",
+      port: "33005",
+      )
+
+    assert_equal conn.query("SELECT @@global.hostname as host").each.first["host"], "mysql-2"
+  end
+
+  it "Should override default hostgroup from user if rule matches" do
+    conn = Mysql2::Client.new(
+      host: '127.0.0.1',
+      username: "reader",
+      password: "password",
+      port: "33005",
+      )
+
+    assert_equal conn.query("/*maintenance:lhm*/SELECT @@global.hostname as host").each.first["host"], "mysql-1"
+  end
+end

data/spec/integration/sql_retry/db_connection_helper.rb

@@ -0,0 +1,52 @@
+require 'yaml'
+require 'mysql2'
+
+class DBConnectionHelper
+
+  DATABASE_CONFIG_FILE = "database.yml"
+
+  class << self
+    def db_config
+      @db_config ||= YAML.load_file(File.expand_path(File.dirname(__FILE__)) + "/../#{DATABASE_CONFIG_FILE}")
+    end
+
+    def new_mysql_connection(role = :master, with_data = false, toxic = false)
+
+      key = role.to_s + toxic_postfix(toxic)
+
+      conn = ActiveRecord::Base.establish_connection(
+        :host => '127.0.0.1',
+        :adapter => "mysql2",
+        :username => db_config[key]['user'],
+        :password => db_config[key]['password'],
+        :database => test_db_name,
+        :port => db_config[key]['port']
+      )
+      conn = conn.connection
+      init_with_dummy_data(conn) if with_data
+      conn
+    end
+
+    def toxic_postfix(toxic)
+      toxic ? "_toxic" : ""
+    end
+
+    def test_db_name
+      @test_db_name ||= "test"
+    end
+
+    def test_table_name
+      @test_table_name ||= "test"
+    end
+
+    def init_with_dummy_data(conn)
+      conn.execute("DROP TABLE IF EXISTS #{test_table_name} ")
+      conn.execute("CREATE TABLE #{test_table_name} (id int)")
+
+      1.upto(9) do |i|
+        query = "INSERT INTO #{test_table_name} (id) VALUE (#{i})"
+        conn.execute(query)
+      end
+    end
+  end
+end

data/spec/integration/sql_retry/lock_wait_spec.rb

@@ -21,6 +21,8 @@ describe Lhm::SqlRetry do
 
     # Assert our pre-conditions
     assert_equal 2, @helper.record_count
+
+    Mysql2::Client.any_instance.stubs(:active?).returns(true)
   end
 
   after(:each) do
@@ -41,7 +43,7 @@ describe Lhm::SqlRetry do
 
     exception = assert_raises { @helper.trigger_wait_lock }
 
-    assert_equal "Lock wait timeout exceeded; try restarting transaction", exception.message
+    assert_match /Lock wait timeout exceeded; try restarting transaction/, exception.message
     assert_equal Mysql2::Error::TimeoutError, exception.class
 
     assert_equal 2, @helper.record_count # no records inserted
@@ -53,12 +55,12 @@ describe Lhm::SqlRetry do
   it "successfully executes the SQL despite the errors encountered" do
     # Start a thread to retry, once the lock is held, execute the block
     @helper.with_waiting_lock do |waiting_connection|
-      sql_retry = Lhm::SqlRetry.new(waiting_connection, {
+      sql_retry = Lhm::SqlRetry.new(waiting_connection, options: {
         base_interval: 0.2, # first retry after 200ms
         multiplier: 1, # subsequent retries wait 1x longer than first retry (no change)
         tries: 3, # we only need 3 tries (including the first) for the scenario described below
         rand_factor: 0 # do not introduce randomness to wait timer
-      })
+      }, reconnect_with_consistent_host: false)
 
       # RetryTestHelper is configured to hold lock for 5 seconds and timeout after 2 seconds.
       # Therefore the sequence of events will be:
@@ -96,12 +98,12 @@ describe Lhm::SqlRetry do
     puts "*" * 64
     # Start a thread to retry, once the lock is held, execute the block
     @helper.with_waiting_lock do |waiting_connection|
-      sql_retry = Lhm::SqlRetry.new(waiting_connection, {
+      sql_retry = Lhm::SqlRetry.new(waiting_connection, options: {
         base_interval: 0.2, # first retry after 200ms
         multiplier: 1, # subsequent retries wait 1x longer than first retry (no change)
         tries: 2, # we need 3 tries (including the first) for the scenario described below, but we only get two...we will fail
         rand_factor: 0 # do not introduce randomness to wait timer
-      })
+      }, reconnect_with_consistent_host: false)
 
       # RetryTestHelper is configured to hold lock for 5 seconds and timeout after 2 seconds.
       # Therefore the sequence of events will be:
@@ -116,7 +118,7 @@ describe Lhm::SqlRetry do
 
     exception = assert_raises { @helper.trigger_wait_lock }
 
-    assert_equal "Lock wait timeout exceeded; try restarting transaction", exception.message
+    assert_match /Lock wait timeout exceeded; try restarting transaction/, exception.message
     assert_equal Mysql2::Error::TimeoutError, exception.class
 
     assert_equal 2, @helper.record_count # no records inserted

data/spec/integration/sql_retry/lock_wait_timeout_test_helper.rb

@@ -1,5 +1,7 @@
-require 'yaml'
+require 'integration/integration_helper'
+
 class LockWaitTimeoutTestHelper
+
   def initialize(lock_duration:, innodb_lock_wait_timeout:)
     # This connection will be used exclusively to setup the test,
     # assert pre-conditions and assert post-conditions.
@@ -68,7 +70,7 @@ class LockWaitTimeoutTestHelper
 
   def insert_records_at_ids(connection, ids)
     ids.each do |id|
-      connection.query "INSERT INTO #{test_table_name} (id) VALUES (#{id})"
+      mysql_exec(connection, "INSERT INTO #{test_table_name} (id) VALUES (#{id})")
     end
   end
 
@@ -79,11 +81,10 @@ class LockWaitTimeoutTestHelper
   def new_mysql_connection
     Mysql2::Client.new(
       host: '127.0.0.1',
-      database: test_db_name,
       username: db_config['master']['user'],
       password: db_config['master']['password'],
       port: db_config['master']['port'],
-      socket: db_config['master']['socket']
+      database: test_db_name
     )
   end
 
@@ -98,4 +99,16 @@ class LockWaitTimeoutTestHelper
   def test_table_name
     @test_table_name ||= "lock_wait"
   end
+
+  private
+
+  def mysql_exec(connection, statement)
+    if connection.class == Mysql2::Client
+      connection.query(statement)
+    elsif connection.class.to_s.include?("ActiveRecord")
+      connection.execute(statement)
+    else
+      raise StandardError.new("Unrecognized MySQL client")
+    end
+  end
 end

data/spec/integration/sql_retry/proxysql_helper.rb

@@ -0,0 +1,22 @@
+class ProxySQLHelper
+  class << self
+    # Flips the destination hostgroup for /maintenance:lhm/ from 0 (i.e. writer) to 1 (i.e. reader)
+    def with_lhm_hostgroup_flip
+      conn = Mysql2::Client.new(
+        host: '127.0.0.1',
+        username: "remote-admin",
+        password: "password",
+        port: "6032",
+      )
+
+      begin
+        conn.query("UPDATE mysql_query_rules SET destination_hostgroup=1 WHERE match_pattern=\"maintenance:lhm\"")
+        conn.query("LOAD MYSQL QUERY RULES TO RUNTIME;")
+        yield
+      ensure
+        conn.query("UPDATE mysql_query_rules SET destination_hostgroup=0 WHERE match_pattern=\"maintenance:lhm\"")
+        conn.query("LOAD MYSQL QUERY RULES TO RUNTIME;")
+      end
+    end
+  end
+end