lhm-shopify 3.5.5 → 4.1.0

data/spec/integration/integration_helper.rb

@@ -22,7 +22,7 @@ module IntegrationHelper
   def self.included(base)
     base.after(:each) do
       cleanup_connection = new_mysql_connection
-      results = cleanup_connection.query("SELECT table_name FROM information_schema.tables WHERE table_schema = '#{$db_name}';")
+      results = DATABASE.query(cleanup_connection, "SELECT table_name FROM information_schema.tables WHERE table_schema = '#{$db_name}';")
       table_names_for_cleanup = results.map { |row| "#{$db_name}." + row.values.first }
       cleanup_connection.query("DROP TABLE IF EXISTS #{table_names_for_cleanup.join(', ')};") if table_names_for_cleanup.length > 0
     end
@@ -35,6 +35,14 @@ module IntegrationHelper
     @connection
   end
 
+  def mysql_version
+    @mysql_version ||= begin
+      # This SQL returns a value of shape: X.Y.ZZ-AA-log
+      result = connection.query("SELECT VERSION()")
+      result.dig(0, 0).split("-", 2)[0]
+    end
+  end
+
   def connect_proxysql!
     connect!(
       '127.0.0.1',
@@ -53,12 +61,12 @@ module IntegrationHelper
     )
   end
 
-  def connect_slave!
+  def connect_replica!
     connect!(
       '127.0.0.1',
-      $db_config['slave']['port'],
-      $db_config['slave']['user'],
-      $db_config['slave']['password'],
+      $db_config['replica']['port'],
+      $db_config['replica']['user'],
+      $db_config['replica']['password'],
     )
   end
 
@@ -81,7 +89,7 @@ module IntegrationHelper
 
   def ar_conn(host, port, user, password)
     ActiveRecord::Base.establish_connection(
-      :adapter  => 'mysql2',
+      :adapter  => DATABASE.adapter,
       :host     => host,
       :username => user,
       :port     => port,
@@ -113,12 +121,12 @@ module IntegrationHelper
     end
   end
 
-  def slave(&block)
-    if master_slave_mode?
-      connect_slave!
+  def replica(&block)
+    if master_replica_mode?
+      connect_replica!
 
-      # need to wait for the slave to catch up. a better method would be to
-      # check the master binlog position and wait for the slave to catch up
+      # need to wait for the replica to catch up. a better method would be to
+      # check the master binlog position and wait for the replica to catch up
       # to that position.
       sleep 1
     else
@@ -127,7 +135,7 @@ module IntegrationHelper
 
     yield block
 
-    if master_slave_mode?
+    if master_replica_mode?
       connect_master!
     end
   end
@@ -171,7 +179,7 @@ module IntegrationHelper
   end
 
   def new_mysql_connection(role='master')
-    Mysql2::Client.new(
+    DATABASE.client.new(
       host: '127.0.0.1',
       database: $db_name,
       username: $db_config[role]['user'],
@@ -215,8 +223,8 @@ module IntegrationHelper
   # Environment
   #
 
-  def master_slave_mode?
-    !!ENV['MASTER_SLAVE']
+  def master_replica_mode?
+    !!ENV['MASTER_REPLICA']
   end
 
   #

data/spec/integration/lhm_spec.rb

@@ -9,6 +9,10 @@ describe Lhm do
 
   before(:each) { connect_master!; Lhm.cleanup(true) }
 
+  let(:collation) do
+    mysql_version.start_with?("8.0") ? "utf8mb3_general_ci" : "utf8_general_ci"
+  end
+
   describe 'id column requirement' do
     it 'should migrate the table when id is pk' do
       table_create(:users)
@@ -17,9 +21,11 @@ describe Lhm do
         t.add_column(:logins, "int(12) default '0'")
       end
 
-      slave do
+      expected_type = mysql_version.start_with?("8.0") ? "int" : "int(12)"
+
+      replica do
         value(table_read(:users).columns['logins']).must_equal({
-          :type           => 'int(12)',
+          :type           => expected_type,
           :is_nullable    => 'YES',
           :column_default => '0',
           :comment => '',
@@ -35,9 +41,11 @@ describe Lhm do
         t.add_column(:logins, "int(12) default '0'")
       end
 
-      slave do
+      expected_type = mysql_version.start_with?("8.0") ? "int" : "int(12)"
+
+      replica do
         value(table_read(:custom_primary_key).columns['logins']).must_equal({
-          :type           => 'int(12)',
+          :type           => expected_type,
           :is_nullable    => 'YES',
           :column_default => '0',
           :comment => '',
@@ -53,9 +61,11 @@ describe Lhm do
         t.add_column(:logins, "int(12) default '0'")
       end
 
-      slave do
+      expected_type = mysql_version.start_with?("8.0") ? "int" : "int(12)"
+
+      replica do
         value(table_read(:composite_primary_key).columns['logins']).must_equal({
-          :type           => 'int(12)',
+          :type           => expected_type,
           :is_nullable    => 'YES',
           :column_default => '0',
           :comment => '',
@@ -82,7 +92,7 @@ describe Lhm do
           t.ddl("ALTER TABLE #{t.name} CHANGE id id bigint (20) NOT NULL AUTO_INCREMENT")
         end
 
-        slave do
+        replica do
           value(connection.primary_key('users')).must_equal(['username', 'id'])
         end
       end
@@ -104,7 +114,7 @@ describe Lhm do
       describe 'when no additional data is inserted into the table' do
 
         it 'migrates the existing data' do
-          slave do
+          replica do
             value(count_all(:permissions)).must_equal(11)
           end
         end
@@ -120,7 +130,7 @@ describe Lhm do
         end
 
         it 'migrates all data' do
-          slave do
+          replica do
             value(count_all(:permissions)).must_equal(13)
           end
         end
@@ -132,9 +142,11 @@ describe Lhm do
         t.add_column(:logins, "INT(12) DEFAULT '0'")
       end
 
-      slave do
+      expected_type = mysql_version.start_with?("8.0") ? "int" : "int(12)"
+
+      replica do
         value(table_read(:users).columns['logins']).must_equal({
-          :type => 'int(12)',
+          :type => expected_type,
           :is_nullable => 'YES',
           :column_default => '0',
           :comment => '',
@@ -150,7 +162,7 @@ describe Lhm do
         t.add_column(:logins, "INT(12) DEFAULT '0'")
       end
 
-      slave do
+      replica do
         value(count_all(:users)).must_equal(23)
       end
     end
@@ -160,7 +172,7 @@ describe Lhm do
         t.remove_column(:comment)
       end
 
-      slave do
+      replica do
         assert_nil table_read(:users).columns['comment']
       end
     end
@@ -170,7 +182,7 @@ describe Lhm do
         t.add_index([:comment, :created_at])
       end
 
-      slave do
+      replica do
         value(index_on_columns?(:users, [:comment, :created_at])).must_equal(true)
       end
     end
@@ -180,7 +192,7 @@ describe Lhm do
         t.add_index([:comment, :created_at], :my_index_name)
       end
 
-      slave do
+      replica do
         value(index?(:users, :my_index_name)).must_equal(true)
       end
     end
@@ -190,7 +202,7 @@ describe Lhm do
         t.add_index(:group)
       end
 
-      slave do
+      replica do
         value(index_on_columns?(:users, :group)).must_equal(true)
       end
     end
@@ -200,7 +212,7 @@ describe Lhm do
         t.add_unique_index(:comment)
       end
 
-      slave do
+      replica do
         value(index_on_columns?(:users, :comment, :unique)).must_equal(true)
       end
     end
@@ -210,7 +222,7 @@ describe Lhm do
         t.remove_index([:username, :created_at])
       end
 
-      slave do
+      replica do
         value(index_on_columns?(:users, [:username, :created_at])).must_equal(false)
       end
     end
@@ -220,7 +232,7 @@ describe Lhm do
         t.remove_index([:username, :group])
       end
 
-      slave do
+      replica do
         value(index?(:users, :index_with_a_custom_name)).must_equal(false)
       end
     end
@@ -230,17 +242,27 @@ describe Lhm do
         t.remove_index(:irrelevant_column_name, :index_with_a_custom_name)
       end
 
-      slave do
+      replica do
         value(index?(:users, :index_with_a_custom_name)).must_equal(false)
       end
     end
 
+    it 'should add an index with column sizes' do
+      Lhm.change_table(:users, :atomic_switch => false) do |t|
+        t.add_index(["username(6)", "group (10)", "comment     (10)"])
+      end
+
+      replica do
+        value(index_on_columns?(:users, [:username, :group, :comment])).must_equal(true)
+      end
+    end
+
     it 'should apply a ddl statement' do
       Lhm.change_table(:users, :atomic_switch => false) do |t|
         t.ddl('alter table %s add column flag tinyint(1)' % t.name)
       end
 
-      slave do
+      replica do
         value(table_read(:users).columns['flag']).must_equal({
           :type => 'tinyint(1)',
           :is_nullable => 'YES',
@@ -256,13 +278,13 @@ describe Lhm do
         t.change_column(:comment, "varchar(20) DEFAULT 'none' NOT NULL")
       end
 
-      slave do
+      replica do
         value(table_read(:users).columns['comment']).must_equal({
           :type => 'varchar(20)',
           :is_nullable => 'NO',
           :column_default => 'none',
           :comment => '',
-          :collate => 'utf8_general_ci',
+          :collate => collation,
         })
       end
     end
@@ -274,9 +296,11 @@ describe Lhm do
         t.change_column(:id, 'int(5)')
       end
 
-      slave do
+      expected_type = mysql_version.start_with?("8.0") ? "int" : "int(5)"
+
+      replica do
         value(table_read(:small_table).columns['id']).must_equal({
-          :type => 'int(5)',
+          :type => expected_type,
           :is_nullable => 'NO',
           :column_default => nil,
           :comment => '',
@@ -293,7 +317,7 @@ describe Lhm do
         t.rename_column(:username, :login)
       end
 
-      slave do
+      replica do
         table_data = table_read(:users)
         assert_nil table_data.columns['username']
         value(table_read(:users).columns['login']).must_equal({
@@ -301,7 +325,7 @@ describe Lhm do
           :is_nullable => 'YES',
           :column_default => nil,
           :comment => '',
-          :collate => 'utf8_general_ci',
+          :collate => collation,
         })
 
         result = select_one('SELECT login from users')
@@ -318,7 +342,7 @@ describe Lhm do
         t.rename_column(:group, :fnord)
       end
 
-      slave do
+      replica do
         table_data = table_read(:users)
         assert_nil table_data.columns['group']
         value(table_read(:users).columns['fnord']).must_equal({
@@ -326,7 +350,7 @@ describe Lhm do
           :is_nullable => 'YES',
           :column_default => 'Superfriends',
           :comment => '',
-          :collate => 'utf8_general_ci',
+          :collate => collation,
         })
 
         result = select_one('SELECT `fnord` from users')
@@ -345,7 +369,7 @@ describe Lhm do
         t.rename_column(:username, :user_name)
       end
 
-      slave do
+      replica do
         table_data = table_read(:users)
         assert_nil table_data.columns['username']
         value(table_read(:users).columns['user_name']).must_equal({
@@ -373,11 +397,13 @@ describe Lhm do
         t.rename_column(:reference, :ref)
       end
 
-      slave do
+      expected_type = mysql_version.start_with?("8.0") ? "int" : "int(11)"
+
+      replica do
         table_data = table_read(:users)
         assert_nil table_data.columns['reference']
         value(table_read(:users).columns['ref']).must_equal({
-           :type => 'int(11)',
+           :type => expected_type,
            :is_nullable => 'YES',
            :column_default => nil,
            :comment => 'RefComment',
@@ -400,7 +426,7 @@ describe Lhm do
         t.rename_column(:group, :fnord)
       end
 
-      slave do
+      replica do
         table_data = table_read(:users)
         assert_nil table_data.columns['group']
         value(table_read(:users).columns['fnord']).must_equal({
@@ -408,7 +434,7 @@ describe Lhm do
            :is_nullable => 'YES',
            :column_default => nil,
            :comment => '',
-           :collate => 'utf8_general_ci',
+           :collate => collation,
          })
 
         result = select_one('SELECT `fnord` from users')
@@ -425,7 +451,7 @@ describe Lhm do
         t.rename_column(:username, :user_name)
       end
 
-      slave do
+      replica do
         table_data = table_read(:users)
         assert_nil table_data.columns['username']
         value(table_read(:users).columns['user_name']).must_equal({
@@ -433,7 +459,7 @@ describe Lhm do
           :is_nullable => 'YES',
           :column_default => nil,
           :comment => '',
-          :collate => 'utf8_general_ci',
+          :collate => collation,
         })
 
         result = select_one('SELECT `user_name` from users')
@@ -452,7 +478,7 @@ describe Lhm do
         t.rename_column(:username, :user_name)
       end
 
-      slave do
+      replica do
         table_data = table_read(:users)
         assert_nil table_data.columns['username']
         value(table_read(:users).columns['user_name']).must_equal({
@@ -460,7 +486,7 @@ describe Lhm do
           :is_nullable => 'NO',
           :column_default => nil,
           :comment => '',
-          :collate => 'utf8_general_ci',
+          :collate => collation,
         })
 
         result = select_one('SELECT `user_name` from users')
@@ -499,7 +525,7 @@ describe Lhm do
         end
       end
 
-      slave do
+      replica do
         table_data = table_read(:users)
         assert_nil table_data.columns['fnord']
         value(table_read(:users).columns['group']).must_equal({
@@ -507,7 +533,7 @@ describe Lhm do
           :is_nullable => 'YES',
           :column_default => 'Superfriends',
           :comment => '',
-          :collate => 'utf8_general_ci',
+          :collate => collation,
         })
       end
     end
@@ -525,7 +551,7 @@ describe Lhm do
         t.remove_index('by')
       end
 
-      slave do
+      replica do
         value(table_read(:lines).columns).must_include 'by'
         value(table_read(:lines).columns).wont_include 'lines'
         value(index_on_columns?(:lines, ['between'], :unique)).must_equal true
@@ -554,7 +580,7 @@ describe Lhm do
 
         insert.join
 
-        slave do
+        replica do
           value(count_all(:users)).must_equal(60)
         end
       end
@@ -577,7 +603,7 @@ describe Lhm do
 
         delete.join
 
-        slave do
+        replica do
           value(count_all(:users)).must_equal(40)
         end
       end
@@ -650,7 +676,7 @@ describe Lhm do
 
         Lhm::ChunkInsert.remove_all_callbacks
 
-        slave do
+        replica do
           value(count_all(:users)).must_equal(100)
         end
       end

data/spec/integration/locked_switcher_spec.rb

@@ -31,7 +31,7 @@ describe Lhm::LockedSwitcher do
       switcher = Lhm::LockedSwitcher.new(@migration, connection)
       switcher.run
 
-      slave do
+      replica do
         value(data_source_exists?(@origin)).must_equal true
         value(table_read(@migration.archive_name).columns.keys).must_include 'origin'
       end
@@ -41,7 +41,7 @@ describe Lhm::LockedSwitcher do
       switcher = Lhm::LockedSwitcher.new(@migration, connection)
       switcher.run
 
-      slave do
+      replica do
         value(data_source_exists?(@destination)).must_equal false
         value(table_read(@origin.name).columns.keys).must_include 'destination'
       end

data/spec/integration/proxysql_spec.rb

@@ -1,34 +1,34 @@
 describe "ProxySQL integration" do
   it "Should contact the writer" do
-    conn = Mysql2::Client.new(
+    conn = DATABASE.client.new(
       host: '127.0.0.1',
       username: "writer",
       password: "password",
-      port: "33005",
+      port: "13005",
       )
 
-    assert_equal conn.query("SELECT @@global.hostname as host").each.first["host"], "mysql-1"
+    assert_equal DATABASE.query(conn, "SELECT @@global.hostname as host").each.first["host"], "mysql-1"
   end
 
   it "Should contact the reader" do
-    conn = Mysql2::Client.new(
+    conn = DATABASE.client.new(
       host: '127.0.0.1',
       username: "reader",
       password: "password",
-      port: "33005",
+      port: "13005",
       )
 
-    assert_equal conn.query("SELECT @@global.hostname as host").each.first["host"], "mysql-2"
+    assert_equal DATABASE.query(conn, "SELECT @@global.hostname as host").each.first["host"], "mysql-2"
   end
 
   it "Should override default hostgroup from user if rule matches" do
-    conn = Mysql2::Client.new(
+    conn = DATABASE.client.new(
       host: '127.0.0.1',
       username: "reader",
       password: "password",
-      port: "33005",
+      port: "13005",
       )
 
-    assert_equal conn.query("SELECT @@global.hostname as host #{Lhm::ProxySQLHelper::ANNOTATION}").each.first["host"], "mysql-1"
+    assert_equal DATABASE.query(conn, "SELECT @@global.hostname as host #{Lhm::ProxySQLHelper::ANNOTATION}").each.first["host"], "mysql-1"
   end
-end
+end

data/spec/integration/sql_retry/db_connection_helper.rb

@@ -1,5 +1,4 @@
 require 'yaml'
-require 'mysql2'
 
 class DBConnectionHelper
 
@@ -11,12 +10,11 @@ class DBConnectionHelper
     end
 
     def new_mysql_connection(role = :master, with_data = false, toxic = false)
-
       key = role.to_s + toxic_postfix(toxic)
 
       conn = ActiveRecord::Base.establish_connection(
         :host => '127.0.0.1',
-        :adapter => "mysql2",
+        :adapter => DATABASE.adapter,
         :username => db_config[key]['user'],
         :password => db_config[key]['password'],
         :database => test_db_name,
@@ -49,4 +47,4 @@ class DBConnectionHelper
       end
     end
   end
-end
+end

data/spec/integration/sql_retry/lock_wait_spec.rb

@@ -1,5 +1,4 @@
 require 'minitest/autorun'
-require 'mysql2'
 require 'integration/sql_retry/lock_wait_timeout_test_helper'
 require 'lhm'
 
@@ -22,7 +21,7 @@ describe Lhm::SqlRetry do
     # Assert our pre-conditions
     assert_equal 2, @helper.record_count
 
-    Mysql2::Client.any_instance.stubs(:active?).returns(true)
+    DATABASE.client.any_instance.stubs(:active?).returns(true)
   end
 
   after(:each) do
@@ -43,8 +42,8 @@ describe Lhm::SqlRetry do
 
     exception = assert_raises { @helper.trigger_wait_lock }
 
-    assert_match /Lock wait timeout exceeded; try restarting transaction/, exception.message
-    assert_equal Mysql2::Error::TimeoutError, exception.class
+    assert_match Regexp.new("Lock wait timeout exceeded; try restarting transaction"), exception.message
+    assert_equal DATABASE.timeout_error, exception.class
 
     assert_equal 2, @helper.record_count # no records inserted
     puts "*" * 64
@@ -82,10 +81,10 @@ describe Lhm::SqlRetry do
     logs = @logger.string.split("\n")
     assert_equal 2, logs.length
 
-    assert logs.first.include?("Mysql2::Error::TimeoutError: 'Lock wait timeout exceeded; try restarting transaction' - 1 tries")
+    assert logs.first.include?("Lock wait timeout exceeded; try restarting transaction' - 1 tries")
     assert logs.first.include?("0.2 seconds until the next try")
 
-    assert logs.last.include?("Mysql2::Error::TimeoutError: 'Lock wait timeout exceeded; try restarting transaction' - 2 tries")
+    assert logs.last.include?("Lock wait timeout exceeded; try restarting transaction' - 2 tries")
     assert logs.last.include?("0.2 seconds until the next try")
   end
 
@@ -118,8 +117,8 @@ describe Lhm::SqlRetry do
 
     exception = assert_raises { @helper.trigger_wait_lock }
 
-    assert_match /Lock wait timeout exceeded; try restarting transaction/, exception.message
-    assert_equal Mysql2::Error::TimeoutError, exception.class
+    assert_match Regexp.new("Lock wait timeout exceeded; try restarting transaction"), exception.message
+    assert_equal DATABASE.timeout_error, exception.class
 
     assert_equal 2, @helper.record_count # no records inserted
     puts "*" * 64

data/spec/integration/sql_retry/lock_wait_timeout_test_helper.rb

@@ -14,13 +14,21 @@ class LockWaitTimeoutTestHelper
 
     @lock_duration = lock_duration
 
-    # While implementing this, I discovered that MySQL seems to have an off-by-one
-    # bug with the innodb_lock_wait_timeout. If you ask it to wait 2 seconds, it will wait 3.
-    # In order to avoid surprisingly the user, let's account for that here, but also
-    # guard against a case where we go below 1, the minimum value.
-    raise ArgumentError, "innodb_lock_wait_timeout must be greater than or equal to 2" unless innodb_lock_wait_timeout >= 2
     raise ArgumentError, "innodb_lock_wait_timeout must be an integer" if innodb_lock_wait_timeout.class != Integer
-    @innodb_lock_wait_timeout = innodb_lock_wait_timeout - 1
+
+    result = DATABASE.query(@main_conn, "SELECT VERSION()")
+    mysql_version = result.to_a.dig(0, "VERSION()").split("-", 2)[0]
+
+    if mysql_version.start_with?("8")
+      @innodb_lock_wait_timeout = innodb_lock_wait_timeout
+    else
+      # While implementing this, I discovered that MySQL seems to have an off-by-one
+      # bug with the innodb_lock_wait_timeout. If you ask it to wait 2 seconds, it will wait 3.
+      # In order to avoid surprisingly the user, let's account for that here, but also
+      # guard against a case where we go below 1, the minimum value.
+      raise ArgumentError, "innodb_lock_wait_timeout must be greater than or equal to 2" unless innodb_lock_wait_timeout >= 2
+      @innodb_lock_wait_timeout = innodb_lock_wait_timeout - 1
+    end
 
     @threads = []
     @queue = Queue.new
@@ -51,7 +59,7 @@ class LockWaitTimeoutTestHelper
   end
 
   def record_count(connection = main_conn)
-    response = connection.query("SELECT COUNT(id) FROM #{test_table_name}")
+    response = mysql_exec(connection, "SELECT COUNT(id) FROM #{test_table_name}")
     response.first.values.first
   end
 
@@ -79,7 +87,7 @@ class LockWaitTimeoutTestHelper
   attr_reader :main_conn, :lock_duration, :innodb_lock_wait_timeout
 
   def new_mysql_connection
-    Mysql2::Client.new(
+    DATABASE.client.new(
       host: '127.0.0.1',
       username: db_config['master']['user'],
       password: db_config['master']['password'],
@@ -103,8 +111,8 @@ class LockWaitTimeoutTestHelper
   private
 
   def mysql_exec(connection, statement)
-    if connection.class == Mysql2::Client
-      connection.query(statement)
+    if connection.class == DATABASE.client
+      DATABASE.query(connection, statement)
     elsif connection.class.to_s.include?("ActiveRecord")
       connection.execute(statement)
     else

data/spec/integration/sql_retry/proxysql_helper.rb

@@ -2,7 +2,7 @@ class ProxySQLHelper
   class << self
     # Flips the destination hostgroup for /maintenance:lhm/ from 0 (i.e. writer) to 1 (i.e. reader)
     def with_lhm_hostgroup_flip
-      conn = Mysql2::Client.new(
+      conn = DATABASE.client.new(
         host: '127.0.0.1',
         username: "remote-admin",
         password: "password",