lhm-shopify 3.4.0 → 3.5.5

data/spec/integration/lhm_spec.rb

@@ -2,6 +2,7 @@
 # Schmidt
 
 require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
+require 'integration/toxiproxy_helper'
 
 describe Lhm do
   include IntegrationHelper
@@ -17,7 +18,7 @@ describe Lhm do
       end
 
       slave do
-        table_read(:users).columns['logins'].must_equal({
+        value(table_read(:users).columns['logins']).must_equal({
           :type           => 'int(12)',
           :is_nullable    => 'YES',
           :column_default => '0',
@@ -35,7 +36,7 @@ describe Lhm do
       end
 
       slave do
-        table_read(:custom_primary_key).columns['logins'].must_equal({
+        value(table_read(:custom_primary_key).columns['logins']).must_equal({
           :type           => 'int(12)',
           :is_nullable    => 'YES',
           :column_default => '0',
@@ -53,7 +54,7 @@ describe Lhm do
       end
 
       slave do
-        table_read(:composite_primary_key).columns['logins'].must_equal({
+        value(table_read(:composite_primary_key).columns['logins']).must_equal({
           :type           => 'int(12)',
           :is_nullable    => 'YES',
           :column_default => '0',
@@ -82,7 +83,7 @@ describe Lhm do
         end
 
         slave do
-          connection.primary_key('users').must_equal(['username', 'id'])
+          value(connection.primary_key('users')).must_equal(['username', 'id'])
         end
       end
 
@@ -104,7 +105,7 @@ describe Lhm do
 
         it 'migrates the existing data' do
           slave do
-            count_all(:permissions).must_equal(11)
+            value(count_all(:permissions)).must_equal(11)
           end
         end
       end
@@ -120,7 +121,7 @@ describe Lhm do
 
         it 'migrates all data' do
           slave do
-            count_all(:permissions).must_equal(13)
+            value(count_all(:permissions)).must_equal(13)
           end
         end
       end
@@ -132,7 +133,7 @@ describe Lhm do
       end
 
       slave do
-        table_read(:users).columns['logins'].must_equal({
+        value(table_read(:users).columns['logins']).must_equal({
           :type => 'int(12)',
           :is_nullable => 'YES',
           :column_default => '0',
@@ -150,7 +151,7 @@ describe Lhm do
       end
 
       slave do
-        count_all(:users).must_equal(23)
+        value(count_all(:users)).must_equal(23)
       end
     end
 
@@ -170,7 +171,7 @@ describe Lhm do
       end
 
       slave do
-        index_on_columns?(:users, [:comment, :created_at]).must_equal(true)
+        value(index_on_columns?(:users, [:comment, :created_at])).must_equal(true)
       end
     end
 
@@ -180,7 +181,7 @@ describe Lhm do
       end
 
       slave do
-        index?(:users, :my_index_name).must_equal(true)
+        value(index?(:users, :my_index_name)).must_equal(true)
       end
     end
 
@@ -190,7 +191,7 @@ describe Lhm do
       end
 
       slave do
-        index_on_columns?(:users, :group).must_equal(true)
+        value(index_on_columns?(:users, :group)).must_equal(true)
       end
     end
 
@@ -200,7 +201,7 @@ describe Lhm do
       end
 
       slave do
-        index_on_columns?(:users, :comment, :unique).must_equal(true)
+        value(index_on_columns?(:users, :comment, :unique)).must_equal(true)
       end
     end
 
@@ -210,7 +211,7 @@ describe Lhm do
       end
 
       slave do
-        index_on_columns?(:users, [:username, :created_at]).must_equal(false)
+        value(index_on_columns?(:users, [:username, :created_at])).must_equal(false)
       end
     end
 
@@ -220,7 +221,7 @@ describe Lhm do
       end
 
       slave do
-        index?(:users, :index_with_a_custom_name).must_equal(false)
+        value(index?(:users, :index_with_a_custom_name)).must_equal(false)
       end
     end
 
@@ -230,7 +231,7 @@ describe Lhm do
       end
 
       slave do
-        index?(:users, :index_with_a_custom_name).must_equal(false)
+        value(index?(:users, :index_with_a_custom_name)).must_equal(false)
       end
     end
 
@@ -240,7 +241,7 @@ describe Lhm do
       end
 
       slave do
-        table_read(:users).columns['flag'].must_equal({
+        value(table_read(:users).columns['flag']).must_equal({
           :type => 'tinyint(1)',
           :is_nullable => 'YES',
           :column_default => nil,
@@ -256,7 +257,7 @@ describe Lhm do
       end
 
       slave do
-        table_read(:users).columns['comment'].must_equal({
+        value(table_read(:users).columns['comment']).must_equal({
           :type => 'varchar(20)',
           :is_nullable => 'NO',
           :column_default => 'none',
@@ -274,7 +275,7 @@ describe Lhm do
       end
 
       slave do
-        table_read(:small_table).columns['id'].must_equal({
+        value(table_read(:small_table).columns['id']).must_equal({
           :type => 'int(5)',
           :is_nullable => 'NO',
           :column_default => nil,
@@ -295,7 +296,7 @@ describe Lhm do
       slave do
         table_data = table_read(:users)
         assert_nil table_data.columns['username']
-        table_read(:users).columns['login'].must_equal({
+        value(table_read(:users).columns['login']).must_equal({
           :type => 'varchar(255)',
           :is_nullable => 'YES',
           :column_default => nil,
@@ -305,7 +306,7 @@ describe Lhm do
 
         result = select_one('SELECT login from users')
         result = result['login'] if result.respond_to?(:has_key?)
-        result.must_equal('a user')
+        value(result).must_equal('a user')
       end
     end
 
@@ -320,7 +321,7 @@ describe Lhm do
       slave do
         table_data = table_read(:users)
         assert_nil table_data.columns['group']
-        table_read(:users).columns['fnord'].must_equal({
+        value(table_read(:users).columns['fnord']).must_equal({
           :type => 'varchar(255)',
           :is_nullable => 'YES',
           :column_default => 'Superfriends',
@@ -330,7 +331,7 @@ describe Lhm do
 
         result = select_one('SELECT `fnord` from users')
         result = result['fnord'] if result.respond_to?(:has_key?)
-        result.must_equal('Superfriends')
+        value(result).must_equal('Superfriends')
       end
     end
 
@@ -347,7 +348,7 @@ describe Lhm do
       slave do
         table_data = table_read(:users)
         assert_nil table_data.columns['username']
-        table_read(:users).columns['user_name'].must_equal({
+        value(table_read(:users).columns['user_name']).must_equal({
              :type => 'varchar(255)',
              :is_nullable => 'YES',
              :column_default => nil,
@@ -357,7 +358,7 @@ describe Lhm do
 
         result = select_one('SELECT `user_name` from users')
         result = result['user_name'] if result.respond_to?(:has_key?)
-        result.must_equal('a user')
+        value(result).must_equal('a user')
       end
     end
 
@@ -375,7 +376,7 @@ describe Lhm do
       slave do
         table_data = table_read(:users)
         assert_nil table_data.columns['reference']
-        table_read(:users).columns['ref'].must_equal({
+        value(table_read(:users).columns['ref']).must_equal({
            :type => 'int(11)',
            :is_nullable => 'YES',
            :column_default => nil,
@@ -385,7 +386,7 @@ describe Lhm do
 
         result = select_one('SELECT `ref` from users')
         result = result['ref'] if result.respond_to?(:has_key?)
-        result.must_equal(10)
+        value(result).must_equal(10)
       end
     end
 
@@ -402,7 +403,7 @@ describe Lhm do
       slave do
         table_data = table_read(:users)
         assert_nil table_data.columns['group']
-        table_read(:users).columns['fnord'].must_equal({
+        value(table_read(:users).columns['fnord']).must_equal({
            :type => 'varchar(255)',
            :is_nullable => 'YES',
            :column_default => nil,
@@ -412,7 +413,7 @@ describe Lhm do
 
         result = select_one('SELECT `fnord` from users')
         result = result['fnord'] if result.respond_to?(:has_key?)
-        result.must_equal(nil)
+        assert_nil(result)
       end
     end
 
@@ -427,7 +428,7 @@ describe Lhm do
       slave do
         table_data = table_read(:users)
         assert_nil table_data.columns['username']
-        table_read(:users).columns['user_name'].must_equal({
+        value(table_read(:users).columns['user_name']).must_equal({
           :type => 'varchar(255)',
           :is_nullable => 'YES',
           :column_default => nil,
@@ -437,7 +438,7 @@ describe Lhm do
 
         result = select_one('SELECT `user_name` from users')
         result = result['user_name'] if result.respond_to?(:has_key?)
-        result.must_equal('a user')
+        value(result).must_equal('a user')
       end
     end
 
@@ -454,7 +455,7 @@ describe Lhm do
       slave do
         table_data = table_read(:users)
         assert_nil table_data.columns['username']
-        table_read(:users).columns['user_name'].must_equal({
+        value(table_read(:users).columns['user_name']).must_equal({
           :type => 'varchar(255)',
           :is_nullable => 'NO',
           :column_default => nil,
@@ -464,7 +465,7 @@ describe Lhm do
 
         result = select_one('SELECT `user_name` from users')
         result = result['user_name'] if result.respond_to?(:has_key?)
-        result.must_equal('a user')
+        value(result).must_equal('a user')
       end
     end
 
@@ -501,7 +502,7 @@ describe Lhm do
       slave do
         table_data = table_read(:users)
         assert_nil table_data.columns['fnord']
-        table_read(:users).columns['group'].must_equal({
+        value(table_read(:users).columns['group']).must_equal({
           :type => 'varchar(255)',
           :is_nullable => 'YES',
           :column_default => 'Superfriends',
@@ -525,11 +526,11 @@ describe Lhm do
       end
 
       slave do
-        table_read(:lines).columns.must_include 'by'
-        table_read(:lines).columns.wont_include 'lines'
-        index_on_columns?(:lines, ['between'], :unique).must_equal true
-        index_on_columns?(:lines, ['by']).must_equal false
-        count_all(:lines).must_equal(2)
+        value(table_read(:lines).columns).must_include 'by'
+        value(table_read(:lines).columns).wont_include 'lines'
+        value(index_on_columns?(:lines, ['between'], :unique)).must_equal true
+        value(index_on_columns?(:lines, ['by'])).must_equal false
+        value(count_all(:lines)).must_equal(2)
       end
     end
 
@@ -554,7 +555,7 @@ describe Lhm do
         insert.join
 
         slave do
-          count_all(:users).must_equal(60)
+          value(count_all(:users)).must_equal(60)
         end
       end
 
@@ -577,7 +578,80 @@ describe Lhm do
         delete.join
 
         slave do
-          count_all(:users).must_equal(40)
+          value(count_all(:users)).must_equal(40)
+        end
+      end
+    end
+
+    describe 'connection' do
+      include ToxiproxyHelper
+
+      before(:each) do
+        @logs = StringIO.new
+        Lhm.logger = Logger.new(@logs)
+      end
+
+      it " should not try to reconnect if reconnect_with_consistent_host is not provided" do
+        connect_master_with_toxiproxy!
+
+        table_create(:users)
+        100.times { |n| execute("insert into users set reference = '#{ n }'") }
+
+        assert_raises ActiveRecord::StatementInvalid do
+          Toxiproxy[:mysql_master].down do
+            Lhm.change_table(:users, :atomic_switch => false) do |t|
+              t.ddl("ALTER TABLE #{t.name} CHANGE id id bigint (20) NOT NULL")
+              t.ddl("ALTER TABLE #{t.name} DROP PRIMARY KEY, ADD PRIMARY KEY (username, id)")
+              t.ddl("ALTER TABLE #{t.name} ADD INDEX (id)")
+              t.ddl("ALTER TABLE #{t.name} CHANGE id id bigint (20) NOT NULL AUTO_INCREMENT")
+            end
+          end
+        end
+      end
+
+      it "should reconnect if reconnect_with_consistent_host is true" do
+        connect_master_with_toxiproxy!
+        mysql_disabled = false
+
+        table_create(:users)
+        100.times { |n| execute("insert into users set reference = '#{ n }'") }
+
+        # Redeclare Lhm::ChunkInsert to use Hook to disable MySQL writer for 3 seconds before first insert
+        Lhm::ChunkInsert.class_eval do
+          extend AfterDo
+
+          before(:insert_and_return_count_of_rows_created) do
+            unless mysql_disabled
+              mysql_disabled = true
+              Thread.new do
+                Toxiproxy[:mysql_master].down do
+                  sleep 3
+                end
+              end
+            end
+          end
+
+          # Need to call `#method_added` manually to have the hooks take into effect
+          method_added(:insert_and_return_count_of_rows_created)
+        end
+
+        Lhm.change_table(:users, atomic_switch: false, reconnect_with_consistent_host: true) do |t|
+          t.ddl("ALTER TABLE #{t.name} CHANGE id id bigint (20) NOT NULL")
+          t.ddl("ALTER TABLE #{t.name} DROP PRIMARY KEY, ADD PRIMARY KEY (username, id)")
+          t.ddl("ALTER TABLE #{t.name} ADD INDEX (id)")
+          t.ddl("ALTER TABLE #{t.name} CHANGE id id bigint (20) NOT NULL AUTO_INCREMENT")
+        end
+
+        log_lines = @logs.string.split("\n")
+
+        assert log_lines.one?{ |line| line.include?("Lost connection to MySQL, will retry to connect to same host")}
+        assert log_lines.one?{ |line| line.include?("LHM successfully reconnected to initial host")}
+        assert log_lines.one?{ |line| line.include?("100% complete")}
+
+        Lhm::ChunkInsert.remove_all_callbacks
+
+        slave do
+          value(count_all(:users)).must_equal(100)
         end
       end
     end

data/spec/integration/lock_wait_timeout_spec.rb

@@ -24,7 +24,7 @@ describe Lhm do
     session_innodb_lock_wait_timeout = connection.select_one("SHOW SESSION VARIABLES LIKE 'innodb_lock_wait_timeout'")['Value'].to_i
     session_lock_wait_timeout = connection.select_one("SHOW SESSION VARIABLES LIKE 'lock_wait_timeout'")['Value'].to_i
 
-    session_lock_wait_timeout.must_equal global_lock_wait_timeout + Lhm::Invoker::LOCK_WAIT_TIMEOUT_DELTA
-    session_innodb_lock_wait_timeout.must_equal global_innodb_lock_wait_timeout + Lhm::Invoker::LOCK_WAIT_TIMEOUT_DELTA
+    value(session_lock_wait_timeout).must_equal global_lock_wait_timeout + Lhm::Invoker::LOCK_WAIT_TIMEOUT_DELTA
+    value(session_innodb_lock_wait_timeout).must_equal global_innodb_lock_wait_timeout + Lhm::Invoker::LOCK_WAIT_TIMEOUT_DELTA
   end
 end

data/spec/integration/locked_switcher_spec.rb

@@ -32,8 +32,8 @@ describe Lhm::LockedSwitcher do
       switcher.run
 
       slave do
-        data_source_exists?(@origin).must_equal true
-        table_read(@migration.archive_name).columns.keys.must_include 'origin'
+        value(data_source_exists?(@origin)).must_equal true
+        value(table_read(@migration.archive_name).columns.keys).must_include 'origin'
       end
     end
 
@@ -42,8 +42,8 @@ describe Lhm::LockedSwitcher do
       switcher.run
 
       slave do
-        data_source_exists?(@destination).must_equal false
-        table_read(@origin.name).columns.keys.must_include 'destination'
+        value(data_source_exists?(@destination)).must_equal false
+        value(table_read(@origin.name).columns.keys).must_include 'destination'
       end
     end
   end

data/spec/integration/proxysql_spec.rb

@@ -0,0 +1,34 @@
+describe "ProxySQL integration" do
+  it "Should contact the writer" do
+    conn = Mysql2::Client.new(
+      host: '127.0.0.1',
+      username: "writer",
+      password: "password",
+      port: "33005",
+      )
+
+    assert_equal conn.query("SELECT @@global.hostname as host").each.first["host"], "mysql-1"
+  end
+
+  it "Should contact the reader" do
+    conn = Mysql2::Client.new(
+      host: '127.0.0.1',
+      username: "reader",
+      password: "password",
+      port: "33005",
+      )
+
+    assert_equal conn.query("SELECT @@global.hostname as host").each.first["host"], "mysql-2"
+  end
+
+  it "Should override default hostgroup from user if rule matches" do
+    conn = Mysql2::Client.new(
+      host: '127.0.0.1',
+      username: "reader",
+      password: "password",
+      port: "33005",
+      )
+
+    assert_equal conn.query("SELECT @@global.hostname as host #{Lhm::ProxySQLHelper::ANNOTATION}").each.first["host"], "mysql-1"
+  end
+end

data/spec/integration/sql_retry/db_connection_helper.rb

@@ -0,0 +1,52 @@
+require 'yaml'
+require 'mysql2'
+
+class DBConnectionHelper
+
+  DATABASE_CONFIG_FILE = "database.yml"
+
+  class << self
+    def db_config
+      @db_config ||= YAML.load_file(File.expand_path(File.dirname(__FILE__)) + "/../#{DATABASE_CONFIG_FILE}")
+    end
+
+    def new_mysql_connection(role = :master, with_data = false, toxic = false)
+
+      key = role.to_s + toxic_postfix(toxic)
+
+      conn = ActiveRecord::Base.establish_connection(
+        :host => '127.0.0.1',
+        :adapter => "mysql2",
+        :username => db_config[key]['user'],
+        :password => db_config[key]['password'],
+        :database => test_db_name,
+        :port => db_config[key]['port']
+      )
+      conn = conn.connection
+      init_with_dummy_data(conn) if with_data
+      conn
+    end
+
+    def toxic_postfix(toxic)
+      toxic ? "_toxic" : ""
+    end
+
+    def test_db_name
+      @test_db_name ||= "test"
+    end
+
+    def test_table_name
+      @test_table_name ||= "test"
+    end
+
+    def init_with_dummy_data(conn)
+      conn.execute("DROP TABLE IF EXISTS #{test_table_name} ")
+      conn.execute("CREATE TABLE #{test_table_name} (id int)")
+
+      1.upto(9) do |i|
+        query = "INSERT INTO #{test_table_name} (id) VALUE (#{i})"
+        conn.execute(query)
+      end
+    end
+  end
+end

data/spec/integration/sql_retry/lock_wait_spec.rb

@@ -21,6 +21,8 @@ describe Lhm::SqlRetry do
 
     # Assert our pre-conditions
     assert_equal 2, @helper.record_count
+
+    Mysql2::Client.any_instance.stubs(:active?).returns(true)
   end
 
   after(:each) do
@@ -41,7 +43,7 @@ describe Lhm::SqlRetry do
 
     exception = assert_raises { @helper.trigger_wait_lock }
 
-    assert_equal "Lock wait timeout exceeded; try restarting transaction", exception.message
+    assert_match /Lock wait timeout exceeded; try restarting transaction/, exception.message
     assert_equal Mysql2::Error::TimeoutError, exception.class
 
     assert_equal 2, @helper.record_count # no records inserted
@@ -53,12 +55,12 @@ describe Lhm::SqlRetry do
   it "successfully executes the SQL despite the errors encountered" do
     # Start a thread to retry, once the lock is held, execute the block
     @helper.with_waiting_lock do |waiting_connection|
-      sql_retry = Lhm::SqlRetry.new(waiting_connection, {
+      sql_retry = Lhm::SqlRetry.new(waiting_connection, retry_options: {
         base_interval: 0.2, # first retry after 200ms
         multiplier: 1, # subsequent retries wait 1x longer than first retry (no change)
         tries: 3, # we only need 3 tries (including the first) for the scenario described below
         rand_factor: 0 # do not introduce randomness to wait timer
-      })
+      }, reconnect_with_consistent_host: false)
 
       # RetryTestHelper is configured to hold lock for 5 seconds and timeout after 2 seconds.
       # Therefore the sequence of events will be:
@@ -96,12 +98,12 @@ describe Lhm::SqlRetry do
     puts "*" * 64
     # Start a thread to retry, once the lock is held, execute the block
     @helper.with_waiting_lock do |waiting_connection|
-      sql_retry = Lhm::SqlRetry.new(waiting_connection, {
+      sql_retry = Lhm::SqlRetry.new(waiting_connection, retry_options: {
         base_interval: 0.2, # first retry after 200ms
         multiplier: 1, # subsequent retries wait 1x longer than first retry (no change)
         tries: 2, # we need 3 tries (including the first) for the scenario described below, but we only get two...we will fail
         rand_factor: 0 # do not introduce randomness to wait timer
-      })
+      }, reconnect_with_consistent_host: false)
 
       # RetryTestHelper is configured to hold lock for 5 seconds and timeout after 2 seconds.
       # Therefore the sequence of events will be:
@@ -116,7 +118,7 @@ describe Lhm::SqlRetry do
 
     exception = assert_raises { @helper.trigger_wait_lock }
 
-    assert_equal "Lock wait timeout exceeded; try restarting transaction", exception.message
+    assert_match /Lock wait timeout exceeded; try restarting transaction/, exception.message
     assert_equal Mysql2::Error::TimeoutError, exception.class
 
     assert_equal 2, @helper.record_count # no records inserted

data/spec/integration/sql_retry/lock_wait_timeout_test_helper.rb

@@ -1,5 +1,7 @@
-require 'yaml'
+require 'integration/integration_helper'
+
 class LockWaitTimeoutTestHelper
+
   def initialize(lock_duration:, innodb_lock_wait_timeout:)
     # This connection will be used exclusively to setup the test,
     # assert pre-conditions and assert post-conditions.
@@ -68,7 +70,7 @@ class LockWaitTimeoutTestHelper
 
   def insert_records_at_ids(connection, ids)
     ids.each do |id|
-      connection.query "INSERT INTO #{test_table_name} (id) VALUES (#{id})"
+      mysql_exec(connection, "INSERT INTO #{test_table_name} (id) VALUES (#{id})")
     end
   end
 
@@ -79,11 +81,10 @@ class LockWaitTimeoutTestHelper
   def new_mysql_connection
     Mysql2::Client.new(
       host: '127.0.0.1',
-      database: test_db_name,
       username: db_config['master']['user'],
       password: db_config['master']['password'],
       port: db_config['master']['port'],
-      socket: db_config['master']['socket']
+      database: test_db_name
     )
   end
 
@@ -98,4 +99,16 @@ class LockWaitTimeoutTestHelper
   def test_table_name
     @test_table_name ||= "lock_wait"
   end
+
+  private
+
+  def mysql_exec(connection, statement)
+    if connection.class == Mysql2::Client
+      connection.query(statement)
+    elsif connection.class.to_s.include?("ActiveRecord")
+      connection.execute(statement)
+    else
+      raise StandardError.new("Unrecognized MySQL client")
+    end
+  end
 end

data/spec/integration/sql_retry/proxysql_helper.rb

@@ -0,0 +1,22 @@
+class ProxySQLHelper
+  class << self
+    # Flips the destination hostgroup for /maintenance:lhm/ from 0 (i.e. writer) to 1 (i.e. reader)
+    def with_lhm_hostgroup_flip
+      conn = Mysql2::Client.new(
+        host: '127.0.0.1',
+        username: "remote-admin",
+        password: "password",
+        port: "6032",
+      )
+
+      begin
+        conn.query("UPDATE mysql_query_rules SET destination_hostgroup=1 WHERE match_pattern=\"maintenance:lhm\"")
+        conn.query("LOAD MYSQL QUERY RULES TO RUNTIME;")
+        yield
+      ensure
+        conn.query("UPDATE mysql_query_rules SET destination_hostgroup=0 WHERE match_pattern=\"maintenance:lhm\"")
+        conn.query("LOAD MYSQL QUERY RULES TO RUNTIME;")
+      end
+    end
+  end
+end