lhm-shopify 3.4.0 → 3.5.5

data/lib/lhm/sql_retry.rb

@@ -1,5 +1,6 @@
 require 'retriable'
 require 'lhm/sql_helper'
+require 'lhm/proxysql_helper'
 
 module Lhm
   # SqlRetry standardizes the interface for retry behavior in components like
@@ -15,22 +16,140 @@ module Lhm
   # https://github.com/kamui/retriable. Additionally, a "log_prefix" option,
   # which is unique to SqlRetry can be used to prefix log output.
   class SqlRetry
-    def initialize(connection, options = {})
+    RECONNECT_SUCCESSFUL_MESSAGE = "LHM successfully reconnected to initial host:"
+    CLOUDSQL_VERSION_COMMENT = "(Google)"
+    # Will retry for 120 seconds (approximately, since connecting takes time).
+    RECONNECT_RETRY_MAX_ITERATION = 120
+    RECONNECT_RETRY_INTERVAL = 1
+    # Will abort the LHM if it had to reconnect more than 25 times in a single run (indicator that there might be
+    # something wrong with the network and would be better to run the LHM at a later time).
+    RECONNECTION_MAXIMUM = 25
+
+    MYSQL_VAR_NAMES = {
+      hostname: "@@global.hostname",
+      server_id: "@@global.server_id",
+      version_comment: "@@version_comment",
+    }
+
+    def initialize(connection, retry_options: {}, reconnect_with_consistent_host: false)
       @connection = connection
-      @log_prefix = options.delete(:log_prefix)
-      @retry_config = default_retry_config.dup.merge!(options)
+      self.retry_config = retry_options
+      self.reconnect_with_consistent_host = reconnect_with_consistent_host
     end
 
-    def with_retries
-      Retriable.retriable(retry_config) do
-        yield(@connection)
+    # Complete explanation of algorithm: https://github.com/Shopify/lhm/pull/112
+    def with_retries(log_prefix: nil)
+      @log_prefix = log_prefix || "" # No prefix. Just logs
+
+      # Amount of time LHM had to reconnect. Aborting if more than RECONNECTION_MAXIMUM
+      reconnection_counter = 0
+
+      Retriable.retriable(@retry_config) do
+        # Using begin -> rescue -> end for Ruby 2.4 compatibility
+        begin
+          if @reconnect_with_consistent_host
+            raise Lhm::Error.new("MySQL host has changed since the start of the LHM. Aborting to avoid data-loss") unless same_host_as_initial?
+          end
+
+          yield(@connection)
+        rescue StandardError => e
+          # Not all errors should trigger a reconnect. Some errors such be raised and abort the LHM (such as reconnecting to the wrong host).
+          # The error will be raised the connection is still active (i.e. no need to reconnect) or if the connection is
+          # dead (i.e. not active) and @reconnect_with_host is false (i.e. instructed not to reconnect)
+          raise e if @connection.active? || (!@connection.active? && !@reconnect_with_consistent_host)
+
+          # Lhm could be stuck in a weird state where it loses connection, reconnects and re looses-connection instantly
+          # after, creating an infinite loop (because of the usage of `retry`). Hence, abort after 25 reconnections
+          raise Lhm::Error.new("LHM reached host reconnection max of #{RECONNECTION_MAXIMUM} times. " \
+            "Please try again later.") if reconnection_counter > RECONNECTION_MAXIMUM
+
+          reconnection_counter += 1
+          if reconnect_with_host_check!
+            retry
+          else
+            raise Lhm::Error.new("LHM tried the reconnection procedure but failed. Aborting")
+          end
+        end
       end
     end
 
-    attr_reader :retry_config
+    # Both attributes will have defined setters
+    attr_reader :retry_config, :reconnect_with_consistent_host
+    attr_accessor :connection
+
+    def retry_config=(retry_options)
+      @retry_config = default_retry_config.dup.merge!(retry_options)
+    end
+
+    def reconnect_with_consistent_host=(reconnect)
+      if (@reconnect_with_consistent_host = reconnect)
+        @initial_hostname = hostname
+        @initial_server_id = server_id
+      end
+    end
 
     private
 
+    def hostname
+      mysql_single_value(MYSQL_VAR_NAMES[:hostname])
+    end
+
+    def server_id
+      mysql_single_value(MYSQL_VAR_NAMES[:server_id])
+    end
+
+    def cloudsql?
+      mysql_single_value(MYSQL_VAR_NAMES[:version_comment]).include?(CLOUDSQL_VERSION_COMMENT)
+    end
+
+    def mysql_single_value(name)
+      query = Lhm::ProxySQLHelper.tagged("SELECT #{name} LIMIT 1")
+
+      @connection.execute(query).to_a.first.tap do |record|
+        return record&.first
+      end
+    end
+
+    def same_host_as_initial?
+      return @initial_server_id == server_id if cloudsql?
+      @initial_hostname == hostname
+    end
+
+    def log_with_prefix(message, level = :info)
+      message.prepend("[#{@log_prefix}] ") if @log_prefix
+      Lhm.logger.public_send(level, message)
+    end
+
+    def reconnect_with_host_check!
+      log_with_prefix("Lost connection to MySQL, will retry to connect to same host")
+
+      RECONNECT_RETRY_MAX_ITERATION.times do
+        begin
+          sleep(RECONNECT_RETRY_INTERVAL)
+
+          # tries to reconnect. On failure will trigger a retry
+          @connection.reconnect!
+
+          if same_host_as_initial?
+            # This is not an actual error, but controlled way to get the parent `Retriable.retriable` to retry
+            # the statement that failed (since the Retriable gem only retries on errors).
+            log_with_prefix("LHM successfully reconnected to initial host: #{@initial_hostname} (server_id: #{@initial_server_id})")
+            return true
+          else
+            # New Master --> abort LHM (reconnecting will not change anything)
+            log_with_prefix("Reconnected to wrong host. Started migration on: #{@initial_hostname} (server_id: #{@initial_server_id}), but reconnected to: #{hostname} (server_id: #{server_id}).", :error)
+            return false
+          end
+        rescue StandardError => e
+          # Retry if ActiveRecord cannot reach host
+          next if /Lost connection to MySQL server at 'reading initial communication packet'/ === e.message
+          log_with_prefix("Encountered error: [#{e.class}] #{e.message}. Will stop reconnection procedure.", :info)
+          return false
+        end
+      end
+      false
+    end
+
     # For a full list of configuration options see https://github.com/kamui/retriable
     def default_retry_config
       {
@@ -43,6 +162,8 @@ module Lhm
             /Lost connection to MySQL server during query/,
             /Max connect timeout reached/,
             /Unknown MySQL server host/,
+            /connection is locked to hostgroup/,
+            /The MySQL server is running with the --read-only option so it cannot execute this statement/,
           ]
         },
         multiplier: 1, # each successive interval grows by this factor
@@ -51,9 +172,7 @@ module Lhm
         rand_factor: 0, # percentage to randomize the next retry interval time
         max_elapsed_time: Float::INFINITY, # max total time in seconds that code is allowed to keep being retried
         on_retry: Proc.new do |exception, try_number, total_elapsed_time, next_interval|
-          log = "#{exception.class}: '#{exception.message}' - #{try_number} tries in #{total_elapsed_time} seconds and #{next_interval} seconds until the next try."
-          log.prepend("[#{@log_prefix}] ") if @log_prefix
-          Lhm.logger.info(log)
+          log_with_prefix("#{exception.class}: '#{exception.message}' - #{try_number} tries in #{total_elapsed_time} seconds and #{next_interval} seconds until the next try.", :error)
         end
       }.freeze
     end

data/lib/lhm/throttler/slave_lag.rb

@@ -124,8 +124,8 @@ module Lhm
           else
             raise ArgumentError, "Expected #{config_proc.inspect} to respond to `call`"
           end
-        else # otherwise default to ActiveRecord provided config
-          ActiveRecord::Base.connection_pool.spec.config.dup
+        else
+          db_config
         end
         config.deep_symbolize_keys!
         config[:host] = @host
@@ -140,6 +140,23 @@ module Lhm
           [nil]
         end
       end
+
+      private
+
+      def db_config
+        if ar_supports_db_config?
+          ActiveRecord::Base.connection_pool.db_config.configuration_hash.dup
+        else
+          ActiveRecord::Base.connection_pool.spec.config.dup
+        end
+      end
+
+      def ar_supports_db_config?
+        # https://api.rubyonrails.org/v6.0/classes/ActiveRecord/ConnectionAdapters/ConnectionPool.html <-- has spec
+        # vs
+        # https://api.rubyonrails.org/v6.1/classes/ActiveRecord/ConnectionAdapters/ConnectionPool.html <-- has db_config
+        ActiveRecord::VERSION::MAJOR > 6 || ActiveRecord::VERSION::MAJOR == 6 && ActiveRecord::VERSION::MINOR >= 1
+      end
     end
   end
 end

data/lib/lhm/version.rb

@@ -2,5 +2,5 @@
 # Schmidt
 
 module Lhm
-  VERSION = '3.4.0'
+  VERSION = '3.5.5'
 end

data/lib/lhm.rb

@@ -8,6 +8,8 @@ require 'lhm/throttler'
 require 'lhm/version'
 require 'lhm/cleanup/current'
 require 'lhm/sql_retry'
+require 'lhm/proxysql_helper'
+require 'lhm/connection'
 require 'lhm/test_support'
 require 'lhm/railtie' if defined?(Rails::Railtie)
 require 'logger'
@@ -26,7 +28,7 @@ module Lhm
   extend Throttler
   extend self
 
-  DEFAULT_LOGGER_OPTIONS =  { level: Logger::INFO, file: STDOUT }
+  DEFAULT_LOGGER_OPTIONS = { level: Logger::INFO, file: STDOUT }
 
   # Alters a table with the changes described in the block
   #
@@ -44,15 +46,19 @@ module Lhm
   #   Use atomic switch to rename tables (defaults to: true)
   #   If using a version of mysql affected by atomic switch bug, LHM forces user
   #   to set this option (see SqlHelper#supports_atomic_switch?)
+  # @option options [Boolean] :reconnect_with_consistent_host
+  #   Active / Deactivate ProxySQL-aware reconnection procedure (default to: false)
   # @yield [Migrator] Yielded Migrator object records the changes
   # @return [Boolean] Returns true if the migration finishes
   # @raise [Error] Raises Lhm::Error in case of a error and aborts the migration
   def change_table(table_name, options = {}, &block)
-    origin = Table.parse(table_name, connection)
-    invoker = Invoker.new(origin, connection)
-    block.call(invoker.migrator)
-    invoker.run(options)
-    true
+    with_flags(options) do
+      origin = Table.parse(table_name, connection)
+      invoker = Invoker.new(origin, connection)
+      block.call(invoker.migrator)
+      invoker.run(options)
+      true
+    end
   end
 
   # Cleanup tables and triggers
@@ -81,16 +87,19 @@ module Lhm
     Lhm::Cleanup::Current.new(run, table_name, connection, options).execute
   end
 
+  # Setups DB connection
+  #
+  # @param [ActiveRecord::Base] connection ActiveRecord Connection
   def setup(connection)
-    @@connection = connection
+    @@connection = Connection.new(connection: connection)
   end
 
+  # Returns DB connection (or initializes it if not created yet)
   def connection
-    @@connection ||=
-      begin
-        raise 'Please call Lhm.setup' unless defined?(ActiveRecord)
-        ActiveRecord::Base.connection
-      end
+    @@connection ||= begin
+                       raise 'Please call Lhm.setup' unless defined?(ActiveRecord)
+                       @@connection = Connection.new(connection: ActiveRecord::Base.connection)
+                     end
   end
 
   def self.logger=(new_logger)
@@ -114,18 +123,34 @@ module Lhm
       triggers.each do |trigger|
         connection.execute("drop trigger if exists #{trigger}")
       end
+      logger.info("Dropped triggers #{triggers.join(', ')}")
+
       tables.each do |table|
         connection.execute("drop table if exists #{table}")
       end
+      logger.info("Dropped tables #{tables.join(', ')}")
+
       true
     elsif tables.empty? && triggers.empty?
-      puts 'Everything is clean. Nothing to do.'
+      logger.info('Everything is clean. Nothing to do.')
       true
     else
-      puts "Would drop LHM backup tables: #{tables.join(', ')}."
-      puts "Would drop LHM triggers: #{triggers.join(', ')}."
-      puts 'Run with Lhm.cleanup(true) to drop all LHM triggers and tables, or Lhm.cleanup_current_run(true, table_name) to clean up a specific LHM.'
+      logger.info("Would drop LHM backup tables: #{tables.join(', ')}.")
+      logger.info("Would drop LHM triggers: #{triggers.join(', ')}.")
+      logger.info('Run with Lhm.cleanup(true) to drop all LHM triggers and tables, or Lhm.cleanup_current_run(true, table_name) to clean up a specific LHM.')
       false
     end
   end
+
+  def with_flags(options)
+    old_flags = {
+      reconnect_with_consistent_host: Lhm.connection.reconnect_with_consistent_host,
+    }
+
+    Lhm.connection.reconnect_with_consistent_host = options[:reconnect_with_consistent_host] || false
+
+    yield
+  ensure
+    Lhm.connection.reconnect_with_consistent_host = old_flags[:reconnect_with_consistent_host]
+  end
 end

data/scripts/helpers/wait-for-dbs.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+# Wait for writer
+echo "Waiting for MySQL-1: "
+while ! (mysqladmin ping --host="127.0.0.1" --port=33006 --user=root --password=password --silent 2> /dev/null); do
+  echo -ne "."
+  sleep 1
+done
+# Wait for reader
+echo "Waiting for MySQL-2: "
+while ! (mysqladmin ping --host="127.0.0.1" --port=33007 --user=root --password=password --silent 2> /dev/null); do
+  echo -ne "."
+  sleep 1
+done
+# Wait for proxysql
+echo "Waiting for ProxySQL:"
+while ! (mysqladmin ping --host="127.0.0.1" --port=33005 --user=root --password=password --silent 2> /dev/null); do
+  echo -ne "."
+  sleep 1
+done
+
+echo "All DBs are ready"

data/scripts/mysql/reader/create_replication.sql

@@ -0,0 +1,10 @@
+STOP SLAVE;
+CHANGE MASTER TO
+MASTER_HOST='mysql-1',
+MASTER_AUTO_POSITION=1,
+MASTER_USER='replication',
+MASTER_PASSWORD='password',
+MASTER_CONNECT_RETRY=1,
+MASTER_RETRY_COUNT=300; -- 5 minutes
+
+start slave;

data/scripts/mysql/writer/create_test_db.sql

@@ -0,0 +1 @@
+CREATE DATABASE test;

data/scripts/mysql/writer/create_users.sql

@@ -0,0 +1,6 @@
+# Creates replication user in Writer
+CREATE USER IF NOT EXISTS 'writer'@'%' IDENTIFIED BY 'password';
+CREATE USER IF NOT EXISTS 'reader'@'%' IDENTIFIED BY 'password';
+
+CREATE USER IF NOT EXISTS 'replication'@'%' IDENTIFIED BY 'password';
+GRANT REPLICATION SLAVE ON *.* TO' replication'@'%' IDENTIFIED BY 'password';

data/scripts/proxysql/proxysql.cnf

@@ -0,0 +1,117 @@
+#file proxysql.cfg
+
+datadir="/var/lib/proxysql"
+restart_on_missing_heartbeats=999999
+query_parser_token_delimiters=","
+query_parser_key_value_delimiters=":"
+unit_of_work_identifiers="consistent_read_id"
+
+admin_variables=
+{
+  mysql_ifaces="0.0.0.0:6032"
+  admin_credentials="admin:password;remote-admin:password"
+}
+
+mysql_servers =
+(
+  {
+    address="mysql-1"
+    port=3306
+    hostgroup=0
+    max_connections=200
+  },
+  {
+    address="mysql-2"
+    port=3306
+    hostgroup=1
+    max_connections=200
+  }
+)
+
+mysql_variables=
+{
+  session_idle_ms=1
+    auto_increment_delay_multiplex=0
+
+    threads=8
+    max_connections=100000
+    interfaces="0.0.0.0:3306"
+    server_version="5.7.18-proxysql"
+    connect_timeout_server=10000
+    connect_timeout_server_max=10000
+    connect_retries_on_failure=0
+    default_charset="utf8mb4"
+    free_connections_pct=100
+    connection_warming=true
+    max_allowed_packet=16777216
+    monitor_enabled=false
+    query_retries_on_failure=0
+    shun_on_failures=999999
+    shun_recovery_time_sec=0
+    kill_backend_connection_when_disconnect=false
+    stats_time_backend_query=false
+    stats_time_query_processor=false
+    max_stmts_per_connection=5
+    default_max_latency_ms=999999
+    wait_timeout=1800000
+    eventslog_format=3
+    log_multiplexing_disabled=true
+    log_unhealthy_connections=false
+}
+
+# defines all the MySQL users
+mysql_users:
+(
+  {
+    username = "root"
+    password = "password"
+    default_hostgroup = 0
+    max_connections=1000
+    active = 1
+  },
+  {
+    username = "writer"
+    password = "password"
+    default_hostgroup = 0
+    max_connections=50000
+    active = 1
+    transaction_persistent=1
+  },
+  {
+    username = "reader"
+    password = "password"
+    default_hostgroup = 1
+    max_connections=50000
+    active = 1
+    transaction_persistent=1
+  }
+)
+
+#defines MySQL Query Rules
+mysql_query_rules:
+(
+  {
+      rule_id = 1
+      active = 1
+      match_digest = "@@SESSION"
+      multiplex = 2
+    },
+    {
+      rule_id = 2
+      active = 1
+      match_digest = "@@global\.server_id"
+      multiplex = 2
+    },
+    {
+      rule_id = 3
+      active = 1
+      match_digest = "@@global\.hostname"
+      multiplex = 2
+    },
+    {
+      rule_id = 4
+      active = 1
+      match_pattern = "maintenance:lhm"
+      destination_hostgroup = 0
+    }
+)

data/spec/integration/atomic_switcher_spec.rb

@@ -6,6 +6,7 @@ require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
 require 'lhm/table'
 require 'lhm/migration'
 require 'lhm/atomic_switcher'
+require 'lhm/connection'
 
 describe Lhm::AtomicSwitcher do
   include IntegrationHelper
@@ -15,9 +16,9 @@ describe Lhm::AtomicSwitcher do
   describe 'switching' do
     before(:each) do
       Thread.abort_on_exception = true
-      @origin      = table_create('origin')
+      @origin = table_create('origin')
       @destination = table_create('destination')
-      @migration   = Lhm::Migration.new(@origin, @destination)
+      @migration = Lhm::Migration.new(@origin, @destination)
       @logs = StringIO.new
       Lhm.logger = Logger.new(@logs)
       @connection.execute('SET GLOBAL innodb_lock_wait_timeout=3')
@@ -29,26 +30,59 @@ describe Lhm::AtomicSwitcher do
     end
 
     it 'should retry and log on lock wait timeouts' do
-      connection = mock()
-      connection.stubs(:data_source_exists?).returns(true)
-      connection.stubs(:execute).raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.').then.returns(true)
+      ar_connection = mock()
+      ar_connection.stubs(:data_source_exists?).returns(true)
+      ar_connection.stubs(:active?).returns(true)
+      ar_connection.stubs(:execute).returns([["dummy"]], [["dummy"]], [["dummy"]])
+                   .then
+                   .raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
+                   .then
+                   .returns([["dummy"]]) # Matches initial host -> triggers retry
+
+      connection = Lhm::Connection.new(connection: ar_connection, options: {
+        reconnect_with_consistent_host: true,
+        retriable: {
+          tries: 3,
+          base_interval: 0
+        }
+      })
 
-      switcher = Lhm::AtomicSwitcher.new(@migration, connection, retriable: {base_interval: 0})
+      switcher = Lhm::AtomicSwitcher.new(@migration, connection)
 
       assert switcher.run
 
       log_messages = @logs.string.split("\n")
       assert_equal(2, log_messages.length)
       assert log_messages[0].include? "Starting run of class=Lhm::AtomicSwitcher"
+      # On failure of this assertion, check for Lhm::Connection#file
       assert log_messages[1].include? "[AtomicSwitcher] ActiveRecord::StatementInvalid: 'Lock wait timeout exceeded; try restarting transaction.' - 1 tries"
     end
 
     it 'should give up on lock wait timeouts after a configured number of tries' do
-      connection = mock()
-      connection.stubs(:data_source_exists?).returns(true)
-      connection.stubs(:execute).twice.raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
+      ar_connection = mock()
+      ar_connection.stubs(:data_source_exists?).returns(true)
+      ar_connection.stubs(:active?).returns(true)
+      ar_connection.stubs(:execute).returns([["dummy"]], [["dummy"]], [["dummy"]])
+                   .then
+                   .raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
+                   .then
+                   .returns([["dummy"]]) # triggers retry 1
+                   .then
+                   .raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
+                   .then
+                   .returns([["dummy"]]) # triggers retry 2
+                   .then
+                   .raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.') # triggers retry 2
+
+      connection = Lhm::Connection.new(connection: ar_connection, options: {
+        reconnect_with_consistent_host: true,
+        retriable: {
+          tries: 2,
+          base_interval: 0
+        }
+      })
 
-      switcher = Lhm::AtomicSwitcher.new(@migration, connection, retriable: {tries: 2, base_interval: 0})
+      switcher = Lhm::AtomicSwitcher.new(@migration, connection)
 
       assert_raises(ActiveRecord::StatementInvalid) { switcher.run }
     end
@@ -58,12 +92,14 @@ describe Lhm::AtomicSwitcher do
       switcher.send :define_singleton_method, :atomic_switch do
         'SELECT * FROM nonexistent'
       end
-      -> { switcher.run }.must_raise(ActiveRecord::StatementInvalid)
+      value(-> { switcher.run }).must_raise(ActiveRecord::StatementInvalid)
     end
 
     it "should raise when destination doesn't exist" do
-      connection = mock()
-      connection.stubs(:data_source_exists?).returns(false)
+      ar_connection = mock()
+      ar_connection.stubs(:data_source_exists?).returns(false)
+
+      connection = Lhm::Connection.new(connection: ar_connection)
 
       switcher = Lhm::AtomicSwitcher.new(@migration, connection)
 
@@ -75,8 +111,8 @@ describe Lhm::AtomicSwitcher do
       switcher.run
 
       slave do
-        data_source_exists?(@origin).must_equal true
-        table_read(@migration.archive_name).columns.keys.must_include 'origin'
+        value(data_source_exists?(@origin)).must_equal true
+        value(table_read(@migration.archive_name).columns.keys).must_include 'origin'
       end
     end
 
@@ -85,8 +121,8 @@ describe Lhm::AtomicSwitcher do
       switcher.run
 
       slave do
-        data_source_exists?(@destination).must_equal false
-        table_read(@origin.name).columns.keys.must_include 'destination'
+        value(data_source_exists?(@destination)).must_equal false
+        value(table_read(@origin.name).columns.keys).must_include 'destination'
       end
     end
   end

data/spec/integration/chunk_insert_spec.rb

@@ -11,7 +11,8 @@ describe Lhm::ChunkInsert do
       @destination = table_create(:destination)
       @migration = Lhm::Migration.new(@origin, @destination)
       execute("insert into origin set id = 1001")
-      @instance = Lhm::ChunkInsert.new(@migration, connection, 1001, 1001)
+      @connection = Lhm::Connection.new(connection: connection)
+      @instance = Lhm::ChunkInsert.new(@migration, @connection, 1001, 1001)
     end
 
     it "returns the count" do
@@ -22,7 +23,7 @@ describe Lhm::ChunkInsert do
       @instance.insert_and_return_count_of_rows_created
 
       slave do
-        count_all(@destination.name).must_equal(1)
+        value(count_all(@destination.name)).must_equal(1)
       end
     end
   end