lhm-shopify 3.5.1 → 3.5.5

data/lib/lhm/sql_retry.rb

@@ -18,6 +18,12 @@ module Lhm
   class SqlRetry
     RECONNECT_SUCCESSFUL_MESSAGE = "LHM successfully reconnected to initial host:"
     CLOUDSQL_VERSION_COMMENT = "(Google)"
+    # Will retry for 120 seconds (approximately, since connecting takes time).
+    RECONNECT_RETRY_MAX_ITERATION = 120
+    RECONNECT_RETRY_INTERVAL = 1
+    # Will abort the LHM if it had to reconnect more than 25 times in a single run (indicator that there might be
+    # something wrong with the network and would be better to run the LHM at a later time).
+    RECONNECTION_MAXIMUM = 25
 
     MYSQL_VAR_NAMES = {
       hostname: "@@global.hostname",
@@ -25,30 +31,24 @@ module Lhm
       version_comment: "@@version_comment",
     }
 
-    # This internal error is used to trigger retries from the parent Retriable.retriable in #with_retries
-    class ReconnectToHostSuccessful < Lhm::Error; end
-
-    def initialize(connection, options: {}, reconnect_with_consistent_host: true)
+    def initialize(connection, retry_options: {}, reconnect_with_consistent_host: false)
       @connection = connection
-      @log_prefix = options.delete(:log_prefix)
-      @global_retry_config = default_retry_config.dup.merge!(options)
-      if (@reconnect_with_consistent_host = reconnect_with_consistent_host)
-        @initial_hostname = hostname
-        @initial_server_id = server_id
-      end
+      self.retry_config = retry_options
+      self.reconnect_with_consistent_host = reconnect_with_consistent_host
     end
 
     # Complete explanation of algorithm: https://github.com/Shopify/lhm/pull/112
-    def with_retries(retry_config = {})
-      @log_prefix = retry_config.delete(:log_prefix)
+    def with_retries(log_prefix: nil)
+      @log_prefix = log_prefix || "" # No prefix. Just logs
 
-      retry_config = @global_retry_config.dup.merge!(retry_config)
+      # Amount of time LHM had to reconnect. Aborting if more than RECONNECTION_MAXIMUM
+      reconnection_counter = 0
 
-      Retriable.retriable(retry_config) do
+      Retriable.retriable(@retry_config) do
         # Using begin -> rescue -> end for Ruby 2.4 compatibility
         begin
           if @reconnect_with_consistent_host
-            raise Lhm::Error.new("Could not reconnected to initial MySQL host. Aborting to avoid data-loss") unless same_host_as_initial?
+            raise Lhm::Error.new("MySQL host has changed since the start of the LHM. Aborting to avoid data-loss") unless same_host_as_initial?
           end
 
           yield(@connection)
@@ -57,13 +57,36 @@ module Lhm
           # The error will be raised the connection is still active (i.e. no need to reconnect) or if the connection is
           # dead (i.e. not active) and @reconnect_with_host is false (i.e. instructed not to reconnect)
           raise e if @connection.active? || (!@connection.active? && !@reconnect_with_consistent_host)
-          reconnect_with_host_check! if @reconnect_with_consistent_host
+
+          # Lhm could be stuck in a weird state where it loses connection, reconnects and re looses-connection instantly
+          # after, creating an infinite loop (because of the usage of `retry`). Hence, abort after 25 reconnections
+          raise Lhm::Error.new("LHM reached host reconnection max of #{RECONNECTION_MAXIMUM} times. " \
+            "Please try again later.") if reconnection_counter > RECONNECTION_MAXIMUM
+
+          reconnection_counter += 1
+          if reconnect_with_host_check!
+            retry
+          else
+            raise Lhm::Error.new("LHM tried the reconnection procedure but failed. Aborting")
+          end
         end
       end
     end
 
-    attr_reader :global_retry_config
-    attr_accessor :reconnect_with_consistent_host
+    # Both attributes will have defined setters
+    attr_reader :retry_config, :reconnect_with_consistent_host
+    attr_accessor :connection
+
+    def retry_config=(retry_options)
+      @retry_config = default_retry_config.dup.merge!(retry_options)
+    end
+
+    def reconnect_with_consistent_host=(reconnect)
+      if (@reconnect_with_consistent_host = reconnect)
+        @initial_hostname = hostname
+        @initial_server_id = server_id
+      end
+    end
 
     private
 
@@ -99,34 +122,32 @@ module Lhm
 
     def reconnect_with_host_check!
       log_with_prefix("Lost connection to MySQL, will retry to connect to same host")
-      begin
-        Retriable.retriable(host_retry_config) do
+
+      RECONNECT_RETRY_MAX_ITERATION.times do
+        begin
+          sleep(RECONNECT_RETRY_INTERVAL)
+
           # tries to reconnect. On failure will trigger a retry
           @connection.reconnect!
 
           if same_host_as_initial?
             # This is not an actual error, but controlled way to get the parent `Retriable.retriable` to retry
             # the statement that failed (since the Retriable gem only retries on errors).
-            raise ReconnectToHostSuccessful.new("LHM successfully reconnected to initial host: #{@initial_hostname} (server_id: #{@initial_server_id})")
+            log_with_prefix("LHM successfully reconnected to initial host: #{@initial_hostname} (server_id: #{@initial_server_id})")
+            return true
           else
             # New Master --> abort LHM (reconnecting will not change anything)
-            raise Lhm::Error.new("Reconnected to wrong host. Started migration on: #{@initial_hostname} (server_id: #{@initial_server_id}), but reconnected to: #{hostname} (server_id: #{@initial_server_id}).")
+            log_with_prefix("Reconnected to wrong host. Started migration on: #{@initial_hostname} (server_id: #{@initial_server_id}), but reconnected to: #{hostname} (server_id: #{server_id}).", :error)
+            return false
           end
+        rescue StandardError => e
+          # Retry if ActiveRecord cannot reach host
+          next if /Lost connection to MySQL server at 'reading initial communication packet'/ === e.message
+          log_with_prefix("Encountered error: [#{e.class}] #{e.message}. Will stop reconnection procedure.", :info)
+          return false
         end
-      rescue StandardError => e
-        # The parent Retriable.retriable is configured to retry if it encounters an error with the success message.
-        # Therefore, if the connection is re-established successfully AND the host is the same, LHM can retry the query
-        # that originally failed.
-        raise e if reconnect_successful?(e)
-        # If the connection was not successful, the parent retriable will raise any error that originated from the
-        # `@connection.reconnect!`
-        # Therefore, this error will cause the LHM to abort
-        raise Lhm::Error.new("LHM tried the reconnection procedure but failed. Latest error: #{e.message}")
       end
-    end
-
-    def reconnect_successful?(e)
-      e.is_a?(ReconnectToHostSuccessful)
+      false
     end
 
     # For a full list of configuration options see https://github.com/kamui/retriable
@@ -143,9 +164,6 @@ module Lhm
             /Unknown MySQL server host/,
             /connection is locked to hostgroup/,
             /The MySQL server is running with the --read-only option so it cannot execute this statement/,
-          ],
-          ReconnectToHostSuccessful => [
-            /#{RECONNECT_SUCCESSFUL_MESSAGE}/
           ]
         },
         multiplier: 1, # each successive interval grows by this factor
@@ -153,28 +171,6 @@ module Lhm
         tries: 20, # Number of attempts to make at running your code block (includes initial attempt).
         rand_factor: 0, # percentage to randomize the next retry interval time
         max_elapsed_time: Float::INFINITY, # max total time in seconds that code is allowed to keep being retried
-        on_retry: Proc.new do |exception, try_number, total_elapsed_time, next_interval|
-          if reconnect_successful?(exception)
-            log_with_prefix("#{exception.message} -- triggering retry", :info)
-          else
-            log_with_prefix("#{exception.class}: '#{exception.message}' - #{try_number} tries in #{total_elapsed_time} seconds and #{next_interval} seconds until the next try.", :error)
-          end
-        end
-      }.freeze
-    end
-
-    def host_retry_config
-      {
-        on: {
-          StandardError => [
-            /Lost connection to MySQL server at 'reading initial communication packet'/
-          ]
-        },
-        multiplier: 1, # each successive interval grows by this factor
-        base_interval: 0.25, # the initial interval in seconds between tries.
-        tries: 20, # Number of attempts to make at running your code block (includes initial attempt).
-        rand_factor: 0, # percentage to randomize the next retry interval time
-        max_elapsed_time: Float::INFINITY, # max total time in seconds that code is allowed to keep being retried
         on_retry: Proc.new do |exception, try_number, total_elapsed_time, next_interval|
           log_with_prefix("#{exception.class}: '#{exception.message}' - #{try_number} tries in #{total_elapsed_time} seconds and #{next_interval} seconds until the next try.", :error)
         end

data/lib/lhm/version.rb

@@ -2,5 +2,5 @@
 # Schmidt
 
 module Lhm
-  VERSION = '3.5.1'
+  VERSION = '3.5.5'
 end

data/lib/lhm.rb

@@ -28,7 +28,7 @@ module Lhm
   extend Throttler
   extend self
 
-  DEFAULT_LOGGER_OPTIONS =  { level: Logger::INFO, file: STDOUT }
+  DEFAULT_LOGGER_OPTIONS = { level: Logger::INFO, file: STDOUT }
 
   # Alters a table with the changes described in the block
   #
@@ -46,15 +46,19 @@ module Lhm
   #   Use atomic switch to rename tables (defaults to: true)
   #   If using a version of mysql affected by atomic switch bug, LHM forces user
   #   to set this option (see SqlHelper#supports_atomic_switch?)
+  # @option options [Boolean] :reconnect_with_consistent_host
+  #   Active / Deactivate ProxySQL-aware reconnection procedure (default to: false)
   # @yield [Migrator] Yielded Migrator object records the changes
   # @return [Boolean] Returns true if the migration finishes
   # @raise [Error] Raises Lhm::Error in case of a error and aborts the migration
   def change_table(table_name, options = {}, &block)
-    origin = Table.parse(table_name, connection)
-    invoker = Invoker.new(origin, connection)
-    block.call(invoker.migrator)
-    invoker.run(options)
-    true
+    with_flags(options) do
+      origin = Table.parse(table_name, connection)
+      invoker = Invoker.new(origin, connection)
+      block.call(invoker.migrator)
+      invoker.run(options)
+      true
+    end
   end
 
   # Cleanup tables and triggers
@@ -86,26 +90,16 @@ module Lhm
   # Setups DB connection
   #
   # @param [ActiveRecord::Base] connection ActiveRecord Connection
-  # @param [Hash] connection_options Optional options (defaults to: empty hash)
-  # @option connection_options [Boolean] :reconnect_with_consistent_host
-  #   Active / Deactivate ProxySQL-aware reconnection procedure (default to: false)
-  def setup(connection, connection_options = {})
-    @@connection = Connection.new(connection: connection, options: connection_options)
+  def setup(connection)
+    @@connection = Connection.new(connection: connection)
   end
 
-  # Setups DB connection
-  #
-  # @param [Hash] connection_options Optional options (defaults to: empty hash)
-  # @option connection_options [Boolean] :reconnect_with_consistent_host
-  #   Active / Deactivate ProxySQL-aware reconnection procedure (default to: false)
-  def connection(connection_options = nil)
-    if @@connection.nil?
-      raise 'Please call Lhm.setup' unless defined?(ActiveRecord)
-      @@connection = Connection.new(connection: ActiveRecord::Base.connection, options: connection_options || {})
-    else
-      @@connection.options = connection_options unless connection_options.nil?
-    end
-    @@connection
+  # Returns DB connection (or initializes it if not created yet)
+  def connection
+    @@connection ||= begin
+                       raise 'Please call Lhm.setup' unless defined?(ActiveRecord)
+                       @@connection = Connection.new(connection: ActiveRecord::Base.connection)
+                     end
   end
 
   def self.logger=(new_logger)
@@ -147,4 +141,16 @@ module Lhm
       false
     end
   end
+
+  def with_flags(options)
+    old_flags = {
+      reconnect_with_consistent_host: Lhm.connection.reconnect_with_consistent_host,
+    }
+
+    Lhm.connection.reconnect_with_consistent_host = options[:reconnect_with_consistent_host] || false
+
+    yield
+  ensure
+    Lhm.connection.reconnect_with_consistent_host = old_flags[:reconnect_with_consistent_host]
+  end
 end

data/spec/integration/atomic_switcher_spec.rb

@@ -39,10 +39,15 @@ describe Lhm::AtomicSwitcher do
                    .then
                    .returns([["dummy"]]) # Matches initial host -> triggers retry
 
-      connection = Lhm::Connection.new(connection: ar_connection, options: {reconnect_with_consistent_host: true})
+      connection = Lhm::Connection.new(connection: ar_connection, options: {
+        reconnect_with_consistent_host: true,
+        retriable: {
+          tries: 3,
+          base_interval: 0
+        }
+      })
 
-
-      switcher = Lhm::AtomicSwitcher.new(@migration, connection, retriable: { tries: 3, base_interval: 0 })
+      switcher = Lhm::AtomicSwitcher.new(@migration, connection)
 
       assert switcher.run
 
@@ -58,20 +63,26 @@ describe Lhm::AtomicSwitcher do
       ar_connection.stubs(:data_source_exists?).returns(true)
       ar_connection.stubs(:active?).returns(true)
       ar_connection.stubs(:execute).returns([["dummy"]], [["dummy"]], [["dummy"]])
-                .then
-                .raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
-                .then
-                .returns([["dummy"]]) # triggers retry 1
-                .then
-                .raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
-                .then
-                .returns([["dummy"]]) # triggers retry 2
-                .then
-                .raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.') # triggers retry 2
-
-      connection = Lhm::Connection.new(connection: ar_connection, options: {reconnect_with_consistent_host: true})
-
-      switcher = Lhm::AtomicSwitcher.new(@migration, connection, retriable: { tries: 2, base_interval: 0 })
+                   .then
+                   .raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
+                   .then
+                   .returns([["dummy"]]) # triggers retry 1
+                   .then
+                   .raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
+                   .then
+                   .returns([["dummy"]]) # triggers retry 2
+                   .then
+                   .raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.') # triggers retry 2
+
+      connection = Lhm::Connection.new(connection: ar_connection, options: {
+        reconnect_with_consistent_host: true,
+        retriable: {
+          tries: 2,
+          base_interval: 0
+        }
+      })
+
+      switcher = Lhm::AtomicSwitcher.new(@migration, connection)
 
       assert_raises(ActiveRecord::StatementInvalid) { switcher.run }
     end

data/spec/integration/chunker_spec.rb

@@ -173,6 +173,9 @@ describe Lhm::Chunker do
       printer.expect(:notify, :return_value, [Integer, Integer])
       printer.expect(:end, :return_value, [])
 
+      Lhm::Throttler::Slave.any_instance.stubs(:slave_hosts).returns(['127.0.0.1'])
+      Lhm::Throttler::SlaveLag.any_instance.stubs(:master_slave_hosts).returns(['127.0.0.1'])
+
       Lhm::Chunker.new(
         @migration, connection, { throttler: Lhm::Throttler::SlaveLag.new(stride: 100), printer: printer }
       ).run
@@ -215,17 +218,16 @@ describe Lhm::Chunker do
       printer.expects(:verify)
       printer.expects(:end)
 
-      throttler = Lhm::Throttler::SlaveLag.new(stride: 10, allowed_lag: 0)
+      Lhm::Throttler::Slave.any_instance.stubs(:slave_hosts).returns(['127.0.0.1'])
+      Lhm::Throttler::SlaveLag.any_instance.stubs(:master_slave_hosts).returns(['127.0.0.1'])
 
-      def throttler.slave_hosts
-        ['127.0.0.1']
-      end
+      throttler = Lhm::Throttler::SlaveLag.new(stride: 10, allowed_lag: 0)
 
       if master_slave_mode?
         def throttler.slave_connection(slave)
           config = ActiveRecord::Base.connection_pool.db_config.configuration_hash.dup
           config[:host] = slave
-          config[:port] = 3307
+          config[:port] = 33007
           ActiveRecord::Base.send('mysql2_connection', config)
         end
       end

data/spec/integration/integration_helper.rb

@@ -62,18 +62,16 @@ module IntegrationHelper
     )
   end
 
-  def connect_master_with_toxiproxy!(with_retry: false)
+  def connect_master_with_toxiproxy!
     connect!(
       '127.0.0.1',
       $db_config['master_toxic']['port'],
       $db_config['master_toxic']['user'],
-      $db_config['master_toxic']['password'],
-      with_retry)
+      $db_config['master_toxic']['password'])
   end
 
-  def connect!(hostname, port, user, password, with_retry = false)
-    adapter = ar_conn(hostname, port, user, password)
-    Lhm.setup(adapter,{reconnect_with_consistent_host: with_retry})
+  def connect!(hostname, port, user, password)
+    Lhm.setup(ar_conn(hostname, port, user, password))
     unless defined?(@@cleaned_up)
       Lhm.cleanup(true)
       @@cleaned_up  = true

data/spec/integration/lhm_spec.rb

@@ -592,7 +592,7 @@ describe Lhm do
       end
 
       it " should not try to reconnect if reconnect_with_consistent_host is not provided" do
-        connect_master_with_toxiproxy!(with_retry: false)
+        connect_master_with_toxiproxy!
 
         table_create(:users)
         100.times { |n| execute("insert into users set reference = '#{ n }'") }
@@ -610,7 +610,7 @@ describe Lhm do
       end
 
       it "should reconnect if reconnect_with_consistent_host is true" do
-        connect_master_with_toxiproxy!(with_retry: true)
+        connect_master_with_toxiproxy!
         mysql_disabled = false
 
         table_create(:users)
@@ -635,7 +635,7 @@ describe Lhm do
           method_added(:insert_and_return_count_of_rows_created)
         end
 
-        Lhm.change_table(:users, :atomic_switch => false) do |t|
+        Lhm.change_table(:users, atomic_switch: false, reconnect_with_consistent_host: true) do |t|
           t.ddl("ALTER TABLE #{t.name} CHANGE id id bigint (20) NOT NULL")
           t.ddl("ALTER TABLE #{t.name} DROP PRIMARY KEY, ADD PRIMARY KEY (username, id)")
           t.ddl("ALTER TABLE #{t.name} ADD INDEX (id)")
@@ -645,7 +645,6 @@ describe Lhm do
         log_lines = @logs.string.split("\n")
 
         assert log_lines.one?{ |line| line.include?("Lost connection to MySQL, will retry to connect to same host")}
-        assert log_lines.any?{ |line| line.include?("Lost connection to MySQL server at 'reading initial communication packet'")}
         assert log_lines.one?{ |line| line.include?("LHM successfully reconnected to initial host")}
         assert log_lines.one?{ |line| line.include?("100% complete")}
 

data/spec/integration/proxysql_spec.rb

@@ -29,6 +29,6 @@ describe "ProxySQL integration" do
       port: "33005",
       )
 
-    assert_equal conn.query("/*maintenance:lhm*/SELECT @@global.hostname as host").each.first["host"], "mysql-1"
+    assert_equal conn.query("SELECT @@global.hostname as host #{Lhm::ProxySQLHelper::ANNOTATION}").each.first["host"], "mysql-1"
   end
 end

data/spec/integration/sql_retry/lock_wait_spec.rb

@@ -55,7 +55,7 @@ describe Lhm::SqlRetry do
   it "successfully executes the SQL despite the errors encountered" do
     # Start a thread to retry, once the lock is held, execute the block
     @helper.with_waiting_lock do |waiting_connection|
-      sql_retry = Lhm::SqlRetry.new(waiting_connection, options: {
+      sql_retry = Lhm::SqlRetry.new(waiting_connection, retry_options: {
         base_interval: 0.2, # first retry after 200ms
         multiplier: 1, # subsequent retries wait 1x longer than first retry (no change)
         tries: 3, # we only need 3 tries (including the first) for the scenario described below
@@ -98,7 +98,7 @@ describe Lhm::SqlRetry do
     puts "*" * 64
     # Start a thread to retry, once the lock is held, execute the block
     @helper.with_waiting_lock do |waiting_connection|
-      sql_retry = Lhm::SqlRetry.new(waiting_connection, options: {
+      sql_retry = Lhm::SqlRetry.new(waiting_connection, retry_options: {
         base_interval: 0.2, # first retry after 200ms
         multiplier: 1, # subsequent retries wait 1x longer than first retry (no change)
         tries: 2, # we need 3 tries (including the first) for the scenario described below, but we only get two...we will fail

data/spec/integration/sql_retry/retry_with_proxysql_spec.rb

@@ -18,7 +18,7 @@ describe Lhm::SqlRetry, "ProxiSQL tests for LHM retry" do
 
     @connection = DBConnectionHelper::new_mysql_connection(:proxysql, true, true)
 
-    @lhm_retry = Lhm::SqlRetry.new(@connection, options: {},
+    @lhm_retry = Lhm::SqlRetry.new(@connection, retry_options: {},
                                    reconnect_with_consistent_host: true)
   end
 
@@ -38,7 +38,7 @@ describe Lhm::SqlRetry, "ProxiSQL tests for LHM retry" do
       end
     end
     assert_equal Lhm::Error, e.class
-    assert_match(/LHM tried the reconnection procedure but failed. Latest error:/, e.message)
+    assert_match(/LHM tried the reconnection procedure but failed. Aborting/, e.message)
   end
 
   it "Will retry until connection is achieved" do
@@ -61,9 +61,10 @@ describe Lhm::SqlRetry, "ProxiSQL tests for LHM retry" do
   it "Will abort if new writer is not same host" do
     # The hostname will be constant before the blip
     Lhm::SqlRetry.any_instance.stubs(:hostname).returns("mysql-1").then.returns("mysql-2")
+    Lhm::SqlRetry.any_instance.stubs(:server_id).returns(1).then.returns(2)
 
     # Need new instance for stub to take into effect
-    lhm_retry = Lhm::SqlRetry.new(@connection, options: {},
+    lhm_retry = Lhm::SqlRetry.new(@connection, retry_options: {},
                                   reconnect_with_consistent_host: true)
 
     e = assert_raises Lhm::Error do
@@ -76,12 +77,12 @@ describe Lhm::SqlRetry, "ProxiSQL tests for LHM retry" do
     end
 
     assert_equal e.class, Lhm::Error
-    assert_match(/LHM tried the reconnection procedure but failed. Latest error: Reconnected to wrong host/, e.message)
+    assert_match(/LHM tried the reconnection procedure but failed. Aborting/, e.message)
 
     logs = @logger.string.split("\n")
 
     assert logs.first.include?("Lost connection to MySQL, will retry to connect to same host")
-    assert logs.last.include?("Lost connection to MySQL server at 'reading initial communication packet")
+    assert logs.last.include?("Reconnected to wrong host. Started migration on: mysql-1 (server_id: 1), but reconnected to: mysql-2 (server_id: 2).")
   end
 
   it "Will abort if failover happens (mimicked with proxySQL)" do
@@ -98,11 +99,11 @@ describe Lhm::SqlRetry, "ProxiSQL tests for LHM retry" do
     end
 
     assert_equal e.class, Lhm::Error
-    assert_match(/LHM tried the reconnection procedure but failed. Latest error: Reconnected to wrong host/, e.message)
+    assert_match(/LHM tried the reconnection procedure but failed. Aborting/, e.message)
 
     logs = @logger.string.split("\n")
 
     assert logs.first.include?("Lost connection to MySQL, will retry to connect to same host")
-    assert logs.last.include?("Lost connection to MySQL server at 'reading initial communication packet")
+    assert logs.last.include?("Reconnected to wrong host. Started migration on: mysql-1 (server_id: 1), but reconnected to: mysql-2 (server_id: 2).")
   end
 end

data/spec/test_helper.rb

@@ -28,6 +28,9 @@ logger = Logger.new STDOUT
 logger.level = Logger::WARN
 Lhm.logger = logger
 
+# Want test to be efficient without having to wait the normal value of 120s
+Lhm::SqlRetry::RECONNECT_RETRY_MAX_ITERATION = 4
+
 def without_verbose(&block)
   old_verbose, $VERBOSE = $VERBOSE, nil
   yield