lhm-shopify 3.5.4 → 4.0.0

data/lib/lhm/sql_retry.rb

@@ -18,6 +18,12 @@ module Lhm
   class SqlRetry
     RECONNECT_SUCCESSFUL_MESSAGE = "LHM successfully reconnected to initial host:"
     CLOUDSQL_VERSION_COMMENT = "(Google)"
+    # Will retry for 120 seconds (approximately, since connecting takes time).
+    RECONNECT_RETRY_MAX_ITERATION = 120
+    RECONNECT_RETRY_INTERVAL = 1
+    # Will abort the LHM if it had to reconnect more than 25 times in a single run (indicator that there might be
+    # something wrong with the network and would be better to run the LHM at a later time).
+    RECONNECTION_MAXIMUM = 25
 
     MYSQL_VAR_NAMES = {
       hostname: "@@global.hostname",
@@ -25,9 +31,6 @@ module Lhm
       version_comment: "@@version_comment",
     }
 
-    # This internal error is used to trigger retries from the parent Retriable.retriable in #with_retries
-    class ReconnectToHostSuccessful < Lhm::Error; end
-
     def initialize(connection, retry_options: {}, reconnect_with_consistent_host: false)
       @connection = connection
       self.retry_config = retry_options
@@ -38,11 +41,14 @@ module Lhm
     def with_retries(log_prefix: nil)
       @log_prefix = log_prefix || "" # No prefix. Just logs
 
+      # Amount of time LHM had to reconnect. Aborting if more than RECONNECTION_MAXIMUM
+      reconnection_counter = 0
+
       Retriable.retriable(@retry_config) do
         # Using begin -> rescue -> end for Ruby 2.4 compatibility
         begin
           if @reconnect_with_consistent_host
-            raise Lhm::Error.new("Could not reconnected to initial MySQL host. Aborting to avoid data-loss") unless same_host_as_initial?
+            raise Lhm::Error.new("MySQL host has changed since the start of the LHM. Aborting to avoid data-loss") unless same_host_as_initial?
           end
 
           yield(@connection)
@@ -51,7 +57,18 @@ module Lhm
           # The error will be raised the connection is still active (i.e. no need to reconnect) or if the connection is
           # dead (i.e. not active) and @reconnect_with_host is false (i.e. instructed not to reconnect)
           raise e if @connection.active? || (!@connection.active? && !@reconnect_with_consistent_host)
-          reconnect_with_host_check! if @reconnect_with_consistent_host
+
+          # Lhm could be stuck in a weird state where it loses connection, reconnects and re looses-connection instantly
+          # after, creating an infinite loop (because of the usage of `retry`). Hence, abort after 25 reconnections
+          raise Lhm::Error.new("LHM reached host reconnection max of #{RECONNECTION_MAXIMUM} times. " \
+            "Please try again later.") if reconnection_counter > RECONNECTION_MAXIMUM
+
+          reconnection_counter += 1
+          if reconnect_with_host_check!
+            retry
+          else
+            raise Lhm::Error.new("LHM tried the reconnection procedure but failed. Aborting")
+          end
         end
       end
     end
@@ -105,34 +122,32 @@ module Lhm
 
     def reconnect_with_host_check!
       log_with_prefix("Lost connection to MySQL, will retry to connect to same host")
-      begin
-        Retriable.retriable(host_retry_config) do
+
+      RECONNECT_RETRY_MAX_ITERATION.times do
+        begin
+          sleep(RECONNECT_RETRY_INTERVAL)
+
           # tries to reconnect. On failure will trigger a retry
           @connection.reconnect!
 
           if same_host_as_initial?
             # This is not an actual error, but controlled way to get the parent `Retriable.retriable` to retry
             # the statement that failed (since the Retriable gem only retries on errors).
-            raise ReconnectToHostSuccessful.new("LHM successfully reconnected to initial host: #{@initial_hostname} (server_id: #{@initial_server_id})")
+            log_with_prefix("LHM successfully reconnected to initial host: #{@initial_hostname} (server_id: #{@initial_server_id})")
+            return true
           else
             # New Master --> abort LHM (reconnecting will not change anything)
-            raise Lhm::Error.new("Reconnected to wrong host. Started migration on: #{@initial_hostname} (server_id: #{@initial_server_id}), but reconnected to: #{hostname} (server_id: #{@initial_server_id}).")
+            log_with_prefix("Reconnected to wrong host. Started migration on: #{@initial_hostname} (server_id: #{@initial_server_id}), but reconnected to: #{hostname} (server_id: #{server_id}).", :error)
+            return false
           end
+        rescue StandardError => e
+          # Retry if ActiveRecord cannot reach host
+          next if /Lost connection to MySQL server at 'reading initial communication packet'/ === e.message
+          log_with_prefix("Encountered error: [#{e.class}] #{e.message}. Will stop reconnection procedure.", :info)
+          return false
         end
-      rescue StandardError => e
-        # The parent Retriable.retriable is configured to retry if it encounters an error with the success message.
-        # Therefore, if the connection is re-established successfully AND the host is the same, LHM can retry the query
-        # that originally failed.
-        raise e if reconnect_successful?(e)
-        # If the connection was not successful, the parent retriable will raise any error that originated from the
-        # `@connection.reconnect!`
-        # Therefore, this error will cause the LHM to abort
-        raise Lhm::Error.new("LHM tried the reconnection procedure but failed. Latest error: #{e.message}")
       end
-    end
-
-    def reconnect_successful?(e)
-      e.is_a?(ReconnectToHostSuccessful)
+      false
     end
 
     # For a full list of configuration options see https://github.com/kamui/retriable
@@ -149,9 +164,6 @@ module Lhm
             /Unknown MySQL server host/,
             /connection is locked to hostgroup/,
             /The MySQL server is running with the --read-only option so it cannot execute this statement/,
-          ],
-          ReconnectToHostSuccessful => [
-            /#{RECONNECT_SUCCESSFUL_MESSAGE}/
           ]
         },
         multiplier: 1, # each successive interval grows by this factor
@@ -159,28 +171,6 @@ module Lhm
         tries: 20, # Number of attempts to make at running your code block (includes initial attempt).
         rand_factor: 0, # percentage to randomize the next retry interval time
         max_elapsed_time: Float::INFINITY, # max total time in seconds that code is allowed to keep being retried
-        on_retry: Proc.new do |exception, try_number, total_elapsed_time, next_interval|
-          if reconnect_successful?(exception)
-            log_with_prefix("#{exception.message} -- triggering retry", :info)
-          else
-            log_with_prefix("#{exception.class}: '#{exception.message}' - #{try_number} tries in #{total_elapsed_time} seconds and #{next_interval} seconds until the next try.", :error)
-          end
-        end
-      }.freeze
-    end
-
-    def host_retry_config
-      {
-        on: {
-          StandardError => [
-            /Lost connection to MySQL server at 'reading initial communication packet'/
-          ]
-        },
-        multiplier: 1, # each successive interval grows by this factor
-        base_interval: 0.25, # the initial interval in seconds between tries.
-        tries: 20, # Number of attempts to make at running your code block (includes initial attempt).
-        rand_factor: 0, # percentage to randomize the next retry interval time
-        max_elapsed_time: Float::INFINITY, # max total time in seconds that code is allowed to keep being retried
         on_retry: Proc.new do |exception, try_number, total_elapsed_time, next_interval|
           log_with_prefix("#{exception.class}: '#{exception.message}' - #{try_number} tries in #{total_elapsed_time} seconds and #{next_interval} seconds until the next try.", :error)
         end

data/lib/lhm/throttler/replica_lag.rb

@@ -0,0 +1,162 @@
+module Lhm
+  module Throttler
+
+    def self.format_hosts(hosts)
+      formatted_hosts = []
+      hosts.each do |host|
+        if host && !host.match(/localhost/) && !host.match(/127.0.0.1/)
+          formatted_hosts << host.partition(':')[0]
+        end
+      end
+      formatted_hosts
+    end
+
+    class ReplicaLag
+      include Command
+
+      INITIAL_TIMEOUT = 0.1
+      DEFAULT_STRIDE = 2_000
+      DEFAULT_MAX_ALLOWED_LAG = 10
+
+      MAX_TIMEOUT = INITIAL_TIMEOUT * 1024
+
+      attr_accessor :timeout_seconds, :allowed_lag, :stride, :connection
+
+      def initialize(options = {})
+        @timeout_seconds = INITIAL_TIMEOUT
+        @stride = options[:stride] || DEFAULT_STRIDE
+        @allowed_lag = options[:allowed_lag] || DEFAULT_MAX_ALLOWED_LAG
+        @replicas = {}
+        @get_config = options[:current_config]
+        @check_only = options[:check_only]
+      end
+
+      def execute
+        sleep(throttle_seconds)
+      end
+
+      private
+
+      def throttle_seconds
+        lag = max_current_replica_lag
+
+        if lag > @allowed_lag && @timeout_seconds < MAX_TIMEOUT
+          Lhm.logger.info("Increasing timeout between strides from #{@timeout_seconds} to #{@timeout_seconds * 2} because #{lag} seconds of replica lag detected is greater than the maximum of #{@allowed_lag} seconds allowed.")
+          @timeout_seconds = @timeout_seconds * 2
+        elsif lag <= @allowed_lag && @timeout_seconds > INITIAL_TIMEOUT
+          Lhm.logger.info("Decreasing timeout between strides from #{@timeout_seconds} to #{@timeout_seconds / 2} because #{lag} seconds of replica lag detected is less than or equal to the #{@allowed_lag} seconds allowed.")
+          @timeout_seconds = @timeout_seconds / 2
+        else
+          @timeout_seconds
+        end
+      end
+
+      def replicas
+        @replicas[@connection] ||= get_replicas
+      end
+
+      def get_replicas
+        replicas = []
+        if @check_only.nil? or !@check_only.respond_to?(:call)
+          replica_hosts = master_replica_hosts
+          while replica_hosts.any? do
+            host = replica_hosts.pop
+            replica = Replica.new(host, @get_config)
+            if !replicas.map(&:host).include?(host) && replica.connection
+              replicas << replica
+              replica_hosts.concat(replica.replica_hosts)
+            end
+          end
+        else
+          replica_config = @check_only.call
+          replicas << Replica.new(replica_config['host'], @get_config)
+        end
+        replicas
+      end
+
+      def master_replica_hosts
+        Throttler.format_hosts(@connection.select_values(Replica::SQL_SELECT_REPLICA_HOSTS))
+      end
+
+      def max_current_replica_lag
+        max = replicas.map { |replica| replica.lag }.push(0).max
+        Lhm.logger.info "Max current replica lag: #{max}"
+        max
+      end
+    end
+
+    class Replica
+      SQL_SELECT_REPLICA_HOSTS = "SELECT host FROM information_schema.processlist WHERE command LIKE 'Binlog Dump%'"
+      SQL_SELECT_MAX_REPLICA_LAG = 'SHOW SLAVE STATUS'
+
+      attr_reader :host, :connection
+
+      def initialize(host, connection_config = nil)
+        @host = host
+        @connection_config = prepare_connection_config(connection_config)
+        @connection = client(@connection_config)
+      end
+
+      def replica_hosts
+        Throttler.format_hosts(query_connection(SQL_SELECT_REPLICA_HOSTS, 'host'))
+      end
+
+      def lag
+        query_connection(SQL_SELECT_MAX_REPLICA_LAG, 'Seconds_Behind_Master').first.to_i
+      end
+
+      private
+
+      def client(config)
+        begin
+          Lhm.logger.info "Connecting to #{@host} on database: #{config[:database]}"
+          Mysql2::Client.new(config)
+        rescue Mysql2::Error => e
+          Lhm.logger.info "Error connecting to #{@host}: #{e}"
+          nil
+        end
+      end
+
+      def prepare_connection_config(config_proc)
+        config = if config_proc
+          if config_proc.respond_to?(:call) # if we get a proc
+            config_proc.call
+          else
+            raise ArgumentError, "Expected #{config_proc.inspect} to respond to `call`"
+          end
+        else
+          db_config
+        end
+        config.deep_symbolize_keys!
+        config[:host] = @host
+        config
+      end
+
+      def query_connection(query, result)
+        begin
+          @connection.query(query).map { |row| row[result] }
+        rescue Mysql2::Error => e
+          Lhm.logger.info "Unable to connect and/or query #{host}: #{e}"
+          [nil]
+        end
+      end
+
+      private
+
+      def db_config
+        if ar_supports_db_config?
+          ActiveRecord::Base.connection_pool.db_config.configuration_hash.dup
+        else
+          ActiveRecord::Base.connection_pool.spec.config.dup
+        end
+      end
+
+      def ar_supports_db_config?
+        # https://api.rubyonrails.org/v6.0/classes/ActiveRecord/ConnectionAdapters/ConnectionPool.html <-- has spec
+        # vs
+        # https://api.rubyonrails.org/v6.1/classes/ActiveRecord/ConnectionAdapters/ConnectionPool.html <-- has db_config
+        ActiveRecord::VERSION::MAJOR > 6 || ActiveRecord::VERSION::MAJOR == 6 && ActiveRecord::VERSION::MINOR >= 1
+      end
+    end
+  end
+end

data/lib/lhm/throttler/slave_lag.rb

@@ -1,161 +1,11 @@
+require 'lhm/throttler/replica_lag'
+
 module Lhm
   module Throttler
-
-    def self.format_hosts(hosts)
-      formatted_hosts = []
-      hosts.each do |host|
-        if host && !host.match(/localhost/) && !host.match(/127.0.0.1/)
-          formatted_hosts << host.partition(':')[0]
-        end
-      end
-      formatted_hosts
-    end
-
-    class SlaveLag
-      include Command
-
-      INITIAL_TIMEOUT = 0.1
-      DEFAULT_STRIDE = 2_000
-      DEFAULT_MAX_ALLOWED_LAG = 10
-
-      MAX_TIMEOUT = INITIAL_TIMEOUT * 1024
-
-      attr_accessor :timeout_seconds, :allowed_lag, :stride, :connection
-
+    class SlaveLag < ReplicaLag
       def initialize(options = {})
-        @timeout_seconds = INITIAL_TIMEOUT
-        @stride = options[:stride] || DEFAULT_STRIDE
-        @allowed_lag = options[:allowed_lag] || DEFAULT_MAX_ALLOWED_LAG
-        @slaves = {}
-        @get_config = options[:current_config]
-        @check_only = options[:check_only]
-      end
-
-      def execute
-        sleep(throttle_seconds)
-      end
-
-      private
-
-      def throttle_seconds
-        lag = max_current_slave_lag
-
-        if lag > @allowed_lag && @timeout_seconds < MAX_TIMEOUT
-          Lhm.logger.info("Increasing timeout between strides from #{@timeout_seconds} to #{@timeout_seconds * 2} because #{lag} seconds of slave lag detected is greater than the maximum of #{@allowed_lag} seconds allowed.")
-          @timeout_seconds = @timeout_seconds * 2
-        elsif lag <= @allowed_lag && @timeout_seconds > INITIAL_TIMEOUT
-          Lhm.logger.info("Decreasing timeout between strides from #{@timeout_seconds} to #{@timeout_seconds / 2} because #{lag} seconds of slave lag detected is less than or equal to the #{@allowed_lag} seconds allowed.")
-          @timeout_seconds = @timeout_seconds / 2
-        else
-          @timeout_seconds
-        end
-      end
-
-      def slaves
-        @slaves[@connection] ||= get_slaves
-      end
-
-      def get_slaves
-        slaves = []
-        if @check_only.nil? or !@check_only.respond_to?(:call)
-          slave_hosts = master_slave_hosts
-          while slave_hosts.any? do
-            host = slave_hosts.pop
-            slave = Slave.new(host, @get_config)
-            if !slaves.map(&:host).include?(host) && slave.connection
-              slaves << slave
-              slave_hosts.concat(slave.slave_hosts)
-            end
-          end
-        else
-          slave_config = @check_only.call
-          slaves << Slave.new(slave_config['host'], @get_config)
-        end
-        slaves
-      end
-
-      def master_slave_hosts
-        Throttler.format_hosts(@connection.select_values(Slave::SQL_SELECT_SLAVE_HOSTS))
-      end
-
-      def max_current_slave_lag
-        max = slaves.map { |slave| slave.lag }.push(0).max
-        Lhm.logger.info "Max current slave lag: #{max}"
-        max
-      end
-    end
-
-    class Slave
-      SQL_SELECT_SLAVE_HOSTS = "SELECT host FROM information_schema.processlist WHERE command LIKE 'Binlog Dump%'"
-      SQL_SELECT_MAX_SLAVE_LAG = 'SHOW SLAVE STATUS'
-
-      attr_reader :host, :connection
-
-      def initialize(host, connection_config = nil)
-        @host = host
-        @connection_config = prepare_connection_config(connection_config)
-        @connection = client(@connection_config)
-      end
-
-      def slave_hosts
-        Throttler.format_hosts(query_connection(SQL_SELECT_SLAVE_HOSTS, 'host'))
-      end
-
-      def lag
-        query_connection(SQL_SELECT_MAX_SLAVE_LAG, 'Seconds_Behind_Master').first.to_i
-      end
-
-      private
-
-      def client(config)
-        begin
-          Lhm.logger.info "Connecting to #{@host} on database: #{config[:database]}"
-          Mysql2::Client.new(config)
-        rescue Mysql2::Error => e
-          Lhm.logger.info "Error connecting to #{@host}: #{e}"
-          nil
-        end
-      end
-
-      def prepare_connection_config(config_proc)
-        config = if config_proc
-          if config_proc.respond_to?(:call) # if we get a proc
-            config_proc.call
-          else
-            raise ArgumentError, "Expected #{config_proc.inspect} to respond to `call`"
-          end
-        else
-          db_config
-        end
-        config.deep_symbolize_keys!
-        config[:host] = @host
-        config
-      end
-
-      def query_connection(query, result)
-        begin
-          @connection.query(query).map { |row| row[result] }
-        rescue Mysql2::Error => e
-          Lhm.logger.info "Unable to connect and/or query #{host}: #{e}"
-          [nil]
-        end
-      end
-
-      private
-
-      def db_config
-        if ar_supports_db_config?
-          ActiveRecord::Base.connection_pool.db_config.configuration_hash.dup
-        else
-          ActiveRecord::Base.connection_pool.spec.config.dup
-        end
-      end
-
-      def ar_supports_db_config?
-        # https://api.rubyonrails.org/v6.0/classes/ActiveRecord/ConnectionAdapters/ConnectionPool.html <-- has spec
-        # vs
-        # https://api.rubyonrails.org/v6.1/classes/ActiveRecord/ConnectionAdapters/ConnectionPool.html <-- has db_config
-        ActiveRecord::VERSION::MAJOR > 6 || ActiveRecord::VERSION::MAJOR == 6 && ActiveRecord::VERSION::MINOR >= 1
+        Lhm.logger.warn("Class `SlaveLag` is deprecated. Use `ReplicaLag` class instead.")
+        super(options)
       end
     end
   end

data/lib/lhm/throttler/threads_running.rb

@@ -3,14 +3,16 @@ module Lhm
     class ThreadsRunning
       include Command
 
+      DEFAULT_STRIDE = 2_000
       DEFAULT_INITIAL_TIMEOUT = 0.1
       DEFAULT_HEALTHY_RANGE = (0..50)
 
-      attr_accessor :timeout_seconds, :healthy_range, :connection
+      attr_accessor :timeout_seconds, :healthy_range, :connection, :stride
       attr_reader :max_timeout_seconds, :initial_timeout_seconds
 
       def initialize(options = {})
         @initial_timeout_seconds = options[:initial_timeout] || DEFAULT_INITIAL_TIMEOUT
+        @stride = options[:stride] || DEFAULT_STRIDE
         @max_timeout_seconds = options[:max_timeout] || (@initial_timeout_seconds * 1024)
         @timeout_seconds = @initial_timeout_seconds
         @healthy_range = options[:healthy_range] || DEFAULT_HEALTHY_RANGE

data/lib/lhm/throttler.rb

@@ -1,12 +1,16 @@
 require 'lhm/throttler/time'
+require 'lhm/throttler/replica_lag'
 require 'lhm/throttler/slave_lag'
 require 'lhm/throttler/threads_running'
 
 module Lhm
   module Throttler
-    CLASSES = { :time_throttler => Throttler::Time,
-                :slave_lag_throttler => Throttler::SlaveLag,
-                :threads_running_throttler => Throttler::ThreadsRunning }
+    CLASSES = {
+      :time_throttler => Throttler::Time,
+      :replica_lag_throttler => Throttler::ReplicaLag,
+      :slave_lag_throttler => Throttler::SlaveLag,
+      :threads_running_throttler => Throttler::ThreadsRunning
+    }
 
     def throttler
       @throttler ||= Throttler::Time.new

data/lib/lhm/version.rb

@@ -2,5 +2,5 @@
 # Schmidt
 
 module Lhm
-  VERSION = '3.5.4'
+  VERSION = '4.0.0'
 end

data/spec/.lhm.example

@@ -1,4 +1,4 @@
 mysqldir=/usr/local/mysql
 basedir=~/lhm-cluster
 master_port=3306
-slave_port=3307
+replica_port=3307

data/spec/README.md

@@ -1,4 +1,4 @@
-# Preparing for master slave integration tests
+# Preparing for master replica integration tests
 
 ## Configuration
 
@@ -7,10 +7,10 @@ create ~/.lhm:
     mysqldir=/usr/local/mysql
     basedir=~/lhm-cluster
     master_port=3306
-    slave_port=3307
+    replica_port=3307
 
 mysqldir specifies the location of your mysql install. basedir is the
-directory master and slave databases will get installed into.
+directory master and replica databases will get installed into.
 
 ## Automatic setup
 
@@ -18,8 +18,8 @@ directory master and slave databases will get installed into.
 
     bin/lhm-spec-clobber.sh
 
-You can set the integration specs up to run against a master slave setup by
-running the included that. This deletes the configured lhm master slave setup and reinstalls and configures a master slave setup.
+You can set the integration specs up to run against a master replica setup by
+running the included that. This deletes the configured lhm master replica setup and reinstalls and configures a master replica setup.
 
 Follow the manual instructions if you want more control over this process.
 
@@ -33,7 +33,7 @@ Follow the manual instructions if you want more control over this process.
 
     basedir=/opt/lhm-luster
     mysqld --defaults-file="$basedir/master/my.cnf"
-    mysqld --defaults-file="$basedir/slave/my.cnf"
+    mysqld --defaults-file="$basedir/replica/my.cnf"
 
 ### run the grants
 
@@ -46,13 +46,12 @@ Setup the dependency gems
     export BUNDLE_GEMFILE=gemfiles/ar-4.2_mysql2.gemfile
     bundle install
 
-To run specs in slave mode, set the MASTER_SLAVE=1 when running tests:
+To run specs in replica mode, set the MASTER_REPLICA=1 when running tests:
 
-    MASTER_SLAVE=1 bundle exec rake specs
+    MASTER_REPLICA=1 bundle exec rake specs
 
 # connecting
 
 you can connect by running (with the respective ports):
 
     mysql --protocol=TCP -p3307
-

data/spec/integration/atomic_switcher_spec.rb

@@ -110,7 +110,7 @@ describe Lhm::AtomicSwitcher do
       switcher = Lhm::AtomicSwitcher.new(@migration, connection)
       switcher.run
 
-      slave do
+      replica do
         value(data_source_exists?(@origin)).must_equal true
         value(table_read(@migration.archive_name).columns.keys).must_include 'origin'
       end
@@ -120,7 +120,7 @@ describe Lhm::AtomicSwitcher do
       switcher = Lhm::AtomicSwitcher.new(@migration, connection)
       switcher.run
 
-      slave do
+      replica do
         value(data_source_exists?(@destination)).must_equal false
         value(table_read(@origin.name).columns.keys).must_include 'destination'
       end

data/spec/integration/chunk_insert_spec.rb

@@ -19,10 +19,10 @@ describe Lhm::ChunkInsert do
       assert_equal 1, @instance.insert_and_return_count_of_rows_created
     end
 
-    it "inserts the record into the slave" do
+    it "inserts the record into the replica" do
       @instance.insert_and_return_count_of_rows_created
 
-      slave do
+      replica do
         value(count_all(@destination.name)).must_equal(1)
       end
     end