lhm-shopify 3.5.0 → 3.5.1

data/lib/lhm/cleanup/current.rb

@@ -54,7 +54,7 @@ module Lhm
 
       def execute_ddls
         ddls.each do |ddl|
-            @connection.execute(ddl, @retry_config)
+            @connection.execute(ddl, should_retry: true, retry_options: @retry_config)
         end
         Lhm.logger.info("Dropped triggers on #{@lhm_triggers_for_origin.join(', ')}")
         Lhm.logger.info("Dropped tables #{@lhm_triggers_for_origin.join(', ')}")

data/lib/lhm/connection.rb

@@ -1,4 +1,5 @@
 require 'delegate'
+require 'lhm/sql_retry'
 
 module Lhm
   class Connection < SimpleDelegator
@@ -8,36 +9,74 @@ module Lhm
     alias connection __getobj__
     alias connection= __setobj__
 
-    def initialize(connection:, default_log_prefix: nil, retry_options: {})
+    def initialize(connection:, default_log_prefix: nil, options: {}, retry_config: {})
       @default_log_prefix = default_log_prefix
-      @retry_options = retry_options || default_retry_config
+      @retry_options = retry_config || default_retry_config
       @sql_retry = Lhm::SqlRetry.new(
         connection,
-        retry_options,
+        options: retry_config,
+        reconnect_with_consistent_host: options[:reconnect_with_consistent_host] || false
       )
 
       # Creates delegation for the ActiveRecord Connection
       super(connection)
     end
 
-    def execute(query, retry_options = {})
-      exec_with_retries(:execute, query, retry_options)
+    def options=(options)
+      # If any other flags are added. Add the "processing" here
+      @sql_retry.reconnect_with_consistent_host = options[:reconnect_with_consistent_host] || false
     end
 
-    def update(query, retry_options = {})
-      exec_with_retries(:update, query, retry_options)
+    def execute(query, should_retry: false, retry_options: {})
+      if should_retry
+        exec_with_retries(:execute, query, retry_options)
+      else
+        exec(:execute, query)
+      end
     end
 
-    def select_value(query, retry_options = {})
-      exec_with_retries(:select_value, query, retry_options)
+    def update(query, should_retry: false, retry_options: {})
+      if should_retry
+        exec_with_retries(:update, query, retry_options)
+      else
+        exec(:update, query)
+      end
+    end
+
+    def select_value(query, should_retry: false, retry_options: {})
+      if should_retry
+        exec_with_retries(:select_value, query, retry_options)
+      else
+        exec(:select_value, query)
+      end
+    end
+
+    def select_values(query, should_retry: false, retry_options: {})
+      if should_retry
+        exec_with_retries(:select_values, query, retry_options)
+      else
+        exec(:select_values, query)
+      end
+    end
+
+    def select_one(query, should_retry: false, retry_options: {})
+      if should_retry
+        exec_with_retries(:select_one, query, retry_options)
+      else
+        exec(:select_one, query)
+      end
     end
 
     private
 
+    def exec(method, sql)
+      connection.public_send(method, Lhm::ProxySQLHelper.tagged(sql))
+    end
+
     def exec_with_retries(method, sql, retry_options = {})
       retry_options[:log_prefix] ||= file
       @sql_retry.with_retries(retry_options) do |conn|
-        conn.public_send(method, sql)
+        conn.public_send(method, Lhm::ProxySQLHelper.tagged(sql))
       end
     end
 
@@ -51,7 +90,7 @@ module Lhm
 
     def relevant_caller
       lhm_stack = caller.select { |x| x.include?("/lhm") }
-      first_candidate_index = lhm_stack.find_index {|line| !line.include?(__FILE__)}
+      first_candidate_index = lhm_stack.find_index { |line| !line.include?(__FILE__) }
 
       # Find the file that called the `#execute` (fallbacks to current file)
       return lhm_stack.first unless first_candidate_index

data/lib/lhm/entangler.rb

@@ -86,14 +86,14 @@ module Lhm
 
     def before
       entangle.each do |stmt|
-        @connection.execute(stmt, @retry_options)
+        @connection.execute(stmt, should_retry: true, retry_options: @retry_options)
       end
       Lhm.logger.info("Created triggers on #{@origin.name}")
     end
 
     def after
       untangle.each do |stmt|
-        @connection.execute(stmt, @retry_options)
+        @connection.execute(stmt, should_retry: true, retry_options: @retry_options)
       end
       Lhm.logger.info("Dropped triggers on #{@origin.name}")
     end

data/lib/lhm/invoker.rb

@@ -16,8 +16,8 @@ module Lhm
   class Invoker
     include SqlHelper
     LOCK_WAIT_TIMEOUT_DELTA = 10
-      INNODB_LOCK_WAIT_TIMEOUT_MAX=1073741824.freeze # https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_lock_wait_timeout
-      LOCK_WAIT_TIMEOUT_MAX=31536000.freeze # https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html
+    INNODB_LOCK_WAIT_TIMEOUT_MAX=1073741824.freeze # https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_lock_wait_timeout
+    LOCK_WAIT_TIMEOUT_MAX=31536000.freeze # https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html
 
     attr_reader :migrator, :connection
 

data/lib/lhm/proxysql_helper.rb

@@ -0,0 +1,10 @@
+module Lhm
+  module ProxySQLHelper
+    extend self
+    ANNOTATION = "/*maintenance:lhm*/"
+
+    def tagged(sql)
+      "#{ANNOTATION}#{sql}"
+    end
+  end
+end

data/lib/lhm/sql_retry.rb

@@ -1,5 +1,6 @@
 require 'retriable'
 require 'lhm/sql_helper'
+require 'lhm/proxysql_helper'
 
 module Lhm
   # SqlRetry standardizes the interface for retry behavior in components like
@@ -15,26 +16,117 @@ module Lhm
   # https://github.com/kamui/retriable. Additionally, a "log_prefix" option,
   # which is unique to SqlRetry can be used to prefix log output.
   class SqlRetry
-    def initialize(connection, options = {})
+    RECONNECT_SUCCESSFUL_MESSAGE = "LHM successfully reconnected to initial host:"
+    CLOUDSQL_VERSION_COMMENT = "(Google)"
+
+    MYSQL_VAR_NAMES = {
+      hostname: "@@global.hostname",
+      server_id: "@@global.server_id",
+      version_comment: "@@version_comment",
+    }
+
+    # This internal error is used to trigger retries from the parent Retriable.retriable in #with_retries
+    class ReconnectToHostSuccessful < Lhm::Error; end
+
+    def initialize(connection, options: {}, reconnect_with_consistent_host: true)
       @connection = connection
+      @log_prefix = options.delete(:log_prefix)
       @global_retry_config = default_retry_config.dup.merge!(options)
+      if (@reconnect_with_consistent_host = reconnect_with_consistent_host)
+        @initial_hostname = hostname
+        @initial_server_id = server_id
+      end
     end
 
+    # Complete explanation of algorithm: https://github.com/Shopify/lhm/pull/112
     def with_retries(retry_config = {})
-      cnf = @global_retry_config.dup.merge!(retry_config)
-      @log_prefix = cnf.delete(:log_prefix) || "SQL Retry"
-      Retriable.retriable(cnf) do
-        yield(@connection)
+      @log_prefix = retry_config.delete(:log_prefix)
+
+      retry_config = @global_retry_config.dup.merge!(retry_config)
+
+      Retriable.retriable(retry_config) do
+        # Using begin -> rescue -> end for Ruby 2.4 compatibility
+        begin
+          if @reconnect_with_consistent_host
+            raise Lhm::Error.new("Could not reconnected to initial MySQL host. Aborting to avoid data-loss") unless same_host_as_initial?
+          end
+
+          yield(@connection)
+        rescue StandardError => e
+          # Not all errors should trigger a reconnect. Some errors such be raised and abort the LHM (such as reconnecting to the wrong host).
+          # The error will be raised the connection is still active (i.e. no need to reconnect) or if the connection is
+          # dead (i.e. not active) and @reconnect_with_host is false (i.e. instructed not to reconnect)
+          raise e if @connection.active? || (!@connection.active? && !@reconnect_with_consistent_host)
+          reconnect_with_host_check! if @reconnect_with_consistent_host
+        end
       end
     end
 
     attr_reader :global_retry_config
+    attr_accessor :reconnect_with_consistent_host
 
     private
 
+    def hostname
+      mysql_single_value(MYSQL_VAR_NAMES[:hostname])
+    end
+
+    def server_id
+      mysql_single_value(MYSQL_VAR_NAMES[:server_id])
+    end
+
+    def cloudsql?
+      mysql_single_value(MYSQL_VAR_NAMES[:version_comment]).include?(CLOUDSQL_VERSION_COMMENT)
+    end
+
+    def mysql_single_value(name)
+      query = Lhm::ProxySQLHelper.tagged("SELECT #{name} LIMIT 1")
+
+      @connection.execute(query).to_a.first.tap do |record|
+        return record&.first
+      end
+    end
+
+    def same_host_as_initial?
+      return @initial_server_id == server_id if cloudsql?
+      @initial_hostname == hostname
+    end
+
     def log_with_prefix(message, level = :info)
       message.prepend("[#{@log_prefix}] ") if @log_prefix
-      Lhm.logger.send(level, message)
+      Lhm.logger.public_send(level, message)
+    end
+
+    def reconnect_with_host_check!
+      log_with_prefix("Lost connection to MySQL, will retry to connect to same host")
+      begin
+        Retriable.retriable(host_retry_config) do
+          # tries to reconnect. On failure will trigger a retry
+          @connection.reconnect!
+
+          if same_host_as_initial?
+            # This is not an actual error, but controlled way to get the parent `Retriable.retriable` to retry
+            # the statement that failed (since the Retriable gem only retries on errors).
+            raise ReconnectToHostSuccessful.new("LHM successfully reconnected to initial host: #{@initial_hostname} (server_id: #{@initial_server_id})")
+          else
+            # New Master --> abort LHM (reconnecting will not change anything)
+            raise Lhm::Error.new("Reconnected to wrong host. Started migration on: #{@initial_hostname} (server_id: #{@initial_server_id}), but reconnected to: #{hostname} (server_id: #{@initial_server_id}).")
+          end
+        end
+      rescue StandardError => e
+        # The parent Retriable.retriable is configured to retry if it encounters an error with the success message.
+        # Therefore, if the connection is re-established successfully AND the host is the same, LHM can retry the query
+        # that originally failed.
+        raise e if reconnect_successful?(e)
+        # If the connection was not successful, the parent retriable will raise any error that originated from the
+        # `@connection.reconnect!`
+        # Therefore, this error will cause the LHM to abort
+        raise Lhm::Error.new("LHM tried the reconnection procedure but failed. Latest error: #{e.message}")
+      end
+    end
+
+    def reconnect_successful?(e)
+      e.is_a?(ReconnectToHostSuccessful)
     end
 
     # For a full list of configuration options see https://github.com/kamui/retriable
@@ -49,6 +141,11 @@ module Lhm
             /Lost connection to MySQL server during query/,
             /Max connect timeout reached/,
             /Unknown MySQL server host/,
+            /connection is locked to hostgroup/,
+            /The MySQL server is running with the --read-only option so it cannot execute this statement/,
+          ],
+          ReconnectToHostSuccessful => [
+            /#{RECONNECT_SUCCESSFUL_MESSAGE}/
           ]
         },
         multiplier: 1, # each successive interval grows by this factor
@@ -57,8 +154,29 @@ module Lhm
         rand_factor: 0, # percentage to randomize the next retry interval time
         max_elapsed_time: Float::INFINITY, # max total time in seconds that code is allowed to keep being retried
         on_retry: Proc.new do |exception, try_number, total_elapsed_time, next_interval|
-          log = "#{exception.class}: '#{exception.message}' - #{try_number} tries in #{total_elapsed_time} seconds and #{next_interval} seconds until the next try."
-          log_with_prefix(log, :info)
+          if reconnect_successful?(exception)
+            log_with_prefix("#{exception.message} -- triggering retry", :info)
+          else
+            log_with_prefix("#{exception.class}: '#{exception.message}' - #{try_number} tries in #{total_elapsed_time} seconds and #{next_interval} seconds until the next try.", :error)
+          end
+        end
+      }.freeze
+    end
+
+    def host_retry_config
+      {
+        on: {
+          StandardError => [
+            /Lost connection to MySQL server at 'reading initial communication packet'/
+          ]
+        },
+        multiplier: 1, # each successive interval grows by this factor
+        base_interval: 0.25, # the initial interval in seconds between tries.
+        tries: 20, # Number of attempts to make at running your code block (includes initial attempt).
+        rand_factor: 0, # percentage to randomize the next retry interval time
+        max_elapsed_time: Float::INFINITY, # max total time in seconds that code is allowed to keep being retried
+        on_retry: Proc.new do |exception, try_number, total_elapsed_time, next_interval|
+          log_with_prefix("#{exception.class}: '#{exception.message}' - #{try_number} tries in #{total_elapsed_time} seconds and #{next_interval} seconds until the next try.", :error)
         end
       }.freeze
     end

data/lib/lhm/throttler/slave_lag.rb

@@ -124,8 +124,8 @@ module Lhm
           else
             raise ArgumentError, "Expected #{config_proc.inspect} to respond to `call`"
           end
-        else # otherwise default to ActiveRecord provided config
-          ActiveRecord::Base.connection_pool.spec.config.dup
+        else
+          db_config
         end
         config.deep_symbolize_keys!
         config[:host] = @host
@@ -140,6 +140,23 @@ module Lhm
           [nil]
         end
       end
+
+      private
+
+      def db_config
+        if ar_supports_db_config?
+          ActiveRecord::Base.connection_pool.db_config.configuration_hash.dup
+        else
+          ActiveRecord::Base.connection_pool.spec.config.dup
+        end
+      end
+
+      def ar_supports_db_config?
+        # https://api.rubyonrails.org/v6.0/classes/ActiveRecord/ConnectionAdapters/ConnectionPool.html <-- has spec
+        # vs
+        # https://api.rubyonrails.org/v6.1/classes/ActiveRecord/ConnectionAdapters/ConnectionPool.html <-- has db_config
+        ActiveRecord::VERSION::MAJOR > 6 || ActiveRecord::VERSION::MAJOR == 6 && ActiveRecord::VERSION::MINOR >= 1
+      end
     end
   end
 end

data/lib/lhm/version.rb

@@ -2,5 +2,5 @@
 # Schmidt
 
 module Lhm
-  VERSION = '3.5.0'
+  VERSION = '3.5.1'
 end

data/lib/lhm.rb

@@ -8,6 +8,7 @@ require 'lhm/throttler'
 require 'lhm/version'
 require 'lhm/cleanup/current'
 require 'lhm/sql_retry'
+require 'lhm/proxysql_helper'
 require 'lhm/connection'
 require 'lhm/test_support'
 require 'lhm/railtie' if defined?(Rails::Railtie)
@@ -82,16 +83,29 @@ module Lhm
     Lhm::Cleanup::Current.new(run, table_name, connection, options).execute
   end
 
-  def setup(connection)
-    @@connection = Lhm::Connection.new(connection: connection)
+  # Setups DB connection
+  #
+  # @param [ActiveRecord::Base] connection ActiveRecord Connection
+  # @param [Hash] connection_options Optional options (defaults to: empty hash)
+  # @option connection_options [Boolean] :reconnect_with_consistent_host
+  #   Active / Deactivate ProxySQL-aware reconnection procedure (default to: false)
+  def setup(connection, connection_options = {})
+    @@connection = Connection.new(connection: connection, options: connection_options)
   end
 
-  def connection
-    @@connection ||=
-      begin
-        raise 'Please call Lhm.setup' unless defined?(ActiveRecord)
-        ActiveRecord::Base.connection
-      end
+  # Setups DB connection
+  #
+  # @param [Hash] connection_options Optional options (defaults to: empty hash)
+  # @option connection_options [Boolean] :reconnect_with_consistent_host
+  #   Active / Deactivate ProxySQL-aware reconnection procedure (default to: false)
+  def connection(connection_options = nil)
+    if @@connection.nil?
+      raise 'Please call Lhm.setup' unless defined?(ActiveRecord)
+      @@connection = Connection.new(connection: ActiveRecord::Base.connection, options: connection_options || {})
+    else
+      @@connection.options = connection_options unless connection_options.nil?
+    end
+    @@connection
   end
 
   def self.logger=(new_logger)

data/scripts/mysql/writer/create_users.sql

@@ -1,3 +1,6 @@
 # Creates replication user in Writer
+CREATE USER IF NOT EXISTS 'writer'@'%' IDENTIFIED BY 'password';
+CREATE USER IF NOT EXISTS 'reader'@'%' IDENTIFIED BY 'password';
+
 CREATE USER IF NOT EXISTS 'replication'@'%' IDENTIFIED BY 'password';
 GRANT REPLICATION SLAVE ON *.* TO' replication'@'%' IDENTIFIED BY 'password';

data/spec/integration/atomic_switcher_spec.rb

@@ -16,9 +16,9 @@ describe Lhm::AtomicSwitcher do
   describe 'switching' do
     before(:each) do
       Thread.abort_on_exception = true
-      @origin      = table_create('origin')
+      @origin = table_create('origin')
       @destination = table_create('destination')
-      @migration   = Lhm::Migration.new(@origin, @destination)
+      @migration = Lhm::Migration.new(@origin, @destination)
       @logs = StringIO.new
       Lhm.logger = Logger.new(@logs)
       @connection.execute('SET GLOBAL innodb_lock_wait_timeout=3')
@@ -32,11 +32,17 @@ describe Lhm::AtomicSwitcher do
     it 'should retry and log on lock wait timeouts' do
       ar_connection = mock()
       ar_connection.stubs(:data_source_exists?).returns(true)
-      ar_connection.stubs(:execute).raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.').then.returns(true)
+      ar_connection.stubs(:active?).returns(true)
+      ar_connection.stubs(:execute).returns([["dummy"]], [["dummy"]], [["dummy"]])
+                   .then
+                   .raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
+                   .then
+                   .returns([["dummy"]]) # Matches initial host -> triggers retry
+
+      connection = Lhm::Connection.new(connection: ar_connection, options: {reconnect_with_consistent_host: true})
 
-      connection = Lhm::Connection.new(connection: ar_connection)
 
-      switcher = Lhm::AtomicSwitcher.new(@migration, connection, retriable: {base_interval: 0})
+      switcher = Lhm::AtomicSwitcher.new(@migration, connection, retriable: { tries: 3, base_interval: 0 })
 
       assert switcher.run
 
@@ -50,11 +56,22 @@ describe Lhm::AtomicSwitcher do
     it 'should give up on lock wait timeouts after a configured number of tries' do
       ar_connection = mock()
       ar_connection.stubs(:data_source_exists?).returns(true)
-      ar_connection.stubs(:execute).twice.raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
-
-      connection = Lhm::Connection.new(connection: ar_connection)
-
-      switcher = Lhm::AtomicSwitcher.new(@migration, connection, retriable: {tries: 2, base_interval: 0})
+      ar_connection.stubs(:active?).returns(true)
+      ar_connection.stubs(:execute).returns([["dummy"]], [["dummy"]], [["dummy"]])
+                .then
+                .raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
+                .then
+                .returns([["dummy"]]) # triggers retry 1
+                .then
+                .raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
+                .then
+                .returns([["dummy"]]) # triggers retry 2
+                .then
+                .raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.') # triggers retry 2
+
+      connection = Lhm::Connection.new(connection: ar_connection, options: {reconnect_with_consistent_host: true})
+
+      switcher = Lhm::AtomicSwitcher.new(@migration, connection, retriable: { tries: 2, base_interval: 0 })
 
       assert_raises(ActiveRecord::StatementInvalid) { switcher.run }
     end

data/spec/integration/chunk_insert_spec.rb

@@ -11,7 +11,8 @@ describe Lhm::ChunkInsert do
       @destination = table_create(:destination)
       @migration = Lhm::Migration.new(@origin, @destination)
       execute("insert into origin set id = 1001")
-      @instance = Lhm::ChunkInsert.new(@migration, connection, 1001, 1001)
+      @connection = Lhm::Connection.new(connection: connection)
+      @instance = Lhm::ChunkInsert.new(@migration, @connection, 1001, 1001)
     end
 
     it "returns the count" do

data/spec/integration/chunker_spec.rb

@@ -223,7 +223,7 @@ describe Lhm::Chunker do
 
       if master_slave_mode?
         def throttler.slave_connection(slave)
-          config = ActiveRecord::Base.connection_pool.spec.config.dup
+          config = ActiveRecord::Base.connection_pool.db_config.configuration_hash.dup
           config[:host] = slave
           config[:port] = 3307
           ActiveRecord::Base.send('mysql2_connection', config)

data/spec/integration/database.yml

@@ -13,3 +13,13 @@ proxysql:
   user: root
   password: password
   port: 33005
+master_toxic:
+  host: toxiproxy
+  user: root
+  password: password
+  port: 22220
+proxysql_toxic:
+  host: toxiproxy
+  user: root
+  password: password
+  port: 22222

data/spec/integration/entangler_spec.rb

@@ -6,6 +6,7 @@ require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
 require 'lhm/table'
 require 'lhm/migration'
 require 'lhm/entangler'
+require 'lhm/connection'
 
 describe Lhm::Entangler do
   include IntegrationHelper
@@ -17,7 +18,8 @@ describe Lhm::Entangler do
       @origin = table_create('origin')
       @destination = table_create('destination')
       @migration = Lhm::Migration.new(@origin, @destination)
-      @entangler = Lhm::Entangler.new(@migration, connection)
+      @connection = Lhm::Connection.new(connection: connection)
+      @entangler = Lhm::Entangler.new(@migration, @connection)
     end
 
     it 'should replay inserts from origin into destination' do

data/spec/integration/integration_helper.rb

@@ -35,6 +35,15 @@ module IntegrationHelper
     @connection
   end
 
+  def connect_proxysql!
+    connect!(
+      '127.0.0.1',
+      $db_config['proxysql']['port'],
+      $db_config['proxysql']['user'],
+      $db_config['proxysql']['password'],
+    )
+  end
+
   def connect_master!
     connect!(
       '127.0.0.1',
@@ -53,14 +62,23 @@ module IntegrationHelper
     )
   end
 
-  def connect!(hostname, port, user, password)
-    adapter = Lhm::Connection.new(connection: ar_conn(hostname, port, user, password))
-    Lhm.setup(adapter)
+  def connect_master_with_toxiproxy!(with_retry: false)
+    connect!(
+      '127.0.0.1',
+      $db_config['master_toxic']['port'],
+      $db_config['master_toxic']['user'],
+      $db_config['master_toxic']['password'],
+      with_retry)
+  end
+
+  def connect!(hostname, port, user, password, with_retry = false)
+    adapter = ar_conn(hostname, port, user, password)
+    Lhm.setup(adapter,{reconnect_with_consistent_host: with_retry})
     unless defined?(@@cleaned_up)
       Lhm.cleanup(true)
       @@cleaned_up  = true
     end
-    @connection = adapter
+    @connection = Lhm.connection
   end
 
   def ar_conn(host, port, user, password)
@@ -119,7 +137,7 @@ module IntegrationHelper
   # Helps testing behaviour when another client locks the db
   def start_locking_thread(lock_for, queue, locking_query)
     Thread.new do
-      conn = Mysql2::Client.new(host: '127.0.0.1', database: $db_name, user: 'root', port: 3306)
+      conn = new_mysql_connection
       conn.query('BEGIN')
       conn.query(locking_query)
       queue.push(true)