lhm-shopify 3.4.2 → 3.5.1

data/lib/lhm/atomic_switcher.rb

@@ -16,17 +16,12 @@ module Lhm
 
     attr_reader :connection
 
-    def initialize(migration, connection = nil, options = {})
+    def initialize(migration, connection = nil, options={})
       @migration = migration
       @connection = connection
       @origin = migration.origin
       @destination = migration.destination
-      @retry_helper = SqlRetry.new(
-        @connection,
-        {
-          log_prefix: "AtomicSwitcher"
-        }.merge!(options.fetch(:retriable, {}))
-      )
+      @retry_options = options[:retriable] || {}
     end
 
     def atomic_switch
@@ -36,7 +31,7 @@ module Lhm
 
     def validate
       unless @connection.data_source_exists?(@origin.name) &&
-             @connection.data_source_exists?(@destination.name)
+        @connection.data_source_exists?(@destination.name)
         error "`#{ @origin.name }` and `#{ @destination.name }` must exist"
       end
     end
@@ -44,9 +39,7 @@ module Lhm
     private
 
     def execute
-      @retry_helper.with_retries do |retriable_connection|
-        retriable_connection.execute atomic_switch
-      end
+      @connection.execute(atomic_switch, should_retry: true, retry_options: @retry_options)
     end
   end
 end

data/lib/lhm/chunk_insert.rb

@@ -1,4 +1,5 @@
 require 'lhm/sql_retry'
+require 'lhm/proxysql_helper'
 
 module Lhm
   class ChunkInsert
@@ -7,18 +8,11 @@ module Lhm
       @connection = connection
       @lowest = lowest
       @highest = highest
-      @retry_helper = SqlRetry.new(
-        @connection,
-        {
-          log_prefix: "Chunker Insert"
-        }.merge!(options.fetch(:retriable, {}))
-      )
+      @retry_options = options[:retriable] || {}
     end
 
     def insert_and_return_count_of_rows_created
-      @retry_helper.with_retries do |retriable_connection|
-        retriable_connection.update sql
-      end
+      @connection.update(sql, should_retry: true, retry_options: @retry_options)
     end
 
     def sql

data/lib/lhm/chunker.rb

@@ -28,11 +28,12 @@ module Lhm
       @start = @chunk_finder.start
       @limit = @chunk_finder.limit
       @printer = options[:printer] || Printer::Percentage.new
+      @retry_options = options[:retriable] || {}
       @retry_helper = SqlRetry.new(
         @connection,
-        {
+        options: {
           log_prefix: "Chunker"
-        }.merge!(options.fetch(:retriable, {}))
+        }.merge!(@retry_options)
       )
     end
 
@@ -46,7 +47,7 @@ module Lhm
         top = upper_id(@next_to_insert, stride)
         verify_can_run
 
-        affected_rows = ChunkInsert.new(@migration, @connection, bottom, top, @options).insert_and_return_count_of_rows_created
+        affected_rows = ChunkInsert.new(@migration, @connection, bottom, top, @retry_options).insert_and_return_count_of_rows_created
         expected_rows = top - bottom + 1
 
         # Only log the chunker progress every 5 minutes instead of every iteration
@@ -78,7 +79,7 @@ module Lhm
     private
 
     def raise_on_non_pk_duplicate_warning
-      @connection.query("show warnings").each do |level, code, message|
+      @connection.execute("show warnings", should_retry: true, retry_options: @retry_options).each do |level, code, message|
         unless message.match?(/Duplicate entry .+ for key 'PRIMARY'/)
           m = "Unexpected warning found for inserted row: #{message}"
           Lhm.logger.warn(m)
@@ -93,16 +94,14 @@ module Lhm
 
     def verify_can_run
       return unless @verifier
-      @retry_helper.with_retries do |retriable_connection|
+      @retry_helper.with_retries(@retry_options) do |retriable_connection|
         raise "Verification failed, aborting early" if !@verifier.call(retriable_connection)
       end
     end
 
     def upper_id(next_id, stride)
       sql = "select id from `#{ @migration.origin_name }` where id >= #{ next_id } order by id limit 1 offset #{ stride - 1}"
-      top = @retry_helper.with_retries do |retriable_connection|
-        retriable_connection.select_value(sql)
-      end
+      top = @connection.select_value(sql, should_retry: true, retry_options: @retry_options)
 
       [top ? top.to_i : @limit, @limit].min
     end

data/lib/lhm/cleanup/current.rb

@@ -4,17 +4,12 @@ require 'lhm/sql_retry'
 module Lhm
   module Cleanup
     class Current
-      def initialize(run, origin_table_name, connection, options = {})
+      def initialize(run, origin_table_name, connection, options={})
         @run = run
         @table_name = TableName.new(origin_table_name)
         @connection = connection
         @ddls = []
-        @retry_helper = SqlRetry.new(
-          @connection,
-          {
-            log_prefix: "Cleanup::Current"
-          }.merge!(options.fetch(:retriable, {}))
-        )
+        @retry_config = options[:retriable] || {}
       end
 
       attr_reader :run, :connection, :ddls
@@ -59,9 +54,7 @@ module Lhm
 
       def execute_ddls
         ddls.each do |ddl|
-          @retry_helper.with_retries do |retriable_connection|
-            retriable_connection.execute(ddl)
-          end
+            @connection.execute(ddl, should_retry: true, retry_options: @retry_config)
         end
         Lhm.logger.info("Dropped triggers on #{@lhm_triggers_for_origin.join(', ')}")
         Lhm.logger.info("Dropped tables #{@lhm_triggers_for_origin.join(', ')}")

data/lib/lhm/connection.rb

@@ -0,0 +1,100 @@
+require 'delegate'
+require 'lhm/sql_retry'
+
+module Lhm
+  class Connection < SimpleDelegator
+
+    # Lhm::Connection inherits from SingleDelegator. It will forward any unknown method calls to the ActiveRecord
+    # connection.
+    alias connection __getobj__
+    alias connection= __setobj__
+
+    def initialize(connection:, default_log_prefix: nil, options: {}, retry_config: {})
+      @default_log_prefix = default_log_prefix
+      @retry_options = retry_config || default_retry_config
+      @sql_retry = Lhm::SqlRetry.new(
+        connection,
+        options: retry_config,
+        reconnect_with_consistent_host: options[:reconnect_with_consistent_host] || false
+      )
+
+      # Creates delegation for the ActiveRecord Connection
+      super(connection)
+    end
+
+    def options=(options)
+      # If any other flags are added. Add the "processing" here
+      @sql_retry.reconnect_with_consistent_host = options[:reconnect_with_consistent_host] || false
+    end
+
+    def execute(query, should_retry: false, retry_options: {})
+      if should_retry
+        exec_with_retries(:execute, query, retry_options)
+      else
+        exec(:execute, query)
+      end
+    end
+
+    def update(query, should_retry: false, retry_options: {})
+      if should_retry
+        exec_with_retries(:update, query, retry_options)
+      else
+        exec(:update, query)
+      end
+    end
+
+    def select_value(query, should_retry: false, retry_options: {})
+      if should_retry
+        exec_with_retries(:select_value, query, retry_options)
+      else
+        exec(:select_value, query)
+      end
+    end
+
+    def select_values(query, should_retry: false, retry_options: {})
+      if should_retry
+        exec_with_retries(:select_values, query, retry_options)
+      else
+        exec(:select_values, query)
+      end
+    end
+
+    def select_one(query, should_retry: false, retry_options: {})
+      if should_retry
+        exec_with_retries(:select_one, query, retry_options)
+      else
+        exec(:select_one, query)
+      end
+    end
+
+    private
+
+    def exec(method, sql)
+      connection.public_send(method, Lhm::ProxySQLHelper.tagged(sql))
+    end
+
+    def exec_with_retries(method, sql, retry_options = {})
+      retry_options[:log_prefix] ||= file
+      @sql_retry.with_retries(retry_options) do |conn|
+        conn.public_send(method, Lhm::ProxySQLHelper.tagged(sql))
+      end
+    end
+
+    # Returns camelized file name of caller (e.g. chunk_insert.rb -> ChunkInsert)
+    def file
+      # Find calling file and extract name
+      /[\/]*(\w+).rb:\d+:in/.match(relevant_caller)
+      name = $1&.camelize || "Connection"
+      "#{name}"
+    end
+
+    def relevant_caller
+      lhm_stack = caller.select { |x| x.include?("/lhm") }
+      first_candidate_index = lhm_stack.find_index { |line| !line.include?(__FILE__) }
+
+      # Find the file that called the `#execute` (fallbacks to current file)
+      return lhm_stack.first unless first_candidate_index
+      lhm_stack.at(first_candidate_index)
+    end
+  end
+end

data/lib/lhm/entangler.rb

@@ -4,6 +4,7 @@
 require 'lhm/command'
 require 'lhm/sql_helper'
 require 'lhm/sql_retry'
+require 'lhm/connection'
 
 module Lhm
   class Entangler
@@ -19,12 +20,7 @@ module Lhm
       @origin = migration.origin
       @destination = migration.destination
       @connection = connection
-      @retry_helper = SqlRetry.new(
-        @connection,
-        {
-          log_prefix: "Entangler"
-        }.merge!(options.fetch(:retriable, {}))
-      )
+      @retry_options = options[:retriable] || {}
     end
 
     def entangle
@@ -90,18 +86,14 @@ module Lhm
 
     def before
       entangle.each do |stmt|
-        @retry_helper.with_retries do |retriable_connection|
-          retriable_connection.execute(stmt)
-        end
+        @connection.execute(stmt, should_retry: true, retry_options: @retry_options)
       end
       Lhm.logger.info("Created triggers on #{@origin.name}")
     end
 
     def after
       untangle.each do |stmt|
-        @retry_helper.with_retries do |retriable_connection|
-          retriable_connection.execute(stmt)
-        end
+        @connection.execute(stmt, should_retry: true, retry_options: @retry_options)
       end
       Lhm.logger.info("Dropped triggers on #{@origin.name}")
     end

data/lib/lhm/invoker.rb

@@ -16,8 +16,8 @@ module Lhm
   class Invoker
     include SqlHelper
     LOCK_WAIT_TIMEOUT_DELTA = 10
-      INNODB_LOCK_WAIT_TIMEOUT_MAX=1073741824.freeze # https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_lock_wait_timeout
-      LOCK_WAIT_TIMEOUT_MAX=31536000.freeze # https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html
+    INNODB_LOCK_WAIT_TIMEOUT_MAX=1073741824.freeze # https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_lock_wait_timeout
+    LOCK_WAIT_TIMEOUT_MAX=31536000.freeze # https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html
 
     attr_reader :migrator, :connection
 
@@ -56,7 +56,7 @@ module Lhm
         Chunker.new(migration, @connection, options).run
         raise "Required triggers do not exist" unless triggers_still_exist?(@connection, entangler)
         if options[:atomic_switch]
-          AtomicSwitcher.new(migration, @connection, options).run
+          AtomicSwitcher.new(migration, @connection).run
         else
           LockedSwitcher.new(migration, @connection).run
         end

data/lib/lhm/proxysql_helper.rb

@@ -0,0 +1,10 @@
+module Lhm
+  module ProxySQLHelper
+    extend self
+    ANNOTATION = "/*maintenance:lhm*/"
+
+    def tagged(sql)
+      "#{ANNOTATION}#{sql}"
+    end
+  end
+end

data/lib/lhm/sql_retry.rb

@@ -1,5 +1,6 @@
 require 'retriable'
 require 'lhm/sql_helper'
+require 'lhm/proxysql_helper'
 
 module Lhm
   # SqlRetry standardizes the interface for retry behavior in components like
@@ -15,22 +16,119 @@ module Lhm
   # https://github.com/kamui/retriable. Additionally, a "log_prefix" option,
   # which is unique to SqlRetry can be used to prefix log output.
   class SqlRetry
-    def initialize(connection, options = {})
+    RECONNECT_SUCCESSFUL_MESSAGE = "LHM successfully reconnected to initial host:"
+    CLOUDSQL_VERSION_COMMENT = "(Google)"
+
+    MYSQL_VAR_NAMES = {
+      hostname: "@@global.hostname",
+      server_id: "@@global.server_id",
+      version_comment: "@@version_comment",
+    }
+
+    # This internal error is used to trigger retries from the parent Retriable.retriable in #with_retries
+    class ReconnectToHostSuccessful < Lhm::Error; end
+
+    def initialize(connection, options: {}, reconnect_with_consistent_host: true)
       @connection = connection
       @log_prefix = options.delete(:log_prefix)
-      @retry_config = default_retry_config.dup.merge!(options)
+      @global_retry_config = default_retry_config.dup.merge!(options)
+      if (@reconnect_with_consistent_host = reconnect_with_consistent_host)
+        @initial_hostname = hostname
+        @initial_server_id = server_id
+      end
     end
 
-    def with_retries
+    # Complete explanation of algorithm: https://github.com/Shopify/lhm/pull/112
+    def with_retries(retry_config = {})
+      @log_prefix = retry_config.delete(:log_prefix)
+
+      retry_config = @global_retry_config.dup.merge!(retry_config)
+
       Retriable.retriable(retry_config) do
-        yield(@connection)
+        # Using begin -> rescue -> end for Ruby 2.4 compatibility
+        begin
+          if @reconnect_with_consistent_host
+            raise Lhm::Error.new("Could not reconnected to initial MySQL host. Aborting to avoid data-loss") unless same_host_as_initial?
+          end
+
+          yield(@connection)
+        rescue StandardError => e
+          # Not all errors should trigger a reconnect. Some errors such be raised and abort the LHM (such as reconnecting to the wrong host).
+          # The error will be raised the connection is still active (i.e. no need to reconnect) or if the connection is
+          # dead (i.e. not active) and @reconnect_with_host is false (i.e. instructed not to reconnect)
+          raise e if @connection.active? || (!@connection.active? && !@reconnect_with_consistent_host)
+          reconnect_with_host_check! if @reconnect_with_consistent_host
+        end
       end
     end
 
-    attr_reader :retry_config
+    attr_reader :global_retry_config
+    attr_accessor :reconnect_with_consistent_host
 
     private
 
+    def hostname
+      mysql_single_value(MYSQL_VAR_NAMES[:hostname])
+    end
+
+    def server_id
+      mysql_single_value(MYSQL_VAR_NAMES[:server_id])
+    end
+
+    def cloudsql?
+      mysql_single_value(MYSQL_VAR_NAMES[:version_comment]).include?(CLOUDSQL_VERSION_COMMENT)
+    end
+
+    def mysql_single_value(name)
+      query = Lhm::ProxySQLHelper.tagged("SELECT #{name} LIMIT 1")
+
+      @connection.execute(query).to_a.first.tap do |record|
+        return record&.first
+      end
+    end
+
+    def same_host_as_initial?
+      return @initial_server_id == server_id if cloudsql?
+      @initial_hostname == hostname
+    end
+
+    def log_with_prefix(message, level = :info)
+      message.prepend("[#{@log_prefix}] ") if @log_prefix
+      Lhm.logger.public_send(level, message)
+    end
+
+    def reconnect_with_host_check!
+      log_with_prefix("Lost connection to MySQL, will retry to connect to same host")
+      begin
+        Retriable.retriable(host_retry_config) do
+          # tries to reconnect. On failure will trigger a retry
+          @connection.reconnect!
+
+          if same_host_as_initial?
+            # This is not an actual error, but controlled way to get the parent `Retriable.retriable` to retry
+            # the statement that failed (since the Retriable gem only retries on errors).
+            raise ReconnectToHostSuccessful.new("LHM successfully reconnected to initial host: #{@initial_hostname} (server_id: #{@initial_server_id})")
+          else
+            # New Master --> abort LHM (reconnecting will not change anything)
+            raise Lhm::Error.new("Reconnected to wrong host. Started migration on: #{@initial_hostname} (server_id: #{@initial_server_id}), but reconnected to: #{hostname} (server_id: #{@initial_server_id}).")
+          end
+        end
+      rescue StandardError => e
+        # The parent Retriable.retriable is configured to retry if it encounters an error with the success message.
+        # Therefore, if the connection is re-established successfully AND the host is the same, LHM can retry the query
+        # that originally failed.
+        raise e if reconnect_successful?(e)
+        # If the connection was not successful, the parent retriable will raise any error that originated from the
+        # `@connection.reconnect!`
+        # Therefore, this error will cause the LHM to abort
+        raise Lhm::Error.new("LHM tried the reconnection procedure but failed. Latest error: #{e.message}")
+      end
+    end
+
+    def reconnect_successful?(e)
+      e.is_a?(ReconnectToHostSuccessful)
+    end
+
     # For a full list of configuration options see https://github.com/kamui/retriable
     def default_retry_config
       {
@@ -43,6 +141,11 @@ module Lhm
             /Lost connection to MySQL server during query/,
             /Max connect timeout reached/,
             /Unknown MySQL server host/,
+            /connection is locked to hostgroup/,
+            /The MySQL server is running with the --read-only option so it cannot execute this statement/,
+          ],
+          ReconnectToHostSuccessful => [
+            /#{RECONNECT_SUCCESSFUL_MESSAGE}/
           ]
         },
         multiplier: 1, # each successive interval grows by this factor
@@ -51,9 +154,29 @@ module Lhm
         rand_factor: 0, # percentage to randomize the next retry interval time
         max_elapsed_time: Float::INFINITY, # max total time in seconds that code is allowed to keep being retried
         on_retry: Proc.new do |exception, try_number, total_elapsed_time, next_interval|
-          log = "#{exception.class}: '#{exception.message}' - #{try_number} tries in #{total_elapsed_time} seconds and #{next_interval} seconds until the next try."
-          log.prepend("[#{@log_prefix}] ") if @log_prefix
-          Lhm.logger.info(log)
+          if reconnect_successful?(exception)
+            log_with_prefix("#{exception.message} -- triggering retry", :info)
+          else
+            log_with_prefix("#{exception.class}: '#{exception.message}' - #{try_number} tries in #{total_elapsed_time} seconds and #{next_interval} seconds until the next try.", :error)
+          end
+        end
+      }.freeze
+    end
+
+    def host_retry_config
+      {
+        on: {
+          StandardError => [
+            /Lost connection to MySQL server at 'reading initial communication packet'/
+          ]
+        },
+        multiplier: 1, # each successive interval grows by this factor
+        base_interval: 0.25, # the initial interval in seconds between tries.
+        tries: 20, # Number of attempts to make at running your code block (includes initial attempt).
+        rand_factor: 0, # percentage to randomize the next retry interval time
+        max_elapsed_time: Float::INFINITY, # max total time in seconds that code is allowed to keep being retried
+        on_retry: Proc.new do |exception, try_number, total_elapsed_time, next_interval|
+          log_with_prefix("#{exception.class}: '#{exception.message}' - #{try_number} tries in #{total_elapsed_time} seconds and #{next_interval} seconds until the next try.", :error)
         end
       }.freeze
     end

data/lib/lhm/throttler/slave_lag.rb

@@ -124,8 +124,8 @@ module Lhm
           else
             raise ArgumentError, "Expected #{config_proc.inspect} to respond to `call`"
           end
-        else # otherwise default to ActiveRecord provided config
-          ActiveRecord::Base.connection_pool.spec.config.dup
+        else
+          db_config
         end
         config.deep_symbolize_keys!
         config[:host] = @host
@@ -140,6 +140,23 @@ module Lhm
           [nil]
         end
       end
+
+      private
+
+      def db_config
+        if ar_supports_db_config?
+          ActiveRecord::Base.connection_pool.db_config.configuration_hash.dup
+        else
+          ActiveRecord::Base.connection_pool.spec.config.dup
+        end
+      end
+
+      def ar_supports_db_config?
+        # https://api.rubyonrails.org/v6.0/classes/ActiveRecord/ConnectionAdapters/ConnectionPool.html <-- has spec
+        # vs
+        # https://api.rubyonrails.org/v6.1/classes/ActiveRecord/ConnectionAdapters/ConnectionPool.html <-- has db_config
+        ActiveRecord::VERSION::MAJOR > 6 || ActiveRecord::VERSION::MAJOR == 6 && ActiveRecord::VERSION::MINOR >= 1
+      end
     end
   end
 end

data/lib/lhm/version.rb

@@ -2,5 +2,5 @@
 # Schmidt
 
 module Lhm
-  VERSION = '3.4.2'
+  VERSION = '3.5.1'
 end

data/lib/lhm.rb

@@ -8,6 +8,8 @@ require 'lhm/throttler'
 require 'lhm/version'
 require 'lhm/cleanup/current'
 require 'lhm/sql_retry'
+require 'lhm/proxysql_helper'
+require 'lhm/connection'
 require 'lhm/test_support'
 require 'lhm/railtie' if defined?(Rails::Railtie)
 require 'logger'
@@ -81,16 +83,29 @@ module Lhm
     Lhm::Cleanup::Current.new(run, table_name, connection, options).execute
   end
 
-  def setup(connection)
-    @@connection = connection
+  # Setups DB connection
+  #
+  # @param [ActiveRecord::Base] connection ActiveRecord Connection
+  # @param [Hash] connection_options Optional options (defaults to: empty hash)
+  # @option connection_options [Boolean] :reconnect_with_consistent_host
+  #   Active / Deactivate ProxySQL-aware reconnection procedure (default to: false)
+  def setup(connection, connection_options = {})
+    @@connection = Connection.new(connection: connection, options: connection_options)
   end
 
-  def connection
-    @@connection ||=
-      begin
-        raise 'Please call Lhm.setup' unless defined?(ActiveRecord)
-        ActiveRecord::Base.connection
-      end
+  # Setups DB connection
+  #
+  # @param [Hash] connection_options Optional options (defaults to: empty hash)
+  # @option connection_options [Boolean] :reconnect_with_consistent_host
+  #   Active / Deactivate ProxySQL-aware reconnection procedure (default to: false)
+  def connection(connection_options = nil)
+    if @@connection.nil?
+      raise 'Please call Lhm.setup' unless defined?(ActiveRecord)
+      @@connection = Connection.new(connection: ActiveRecord::Base.connection, options: connection_options || {})
+    else
+      @@connection.options = connection_options unless connection_options.nil?
+    end
+    @@connection
   end
 
   def self.logger=(new_logger)

data/scripts/helpers/wait-for-dbs.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+# Wait for writer
+echo "Waiting for MySQL-1: "
+while ! (mysqladmin ping --host="127.0.0.1" --port=33006 --user=root --password=password --silent 2> /dev/null); do
+  echo -ne "."
+  sleep 1
+done
+# Wait for reader
+echo "Waiting for MySQL-2: "
+while ! (mysqladmin ping --host="127.0.0.1" --port=33007 --user=root --password=password --silent 2> /dev/null); do
+  echo -ne "."
+  sleep 1
+done
+# Wait for proxysql
+echo "Waiting for ProxySQL:"
+while ! (mysqladmin ping --host="127.0.0.1" --port=33005 --user=root --password=password --silent 2> /dev/null); do
+  echo -ne "."
+  sleep 1
+done
+
+echo "All DBs are ready"

data/scripts/mysql/reader/create_replication.sql

@@ -0,0 +1,10 @@
+STOP SLAVE;
+CHANGE MASTER TO
+MASTER_HOST='mysql-1',
+MASTER_AUTO_POSITION=1,
+MASTER_USER='replication',
+MASTER_PASSWORD='password',
+MASTER_CONNECT_RETRY=1,
+MASTER_RETRY_COUNT=300; -- 5 minutes
+
+start slave;

data/scripts/mysql/writer/create_test_db.sql

@@ -0,0 +1 @@
+CREATE DATABASE test;

data/scripts/mysql/writer/create_users.sql

@@ -0,0 +1,6 @@
+# Creates replication user in Writer
+CREATE USER IF NOT EXISTS 'writer'@'%' IDENTIFIED BY 'password';
+CREATE USER IF NOT EXISTS 'reader'@'%' IDENTIFIED BY 'password';
+
+CREATE USER IF NOT EXISTS 'replication'@'%' IDENTIFIED BY 'password';
+GRANT REPLICATION SLAVE ON *.* TO' replication'@'%' IDENTIFIED BY 'password';