lhm-teak 3.6.0

data/lib/lhm/railtie.rb

@@ -0,0 +1,9 @@
+module Lhm
+  class Railtie < Rails::Railtie
+    initializer "lhm.test_setup" do
+      if Rails.env.test? || Rails.env.development?
+        Lhm.execute_inline!
+      end
+    end
+  end
+end

data/lib/lhm/sql_helper.rb

@@ -0,0 +1,77 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+module Lhm
+  module SqlHelper
+    extend self
+
+    def annotation
+      '/* large hadron migration */'
+    end
+
+    def idx_name(table_name, cols)
+      column_names = column_definition(cols).map(&:first)
+      "index_#{ table_name }_on_#{ column_names.join('_and_') }"
+    end
+
+    def idx_spec(cols)
+      column_definition(cols).map do |name, length|
+        "`#{ name }`#{ length }"
+      end.join(', ')
+    end
+
+    def version_string
+      row = connection.select_one("show variables like 'version'")
+      value = struct_key(row, 'Value')
+      row[value]
+    end
+
+    def tagged(statement)
+      "#{ statement } #{ SqlHelper.annotation }"
+    end
+
+    private
+
+    def column_definition(cols)
+      Array(cols).map do |column|
+        column.to_s.match(/`?([^\(]+)`?(\([^\)]+\))?/).captures
+      end
+    end
+
+    # Older versions of MySQL contain an atomic rename bug affecting bin
+    # log order. Affected versions extracted from bug report:
+    #
+    #   http://bugs.mysql.com/bug.php?id=39675
+    #
+    # More Info: http://dev.mysql.com/doc/refman/5.5/en/metadata-locking.html
+    def supports_atomic_switch?
+      major, minor, tiny = version_string.split('.').map(&:to_i)
+
+      case major
+      when 4 then return false if minor and minor < 2
+      when 5
+        case minor
+        when 0 then return false if tiny and tiny < 52
+        when 1 then return false
+        when 4 then return false if tiny and tiny < 4
+        when 5 then return false if tiny and tiny < 3
+        end
+      when 6
+        case minor
+        when 0 then return false if tiny and tiny < 11
+        end
+      end
+      true
+    end
+
+    def struct_key(struct, key)
+      keys = if struct.is_a? Hash
+               struct.keys
+             else
+               struct.members
+             end
+
+      keys.find { |k| k.to_s.downcase == key.to_s.downcase }
+    end
+  end
+end

data/lib/lhm/sql_retry.rb

@@ -0,0 +1,180 @@
+require 'retriable'
+require 'lhm/sql_helper'
+require 'lhm/proxysql_helper'
+
+module Lhm
+  # SqlRetry standardizes the interface for retry behavior in components like
+  # Entangler, AtomicSwitcher, ChunkerInsert.
+  #
+  # By default if an error includes the message "Lock wait timeout exceeded", or
+  # "Deadlock found when trying to get lock", SqlRetry will retry again
+  # once the MySQL client returns control to the caller, plus one second.
+  # It will retry a total of 10 times and output to the logger a description
+  # of the retry with error information, retry count, and elapsed time.
+  #
+  # This behavior can be modified by passing `options` that are documented in
+  # https://github.com/kamui/retriable. Additionally, a "log_prefix" option,
+  # which is unique to SqlRetry can be used to prefix log output.
+  class SqlRetry
+    RECONNECT_SUCCESSFUL_MESSAGE = "LHM successfully reconnected to initial host:"
+    CLOUDSQL_VERSION_COMMENT = "(Google)"
+    # Will retry for 120 seconds (approximately, since connecting takes time).
+    RECONNECT_RETRY_MAX_ITERATION = 120
+    RECONNECT_RETRY_INTERVAL = 1
+    # Will abort the LHM if it had to reconnect more than 25 times in a single run (indicator that there might be
+    # something wrong with the network and would be better to run the LHM at a later time).
+    RECONNECTION_MAXIMUM = 25
+
+    MYSQL_VAR_NAMES = {
+      hostname: "@@global.hostname",
+      server_id: "@@global.server_id",
+      version_comment: "@@version_comment",
+    }
+
+    def initialize(connection, retry_options: {}, reconnect_with_consistent_host: false)
+      @connection = connection
+      self.retry_config = retry_options
+      self.reconnect_with_consistent_host = reconnect_with_consistent_host
+    end
+
+    # Complete explanation of algorithm: https://github.com/Shopify/lhm/pull/112
+    def with_retries(log_prefix: nil)
+      @log_prefix = log_prefix || "" # No prefix. Just logs
+
+      # Amount of time LHM had to reconnect. Aborting if more than RECONNECTION_MAXIMUM
+      reconnection_counter = 0
+
+      Retriable.retriable(@retry_config) do
+        # Using begin -> rescue -> end for Ruby 2.4 compatibility
+        begin
+          if @reconnect_with_consistent_host
+            raise Lhm::Error.new("MySQL host has changed since the start of the LHM. Aborting to avoid data-loss") unless same_host_as_initial?
+          end
+
+          yield(@connection)
+        rescue StandardError => e
+          # Not all errors should trigger a reconnect. Some errors such be raised and abort the LHM (such as reconnecting to the wrong host).
+          # The error will be raised the connection is still active (i.e. no need to reconnect) or if the connection is
+          # dead (i.e. not active) and @reconnect_with_host is false (i.e. instructed not to reconnect)
+          raise e if @connection.active? || (!@connection.active? && !@reconnect_with_consistent_host)
+
+          # Lhm could be stuck in a weird state where it loses connection, reconnects and re looses-connection instantly
+          # after, creating an infinite loop (because of the usage of `retry`). Hence, abort after 25 reconnections
+          raise Lhm::Error.new("LHM reached host reconnection max of #{RECONNECTION_MAXIMUM} times. " \
+            "Please try again later.") if reconnection_counter > RECONNECTION_MAXIMUM
+
+          reconnection_counter += 1
+          if reconnect_with_host_check!
+            retry
+          else
+            raise Lhm::Error.new("LHM tried the reconnection procedure but failed. Aborting")
+          end
+        end
+      end
+    end
+
+    # Both attributes will have defined setters
+    attr_reader :retry_config, :reconnect_with_consistent_host
+    attr_accessor :connection
+
+    def retry_config=(retry_options)
+      @retry_config = default_retry_config.dup.merge!(retry_options)
+    end
+
+    def reconnect_with_consistent_host=(reconnect)
+      if (@reconnect_with_consistent_host = reconnect)
+        @initial_hostname = hostname
+        @initial_server_id = server_id
+      end
+    end
+
+    private
+
+    def hostname
+      mysql_single_value(MYSQL_VAR_NAMES[:hostname])
+    end
+
+    def server_id
+      mysql_single_value(MYSQL_VAR_NAMES[:server_id])
+    end
+
+    def cloudsql?
+      mysql_single_value(MYSQL_VAR_NAMES[:version_comment]).include?(CLOUDSQL_VERSION_COMMENT)
+    end
+
+    def mysql_single_value(name)
+      query = Lhm::ProxySQLHelper.tagged("SELECT #{name} LIMIT 1")
+
+      @connection.execute(query).to_a.first.tap do |record|
+        return record&.first
+      end
+    end
+
+    def same_host_as_initial?
+      return @initial_server_id == server_id if cloudsql?
+      @initial_hostname == hostname
+    end
+
+    def log_with_prefix(message, level = :info)
+      message.prepend("[#{@log_prefix}] ") if @log_prefix
+      Lhm.logger.public_send(level, message)
+    end
+
+    def reconnect_with_host_check!
+      log_with_prefix("Lost connection to MySQL, will retry to connect to same host")
+
+      RECONNECT_RETRY_MAX_ITERATION.times do
+        begin
+          sleep(RECONNECT_RETRY_INTERVAL)
+
+          # tries to reconnect. On failure will trigger a retry
+          @connection.reconnect!
+
+          if same_host_as_initial?
+            # This is not an actual error, but controlled way to get the parent `Retriable.retriable` to retry
+            # the statement that failed (since the Retriable gem only retries on errors).
+            log_with_prefix("LHM successfully reconnected to initial host: #{@initial_hostname} (server_id: #{@initial_server_id})")
+            return true
+          else
+            # New Master --> abort LHM (reconnecting will not change anything)
+            log_with_prefix("Reconnected to wrong host. Started migration on: #{@initial_hostname} (server_id: #{@initial_server_id}), but reconnected to: #{hostname} (server_id: #{server_id}).", :error)
+            return false
+          end
+        rescue StandardError => e
+          # Retry if ActiveRecord cannot reach host
+          next if /Lost connection to MySQL server at 'reading initial communication packet'/ === e.message
+          log_with_prefix("Encountered error: [#{e.class}] #{e.message}. Will stop reconnection procedure.", :info)
+          return false
+        end
+      end
+      false
+    end
+
+    # For a full list of configuration options see https://github.com/kamui/retriable
+    def default_retry_config
+      {
+        on: {
+          StandardError => [
+            /Lock wait timeout exceeded/,
+            /Timeout waiting for a response from the last query/,
+            /Deadlock found when trying to get lock/,
+            /Query execution was interrupted/,
+            /Lost connection to MySQL server during query/,
+            /Max connect timeout reached/,
+            /Unknown MySQL server host/,
+            /connection is locked to hostgroup/,
+            /The MySQL server is running with the --read-only option so it cannot execute this statement/,
+          ]
+        },
+        multiplier: 1, # each successive interval grows by this factor
+        base_interval: 1, # the initial interval in seconds between tries.
+        tries: 20, # Number of attempts to make at running your code block (includes initial attempt).
+        rand_factor: 0, # percentage to randomize the next retry interval time
+        max_elapsed_time: Float::INFINITY, # max total time in seconds that code is allowed to keep being retried
+        on_retry: Proc.new do |exception, try_number, total_elapsed_time, next_interval|
+          log_with_prefix("#{exception.class}: '#{exception.message}' - #{try_number} tries in #{total_elapsed_time} seconds and #{next_interval} seconds until the next try.", :error)
+        end
+      }.freeze
+    end
+  end
+end

data/lib/lhm/table.rb

@@ -0,0 +1,121 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require 'lhm/sql_helper'
+
+module Lhm
+  class Table
+    attr_reader :name, :columns, :indices, :pk, :ddl
+
+    def initialize(name, pk = 'id', ddl = nil)
+      @name = name
+      @table_name = TableName.new(name)
+      @columns = {}
+      @indices = {}
+      @pk = pk
+      @ddl = ddl
+    end
+
+    def satisfies_id_column_requirement?
+      !!((id = columns['id']) &&
+        id[:type] =~ /(bigint|int)(\(\d+\))?/)
+    end
+
+    def destination_name
+      @destination_name ||= @table_name.new
+    end
+
+    def self.parse(table_name, connection)
+      Parser.new(table_name, connection).parse
+    end
+
+    class Parser
+      include SqlHelper
+
+      def initialize(table_name, connection)
+        @table_name = table_name.to_s
+        @schema_name = connection.current_database
+        @connection = connection
+      end
+
+      def ddl
+        sql = "show create table `#{ @table_name }`"
+        specification = nil
+        @connection.execute(sql).each { |row| specification = row.last }
+        specification
+      end
+
+      def parse
+        schema = read_information_schema
+
+        Table.new(@table_name, extract_primary_key(schema), ddl).tap do |table|
+          schema.each do |defn|
+            column_name    = struct_key(defn, 'COLUMN_NAME')
+            column_type    = struct_key(defn, 'COLUMN_TYPE')
+            is_nullable    = struct_key(defn, 'IS_NULLABLE')
+            column_default = struct_key(defn, 'COLUMN_DEFAULT')
+            comment = struct_key(defn, 'COLUMN_COMMENT')
+            collate = struct_key(defn, 'COLLATION_NAME')
+
+            table.columns[defn[column_name]] = {
+              :type => defn[column_type],
+              :is_nullable => defn[is_nullable],
+              :column_default => defn[column_default],
+              :comment => defn[comment],
+              :collate => defn[collate],
+            }
+          end
+
+          extract_indices(read_indices).each do |idx, columns|
+            table.indices[idx] = columns
+          end
+        end
+      end
+
+      private
+
+      def read_information_schema
+        @connection.select_all %Q{
+          select *
+            from information_schema.columns
+           where table_name = '#{ @table_name }'
+             and table_schema = '#{ @schema_name }'
+        }
+      end
+
+      def read_indices
+        @connection.select_all %Q{
+          show indexes from `#{ @schema_name }`.`#{ @table_name }`
+         where key_name != 'PRIMARY'
+        }
+      end
+
+      def extract_indices(indices)
+        indices.
+          map do |row|
+            key_name = struct_key(row, 'Key_name')
+            column_name = struct_key(row, 'COLUMN_NAME')
+            [row[key_name], row[column_name]]
+          end.
+          inject(Hash.new { |h, k| h[k] = [] }) do |memo, (idx, column)|
+            memo[idx] << column
+            memo
+          end
+      end
+
+      def extract_primary_key(schema)
+        cols = schema.select do |defn|
+          column_key = struct_key(defn, 'COLUMN_KEY')
+          defn[column_key] == 'PRI'
+        end
+
+        keys = cols.map do |defn|
+          column_name = struct_key(defn, 'COLUMN_NAME')
+          defn[column_name]
+        end
+
+        keys.length == 1 ? keys.first : keys
+      end
+    end
+  end
+end

data/lib/lhm/table_name.rb

@@ -0,0 +1,23 @@
+module Lhm
+  class TableName
+    def initialize(original, time = Time.now)
+      @original = original
+      @time = time
+      @timestamp = Timestamp.new(time)
+    end
+
+    attr_reader :original
+
+    def archived
+      "lhma_#{@timestamp}_#{@original}"[0...64]
+    end
+
+    def failed
+      archived[0...57] + "_failed"
+    end
+
+    def new
+      "lhmn_#{@original}"[0...64]
+    end
+  end
+end

data/lib/lhm/test_support.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+module Lhm
+  module TestMigrator
+    def initialize(*)
+      super
+      @name = @origin.name
+    end
+
+    def execute
+      @statements.each do |stmt|
+        @connection.execute(tagged(stmt))
+      end
+    end
+  end
+
+  module TestInvoker
+    def run(options = {})
+      normalize_options(options)
+      set_session_lock_wait_timeouts
+      @migrator.run
+    rescue => e
+      Lhm.logger.error("LHM run failed with exception=#{e.class} message=#{e.message}")
+      raise
+    end
+  end
+
+  # Patch LHM to execute ALTER TABLE directly on original tables,
+  # without the online migration dance.
+  # This mode is designed for local/CI environments where we can speed
+  # things up by not invoking "real" LHM logic.
+  def self.execute_inline!
+    Lhm::Migrator.prepend(TestMigrator)
+    Lhm::Invoker.prepend(TestInvoker)
+  end
+end

data/lib/lhm/throttler/slave_lag.rb

@@ -0,0 +1,162 @@
+module Lhm
+  module Throttler
+
+    def self.format_hosts(hosts)
+      formatted_hosts = []
+      hosts.each do |host|
+        if host && !host.match(/localhost/) && !host.match(/127.0.0.1/)
+          formatted_hosts << host.partition(':')[0]
+        end
+      end
+      formatted_hosts
+    end
+
+    class SlaveLag
+      include Command
+
+      INITIAL_TIMEOUT = 0.1
+      DEFAULT_STRIDE = 2_000
+      DEFAULT_MAX_ALLOWED_LAG = 10
+
+      MAX_TIMEOUT = INITIAL_TIMEOUT * 1024
+
+      attr_accessor :timeout_seconds, :allowed_lag, :stride, :connection
+
+      def initialize(options = {})
+        @timeout_seconds = INITIAL_TIMEOUT
+        @stride = options[:stride] || DEFAULT_STRIDE
+        @allowed_lag = options[:allowed_lag] || DEFAULT_MAX_ALLOWED_LAG
+        @slaves = {}
+        @get_config = options[:current_config]
+        @check_only = options[:check_only]
+      end
+
+      def execute
+        sleep(throttle_seconds)
+      end
+
+      private
+
+      def throttle_seconds
+        lag = max_current_slave_lag
+
+        if lag > @allowed_lag && @timeout_seconds < MAX_TIMEOUT
+          Lhm.logger.info("Increasing timeout between strides from #{@timeout_seconds} to #{@timeout_seconds * 2} because #{lag} seconds of slave lag detected is greater than the maximum of #{@allowed_lag} seconds allowed.")
+          @timeout_seconds = @timeout_seconds * 2
+        elsif lag <= @allowed_lag && @timeout_seconds > INITIAL_TIMEOUT
+          Lhm.logger.info("Decreasing timeout between strides from #{@timeout_seconds} to #{@timeout_seconds / 2} because #{lag} seconds of slave lag detected is less than or equal to the #{@allowed_lag} seconds allowed.")
+          @timeout_seconds = @timeout_seconds / 2
+        else
+          @timeout_seconds
+        end
+      end
+
+      def slaves
+        @slaves[@connection] ||= get_slaves
+      end
+
+      def get_slaves
+        slaves = []
+        if @check_only.nil? or !@check_only.respond_to?(:call)
+          slave_hosts = master_slave_hosts
+          while slave_hosts.any? do
+            host = slave_hosts.pop
+            slave = Slave.new(host, @get_config)
+            if !slaves.map(&:host).include?(host) && slave.connection
+              slaves << slave
+              slave_hosts.concat(slave.slave_hosts)
+            end
+          end
+        else
+          slave_config = @check_only.call
+          slaves << Slave.new(slave_config['host'], @get_config)
+        end
+        slaves
+      end
+
+      def master_slave_hosts
+        Throttler.format_hosts(@connection.select_values(Slave::SQL_SELECT_SLAVE_HOSTS))
+      end
+
+      def max_current_slave_lag
+        max = slaves.map { |slave| slave.lag }.push(0).max
+        Lhm.logger.info "Max current slave lag: #{max}"
+        max
+      end
+    end
+
+    class Slave
+      SQL_SELECT_SLAVE_HOSTS = "SELECT host FROM information_schema.processlist WHERE command LIKE 'Binlog Dump%'"
+      SQL_SELECT_MAX_SLAVE_LAG = 'SHOW SLAVE STATUS'
+
+      attr_reader :host, :connection
+
+      def initialize(host, connection_config = nil)
+        @host = host
+        @connection_config = prepare_connection_config(connection_config)
+        @connection = client(@connection_config)
+      end
+
+      def slave_hosts
+        Throttler.format_hosts(query_connection(SQL_SELECT_SLAVE_HOSTS, 'host'))
+      end
+
+      def lag
+        query_connection(SQL_SELECT_MAX_SLAVE_LAG, 'Seconds_Behind_Master').first.to_i
+      end
+
+      private
+
+      def client(config)
+        begin
+          Lhm.logger.info "Connecting to #{@host} on database: #{config[:database]}"
+          Mysql2::Client.new(config)
+        rescue Mysql2::Error => e
+          Lhm.logger.info "Error connecting to #{@host}: #{e}"
+          nil
+        end
+      end
+
+      def prepare_connection_config(config_proc)
+        config = if config_proc
+          if config_proc.respond_to?(:call) # if we get a proc
+            config_proc.call
+          else
+            raise ArgumentError, "Expected #{config_proc.inspect} to respond to `call`"
+          end
+        else
+          db_config
+        end
+        config.deep_symbolize_keys!
+        config[:host] = @host
+        config
+      end
+
+      def query_connection(query, result)
+        begin
+          @connection.query(query).map { |row| row[result] }
+        rescue Mysql2::Error => e
+          Lhm.logger.info "Unable to connect and/or query #{host}: #{e}"
+          [nil]
+        end
+      end
+
+      private
+
+      def db_config
+        if ar_supports_db_config?
+          ActiveRecord::Base.connection_pool.db_config.configuration_hash.dup
+        else
+          ActiveRecord::Base.connection_pool.spec.config.dup
+        end
+      end
+
+      def ar_supports_db_config?
+        # https://api.rubyonrails.org/v6.0/classes/ActiveRecord/ConnectionAdapters/ConnectionPool.html <-- has spec
+        # vs
+        # https://api.rubyonrails.org/v6.1/classes/ActiveRecord/ConnectionAdapters/ConnectionPool.html <-- has db_config
+        ActiveRecord::VERSION::MAJOR > 6 || ActiveRecord::VERSION::MAJOR == 6 && ActiveRecord::VERSION::MINOR >= 1
+      end
+    end
+  end
+end

data/lib/lhm/throttler/threads_running.rb

@@ -0,0 +1,53 @@
+module Lhm
+  module Throttler
+    class ThreadsRunning
+      include Command
+
+      DEFAULT_INITIAL_TIMEOUT = 0.1
+      DEFAULT_HEALTHY_RANGE = (0..50)
+
+      attr_accessor :timeout_seconds, :healthy_range, :connection
+      attr_reader :max_timeout_seconds, :initial_timeout_seconds
+
+      def initialize(options = {})
+        @initial_timeout_seconds = options[:initial_timeout] || DEFAULT_INITIAL_TIMEOUT
+        @max_timeout_seconds = options[:max_timeout] || (@initial_timeout_seconds * 1024)
+        @timeout_seconds = @initial_timeout_seconds
+        @healthy_range = options[:healthy_range] || DEFAULT_HEALTHY_RANGE
+        @connection = options[:connection]
+      end
+
+      def threads_running
+        query = <<~SQL.squish
+              SELECT COUNT(*) as Threads_running
+              FROM (
+                SELECT 1 FROM performance_schema.threads
+                WHERE NAME='thread/sql/one_connection'
+                  AND PROCESSLIST_STATE IS NOT NULL
+                LIMIT #{@healthy_range.max + 1}
+              ) AS LIM
+        SQL
+
+        @connection.select_value(query)
+      end
+
+      def throttle_seconds
+        current_threads_running = threads_running
+
+        if !healthy_range.cover?(current_threads_running) && @timeout_seconds < @max_timeout_seconds
+          Lhm.logger.info("Increasing timeout between strides from #{@timeout_seconds} to #{@timeout_seconds * 2} because threads running is greater than the maximum of #{@healthy_range.max} allowed.")
+          @timeout_seconds = @timeout_seconds * 2
+        elsif healthy_range.cover?(current_threads_running) && @timeout_seconds > @initial_timeout_seconds
+          Lhm.logger.info("Decreasing timeout between strides from #{@timeout_seconds} to #{@timeout_seconds / 2} because threads running is less than the maximum of #{@healthy_range.max} allowed.")
+          @timeout_seconds = @timeout_seconds / 2
+        else
+          @timeout_seconds
+        end
+      end
+
+      def execute
+        sleep throttle_seconds
+      end
+    end
+  end
+end

data/lib/lhm/throttler/time.rb

@@ -0,0 +1,29 @@
+module Lhm
+  module Throttler
+    class Time
+      include Command
+
+      DEFAULT_TIMEOUT = 0.1
+      DEFAULT_STRIDE = 2_000
+
+      attr_accessor :timeout_seconds
+      attr_accessor :stride
+
+      def initialize(options = {})
+        @timeout_seconds = options[:delay] || DEFAULT_TIMEOUT
+        @stride = options[:stride] || DEFAULT_STRIDE
+      end
+
+      def execute
+        sleep timeout_seconds
+      end
+    end
+
+    class LegacyTime < Time
+      def initialize(timeout, stride)
+        @timeout_seconds = timeout / 1000.0
+        @stride = stride
+      end
+    end
+  end
+end