lhm-shopify 3.5.5 → 4.1.0

data/gemfiles/{activerecord_7.0.0.alpha2.gemfile.lock → activerecord_7.0.gemfile.lock}

@@ -1,54 +1,63 @@
 PATH
   remote: ..
   specs:
-    lhm-shopify (3.5.5)
+    lhm-shopify (4.1.0)
       retriable (>= 3.0.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (7.0.0.alpha2)
-      activesupport (= 7.0.0.alpha2)
-    activerecord (7.0.0.alpha2)
-      activemodel (= 7.0.0.alpha2)
-      activesupport (= 7.0.0.alpha2)
-    activesupport (7.0.0.alpha2)
+    activemodel (7.0.8)
+      activesupport (= 7.0.8)
+    activerecord (7.0.8)
+      activemodel (= 7.0.8)
+      activesupport (= 7.0.8)
+    activerecord-trilogy-adapter (3.1.2)
+      activerecord (>= 6.0.a, < 7.1.a)
+      trilogy (>= 2.4.0)
+    activesupport (7.0.8)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
     after_do (0.4.0)
-    appraisal (2.4.1)
+    appraisal (2.5.0)
       bundler
       rake
       thor (>= 0.14.0)
     byebug (11.1.3)
-    concurrent-ruby (1.1.9)
+    concurrent-ruby (1.2.2)
     docile (1.4.0)
-    i18n (1.8.11)
+    i18n (1.14.1)
       concurrent-ruby (~> 1.0)
-    minitest (5.14.4)
-    mocha (1.13.0)
-    mysql2 (0.5.3)
+    minitest (5.20.0)
+    mocha (2.1.0)
+      ruby2_keywords (>= 0.0.5)
+    mysql2 (0.5.5)
     rake (13.0.6)
     retriable (3.1.2)
-    simplecov (0.21.2)
+    ruby2_keywords (0.0.5)
+    simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
-    simplecov_json_formatter (0.1.3)
-    thor (1.1.0)
-    toxiproxy (2.0.0)
-    tzinfo (2.0.4)
+    simplecov_json_formatter (0.1.4)
+    thor (1.2.2)
+    toxiproxy (2.0.2)
+    trilogy (2.6.0)
+    tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
 
 PLATFORMS
+  arm64-darwin-21
+  arm64-darwin-22
   x86_64-darwin-20
   x86_64-linux
 
 DEPENDENCIES
-  activerecord (= 7.0.0.alpha2)
+  activerecord (= 7.0.8)
+  activerecord-trilogy-adapter
   after_do
   appraisal
   byebug
@@ -59,6 +68,7 @@ DEPENDENCIES
   rake
   simplecov
   toxiproxy
+  trilogy
 
 BUNDLED WITH
    2.2.22

data/gemfiles/{activerecord_6.0.gemfile → activerecord_7.1.gemfile}

@@ -2,6 +2,6 @@
 
 source "https://rubygems.org"
 
-gem "activerecord", "6.0.0"
+gem "activerecord", "7.1.1"
 
 gemspec path: "../"

data/gemfiles/activerecord_7.1.gemfile.lock

@@ -0,0 +1,83 @@
+PATH
+  remote: ..
+  specs:
+    lhm-shopify (4.1.0)
+      retriable (>= 3.0.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (7.1.1)
+      activesupport (= 7.1.1)
+    activerecord (7.1.1)
+      activemodel (= 7.1.1)
+      activesupport (= 7.1.1)
+      timeout (>= 0.4.0)
+    activesupport (7.1.1)
+      base64
+      bigdecimal
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      connection_pool (>= 2.2.5)
+      drb
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      mutex_m
+      tzinfo (~> 2.0)
+    after_do (0.4.0)
+    appraisal (2.5.0)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    base64 (0.1.1)
+    bigdecimal (3.1.4)
+    byebug (11.1.3)
+    concurrent-ruby (1.2.2)
+    connection_pool (2.4.1)
+    docile (1.4.0)
+    drb (2.1.1)
+      ruby2_keywords
+    i18n (1.14.1)
+      concurrent-ruby (~> 1.0)
+    minitest (5.20.0)
+    mocha (2.1.0)
+      ruby2_keywords (>= 0.0.5)
+    mutex_m (0.1.2)
+    mysql2 (0.5.5)
+    rake (13.0.6)
+    retriable (3.1.2)
+    ruby2_keywords (0.0.5)
+    simplecov (0.22.0)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.4)
+    thor (1.2.2)
+    timeout (0.4.0)
+    toxiproxy (2.0.2)
+    trilogy (2.6.0)
+    tzinfo (2.0.6)
+      concurrent-ruby (~> 1.0)
+
+PLATFORMS
+  arm64-darwin-21
+  arm64-darwin-22
+  x86_64-darwin-20
+  x86_64-linux
+
+DEPENDENCIES
+  activerecord (= 7.1.1)
+  after_do
+  appraisal
+  byebug
+  lhm-shopify!
+  minitest
+  mocha
+  mysql2
+  rake
+  simplecov
+  toxiproxy
+  trilogy
+
+BUNDLED WITH
+   2.2.22

data/lhm.gemspec

@@ -21,7 +21,7 @@ Gem::Specification.new do |s|
   s.executables   = []
   s.metadata['allowed_push_host'] = "https://rubygems.org"
 
-  s.required_ruby_version = '>= 2.3.0'
+  s.required_ruby_version = '>= 3.0.0'
 
   s.add_dependency 'retriable', '>= 3.0.0'
 
@@ -31,6 +31,7 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'after_do'
   s.add_development_dependency 'rake'
   s.add_development_dependency 'mysql2'
+  s.add_development_dependency 'trilogy'
   s.add_development_dependency 'simplecov'
   s.add_development_dependency 'toxiproxy'
   s.add_development_dependency 'appraisal'

data/lib/lhm/atomic_switcher.rb

@@ -26,14 +26,14 @@ module Lhm
     end
 
     def atomic_switch
-      "rename table `#{ @origin.name }` to `#{ @migration.archive_name }`, " \
-      "`#{ @destination.name }` to `#{ @origin.name }`"
+      "RENAME TABLE `#{ @origin.name }` TO `#{ @migration.archive_name }`, " \
+      "`#{ @destination.name }` TO `#{ @origin.name }`"
     end
 
     def validate
       unless @connection.data_source_exists?(@origin.name) &&
         @connection.data_source_exists?(@destination.name)
-        error "`#{ @origin.name }` and `#{ @destination.name }` must exist"
+        error "`#{ @origin.name }` AND `#{ @destination.name }` MUST EXIST"
       end
     end
 

data/lib/lhm/chunker.rb

@@ -79,9 +79,9 @@ module Lhm
     private
 
     def raise_on_non_pk_duplicate_warning
-      @connection.execute("show warnings", should_retry: true, log_prefix: LOG_PREFIX).each do |level, code, message|
-        unless message.match?(/Duplicate entry .+ for key 'PRIMARY'/)
-          m = "Unexpected warning found for inserted row: #{message}"
+      @connection.select_all("SHOW WARNINGS", should_retry: true, log_prefix: LOG_PREFIX).each do |row|
+        unless row["Message"].match?(/Duplicate entry .+ for key 'PRIMARY'/)
+          m = "Unexpected warning found for inserted row: #{row["Message"]}"
           Lhm.logger.warn(m)
           raise Error.new(m) if @raise_on_warnings
         end
@@ -100,7 +100,7 @@ module Lhm
     end
 
     def upper_id(next_id, stride)
-      sql = "select id from `#{ @migration.origin_name }` where id >= #{ next_id } order by id limit 1 offset #{ stride - 1}"
+      sql = "SELECT id FROM `#{ @migration.origin_name }` WHERE id >= #{ next_id } ORDER BY id LIMIT 1 OFFSET #{ stride - 1}"
       top = @connection.select_value(sql, should_retry: true, log_prefix: LOG_PREFIX)
 
       [top ? top.to_i : @limit, @limit].min

data/lib/lhm/connection.rb

@@ -75,6 +75,14 @@ module Lhm
       end
     end
 
+    def select_all(query, should_retry: false, log_prefix: nil)
+      if should_retry
+        exec_with_retries(:select_all, query, log_prefix)
+      else
+        exec(:select_all, query)
+      end
+    end
+
     private
 
     def exec(method, sql)
@@ -105,4 +113,4 @@ module Lhm
       lhm_stack.at(first_candidate_index)
     end
   end
-end
+end

data/lib/lhm/sql_helper.rb

@@ -34,7 +34,7 @@ module Lhm
 
     def column_definition(cols)
       Array(cols).map do |column|
-        column.to_s.match(/`?([^\(]+)`?(\([^\)]+\))?/).captures
+        column.to_s.match(/`?([^\(\s]+)`?\s*(\([^\)]+\))?/).captures
       end
     end
 

data/lib/lhm/sql_retry.rb

@@ -105,9 +105,7 @@ module Lhm
     def mysql_single_value(name)
       query = Lhm::ProxySQLHelper.tagged("SELECT #{name} LIMIT 1")
 
-      @connection.execute(query).to_a.first.tap do |record|
-        return record&.first
-      end
+      @connection.select_value(query)
     end
 
     def same_host_as_initial?
@@ -140,32 +138,22 @@ module Lhm
             log_with_prefix("Reconnected to wrong host. Started migration on: #{@initial_hostname} (server_id: #{@initial_server_id}), but reconnected to: #{hostname} (server_id: #{server_id}).", :error)
             return false
           end
-        rescue StandardError => e
+        rescue ActiveRecord::ConnectionNotEstablished
           # Retry if ActiveRecord cannot reach host
-          next if /Lost connection to MySQL server at 'reading initial communication packet'/ === e.message
+          next
+        rescue StandardError => e
           log_with_prefix("Encountered error: [#{e.class}] #{e.message}. Will stop reconnection procedure.", :info)
           return false
         end
       end
+
       false
     end
 
     # For a full list of configuration options see https://github.com/kamui/retriable
     def default_retry_config
       {
-        on: {
-          StandardError => [
-            /Lock wait timeout exceeded/,
-            /Timeout waiting for a response from the last query/,
-            /Deadlock found when trying to get lock/,
-            /Query execution was interrupted/,
-            /Lost connection to MySQL server during query/,
-            /Max connect timeout reached/,
-            /Unknown MySQL server host/,
-            /connection is locked to hostgroup/,
-            /The MySQL server is running with the --read-only option so it cannot execute this statement/,
-          ]
-        },
+        on: retriable_mysql2_errors || retriable_trilogy_errors,
         multiplier: 1, # each successive interval grows by this factor
         base_interval: 1, # the initial interval in seconds between tries.
         tries: 20, # Number of attempts to make at running your code block (includes initial attempt).
@@ -176,5 +164,35 @@ module Lhm
         end
       }.freeze
     end
+
+    def retriable_mysql2_errors
+      return unless defined?(Mysql2::Error)
+
+      {
+        StandardError => [
+          /Lock wait timeout exceeded/,
+          /Timeout waiting for a response from the last query/,
+          /Deadlock found when trying to get lock/,
+          /Query execution was interrupted/,
+          /Lost connection to MySQL server during query/,
+          /Max connect timeout reached/,
+          /Unknown MySQL server host/,
+          /connection is locked to hostgroup/,
+          /The MySQL server is running with the --read-only option so it cannot execute this statement/,
+        ],
+      }
+    end
+
+    def retriable_trilogy_errors
+      return unless defined?(Trilogy::BaseError)
+
+      {
+        ActiveRecord::StatementInvalid => [
+          # Those sometimes appear as Trilogy::TimeoutError, and sometimes as ActiveRecord::StatementInvalid
+          /Lock wait timeout exceeded/,
+        ],
+        Trilogy::ConnectionError => nil,
+      }
+    end
   end
 end

data/lib/lhm/table.rb

@@ -39,10 +39,9 @@ module Lhm
       end
 
       def ddl
-        sql = "show create table `#{ @table_name }`"
-        specification = nil
-        @connection.execute(sql).each { |row| specification = row.last }
-        specification
+        query = "SHOW CREATE TABLE #{ @connection.quote_table_name(@table_name) }"
+
+        @connection.select_one(query)["Create Table"]
       end
 
       def parse

data/lib/lhm/throttler/replica_lag.rb

@@ -0,0 +1,166 @@
+module Lhm
+  module Throttler
+
+    def self.format_hosts(hosts)
+      formatted_hosts = []
+      hosts.each do |host|
+        if host && !host.match(/localhost/) && !host.match(/127.0.0.1/)
+          formatted_hosts << host.partition(':')[0]
+        end
+      end
+      formatted_hosts
+    end
+
+    class ReplicaLag
+      include Command
+
+      INITIAL_TIMEOUT = 0.1
+      DEFAULT_STRIDE = 2_000
+      DEFAULT_MAX_ALLOWED_LAG = 10
+
+      MAX_TIMEOUT = INITIAL_TIMEOUT * 1024
+
+      attr_accessor :timeout_seconds, :allowed_lag, :stride, :connection
+
+      def initialize(options = {})
+        @timeout_seconds = INITIAL_TIMEOUT
+        @stride = options[:stride] || DEFAULT_STRIDE
+        @allowed_lag = options[:allowed_lag] || DEFAULT_MAX_ALLOWED_LAG
+        @replicas = {}
+        @get_config = options[:current_config]
+        @check_only = options[:check_only]
+      end
+
+      def execute
+        sleep(throttle_seconds)
+      end
+
+      private
+
+      def throttle_seconds
+        lag = max_current_replica_lag
+
+        if lag > @allowed_lag && @timeout_seconds < MAX_TIMEOUT
+          Lhm.logger.info("Increasing timeout between strides from #{@timeout_seconds} to #{@timeout_seconds * 2} because #{lag} seconds of replica lag detected is greater than the maximum of #{@allowed_lag} seconds allowed.")
+          @timeout_seconds = @timeout_seconds * 2
+        elsif lag <= @allowed_lag && @timeout_seconds > INITIAL_TIMEOUT
+          Lhm.logger.info("Decreasing timeout between strides from #{@timeout_seconds} to #{@timeout_seconds / 2} because #{lag} seconds of replica lag detected is less than or equal to the #{@allowed_lag} seconds allowed.")
+          @timeout_seconds = @timeout_seconds / 2
+        else
+          @timeout_seconds
+        end
+      end
+
+      def replicas
+        @replicas[@connection] ||= get_replicas
+      end
+
+      def get_replicas
+        replicas = []
+        if @check_only.nil? or !@check_only.respond_to?(:call)
+          replica_hosts = master_replica_hosts
+          while replica_hosts.any? do
+            host = replica_hosts.pop
+            replica = Replica.new(host, @get_config)
+            if !replicas.map(&:host).include?(host) && replica.connection
+              replicas << replica
+              replica_hosts.concat(replica.replica_hosts)
+            end
+          end
+        else
+          replica_config = @check_only.call
+          replicas << Replica.new(replica_config['host'], @get_config)
+        end
+        replicas
+      end
+
+      def master_replica_hosts
+        Throttler.format_hosts(@connection.select_values(Replica::SQL_SELECT_REPLICA_HOSTS))
+      end
+
+      def max_current_replica_lag
+        max = replicas.map { |replica| replica.lag }.push(0).max
+        Lhm.logger.info "Max current replica lag: #{max}"
+        max
+      end
+    end
+
+    class Replica
+      SQL_SELECT_REPLICA_HOSTS = "SELECT host FROM information_schema.processlist WHERE command LIKE 'Binlog Dump%'"
+      SQL_SELECT_MAX_REPLICA_LAG = 'SHOW SLAVE STATUS'
+
+      attr_reader :host, :connection
+
+      def self.client
+        defined?(Mysql2::Client) ? Mysql2::Client : Trilogy
+      end
+
+      def self.client_error
+        defined?(Mysql2::Error) ? Mysql2::Error : Trilogy::Error
+      end
+
+      def initialize(host, connection_config = nil)
+        @host = host
+        @connection_config = prepare_connection_config(connection_config)
+        @connection = client(@connection_config)
+      end
+
+      def replica_hosts
+        Throttler.format_hosts(query_connection(SQL_SELECT_REPLICA_HOSTS, 'host'))
+      end
+
+      def lag
+        query_connection(SQL_SELECT_MAX_REPLICA_LAG, 'Seconds_Behind_Master').first.to_i
+      end
+
+      private
+
+      def client(config)
+        Lhm.logger.info "Connecting to #{@host} on database: #{config[:database]}"
+        self.class.client.new(config)
+      rescue self.class.client_error => e
+        Lhm.logger.info "Error connecting to #{@host}: #{e}"
+        nil
+      end
+
+      def prepare_connection_config(config_proc)
+        config = if config_proc
+          if config_proc.respond_to?(:call) # if we get a proc
+            config_proc.call
+          else
+            raise ArgumentError, "Expected #{config_proc.inspect} to respond to `call`"
+          end
+        else
+          db_config
+        end
+        config.deep_symbolize_keys!
+        config[:host] = @host
+        config
+      end
+
+      def query_connection(query, result)
+        @connection.query(query).map { |row| row[result] }
+      rescue self.class.client_error => e
+        Lhm.logger.info "Unable to connect and/or query #{host}: #{e}"
+        [nil]
+      end
+
+      private
+
+      def db_config
+        if ar_supports_db_config?
+          ActiveRecord::Base.connection_pool.db_config.configuration_hash.dup
+        else
+          ActiveRecord::Base.connection_pool.spec.config.dup
+        end
+      end
+
+      def ar_supports_db_config?
+        # https://api.rubyonrails.org/v6.0/classes/ActiveRecord/ConnectionAdapters/ConnectionPool.html <-- has spec
+        # vs
+        # https://api.rubyonrails.org/v6.1/classes/ActiveRecord/ConnectionAdapters/ConnectionPool.html <-- has db_config
+        ActiveRecord::VERSION::MAJOR > 6 || ActiveRecord::VERSION::MAJOR == 6 && ActiveRecord::VERSION::MINOR >= 1
+      end
+    end
+  end
+end