lhm-shopify 3.4.0 → 3.5.5

data/gemfiles/activerecord_7.0.0.alpha2.gemfile.lock

@@ -0,0 +1,64 @@
+PATH
+  remote: ..
+  specs:
+    lhm-shopify (3.5.5)
+      retriable (>= 3.0.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (7.0.0.alpha2)
+      activesupport (= 7.0.0.alpha2)
+    activerecord (7.0.0.alpha2)
+      activemodel (= 7.0.0.alpha2)
+      activesupport (= 7.0.0.alpha2)
+    activesupport (7.0.0.alpha2)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+    after_do (0.4.0)
+    appraisal (2.4.1)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    byebug (11.1.3)
+    concurrent-ruby (1.1.9)
+    docile (1.4.0)
+    i18n (1.8.11)
+      concurrent-ruby (~> 1.0)
+    minitest (5.14.4)
+    mocha (1.13.0)
+    mysql2 (0.5.3)
+    rake (13.0.6)
+    retriable (3.1.2)
+    simplecov (0.21.2)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.3)
+    thor (1.1.0)
+    toxiproxy (2.0.0)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+
+PLATFORMS
+  x86_64-darwin-20
+  x86_64-linux
+
+DEPENDENCIES
+  activerecord (= 7.0.0.alpha2)
+  after_do
+  appraisal
+  byebug
+  lhm-shopify!
+  minitest
+  mocha
+  mysql2
+  rake
+  simplecov
+  toxiproxy
+
+BUNDLED WITH
+   2.2.22

data/lhm.gemspec

@@ -1,6 +1,6 @@
-# -*- encoding: utf-8 -*-
+# frozen_string_literal: true
 
-lib = File.expand_path('../lib', __FILE__)
+lib = File.expand_path('lib', __dir__)
 $:.unshift(lib) unless $:.include?(lib)
 
 require 'lhm/version'
@@ -25,10 +25,14 @@ Gem::Specification.new do |s|
 
   s.add_dependency 'retriable', '>= 3.0.0'
 
+  s.add_development_dependency 'activerecord'
   s.add_development_dependency 'minitest'
   s.add_development_dependency 'mocha'
+  s.add_development_dependency 'after_do'
   s.add_development_dependency 'rake'
-  s.add_development_dependency 'activerecord'
   s.add_development_dependency 'mysql2'
   s.add_development_dependency 'simplecov'
+  s.add_development_dependency 'toxiproxy'
+  s.add_development_dependency 'appraisal'
+  s.add_development_dependency 'byebug'
 end

data/lib/lhm/atomic_switcher.rb

@@ -16,17 +16,13 @@ module Lhm
 
     attr_reader :connection
 
-    def initialize(migration, connection = nil, options = {})
+    LOG_PREFIX = "AtomicSwitcher"
+
+    def initialize(migration, connection = nil)
       @migration = migration
       @connection = connection
       @origin = migration.origin
       @destination = migration.destination
-      @retry_helper = SqlRetry.new(
-        @connection,
-        {
-          log_prefix: "AtomicSwitcher"
-        }.merge!(options.fetch(:retriable, {}))
-      )
     end
 
     def atomic_switch
@@ -36,7 +32,7 @@ module Lhm
 
     def validate
       unless @connection.data_source_exists?(@origin.name) &&
-             @connection.data_source_exists?(@destination.name)
+        @connection.data_source_exists?(@destination.name)
         error "`#{ @origin.name }` and `#{ @destination.name }` must exist"
       end
     end
@@ -44,9 +40,7 @@ module Lhm
     private
 
     def execute
-      @retry_helper.with_retries do |retriable_connection|
-        retriable_connection.execute atomic_switch
-      end
+      @connection.execute(atomic_switch, should_retry: true, log_prefix: LOG_PREFIX)
     end
   end
 end

data/lib/lhm/chunk_insert.rb

@@ -1,24 +1,21 @@
 require 'lhm/sql_retry'
+require 'lhm/proxysql_helper'
 
 module Lhm
   class ChunkInsert
-    def initialize(migration, connection, lowest, highest, options = {})
+
+    LOG_PREFIX = "ChunkInsert"
+
+    def initialize(migration, connection, lowest, highest, retry_options = {})
       @migration = migration
       @connection = connection
       @lowest = lowest
       @highest = highest
-      @retry_helper = SqlRetry.new(
-        @connection,
-        {
-          log_prefix: "Chunker Insert"
-        }.merge!(options.fetch(:retriable, {}))
-      )
+      @retry_options = retry_options
     end
 
     def insert_and_return_count_of_rows_created
-      @retry_helper.with_retries do |retriable_connection|
-        retriable_connection.update sql
-      end
+      @connection.update(sql, should_retry: true, log_prefix: LOG_PREFIX)
     end
 
     def sql

data/lib/lhm/chunker.rb

@@ -13,6 +13,8 @@ module Lhm
 
     attr_reader :connection
 
+    LOG_PREFIX = "Chunker"
+
     # Copy from origin to destination in chunks of size `stride`.
     # Use the `throttler` class to sleep between each stride.
     def initialize(migration, connection = nil, options = {})
@@ -28,15 +30,16 @@ module Lhm
       @start = @chunk_finder.start
       @limit = @chunk_finder.limit
       @printer = options[:printer] || Printer::Percentage.new
+      @retry_options = options[:retriable] || {}
       @retry_helper = SqlRetry.new(
         @connection,
-        {
-          log_prefix: "Chunker"
-        }.merge!(options.fetch(:retriable, {}))
+        retry_options: @retry_options
       )
     end
 
     def execute
+      @start_time = Time.now
+
       return if @chunk_finder.table_empty?
       @next_to_insert = @start
       while @next_to_insert <= @limit || (@start == @limit)
@@ -44,9 +47,16 @@ module Lhm
         top = upper_id(@next_to_insert, stride)
         verify_can_run
 
-        affected_rows = ChunkInsert.new(@migration, @connection, bottom, top, @options).insert_and_return_count_of_rows_created
+        affected_rows = ChunkInsert.new(@migration, @connection, bottom, top, @retry_options).insert_and_return_count_of_rows_created
         expected_rows = top - bottom + 1
 
+        # Only log the chunker progress every 5 minutes instead of every iteration
+        current_time = Time.now
+        if current_time - @start_time > (5 * 60)
+          Lhm.logger.info("Inserted #{affected_rows} rows into the destination table from #{bottom} to #{top}")
+          @start_time = current_time
+        end
+
         if affected_rows < expected_rows
           raise_on_non_pk_duplicate_warning
         end
@@ -54,8 +64,10 @@ module Lhm
         if @throttler && affected_rows > 0
           @throttler.run
         end
-        @printer.notify(bottom, @limit)
+
         @next_to_insert = top + 1
+        @printer.notify(bottom, @limit)
+
         break if @start == @limit
       end
       @printer.end
@@ -67,7 +79,7 @@ module Lhm
     private
 
     def raise_on_non_pk_duplicate_warning
-      @connection.query("show warnings").each do |level, code, message|
+      @connection.execute("show warnings", should_retry: true, log_prefix: LOG_PREFIX).each do |level, code, message|
         unless message.match?(/Duplicate entry .+ for key 'PRIMARY'/)
           m = "Unexpected warning found for inserted row: #{message}"
           Lhm.logger.warn(m)
@@ -82,16 +94,14 @@ module Lhm
 
     def verify_can_run
       return unless @verifier
-      @retry_helper.with_retries do |retriable_connection|
+      @retry_helper.with_retries(log_prefix: LOG_PREFIX) do |retriable_connection|
         raise "Verification failed, aborting early" if !@verifier.call(retriable_connection)
       end
     end
 
     def upper_id(next_id, stride)
       sql = "select id from `#{ @migration.origin_name }` where id >= #{ next_id } order by id limit 1 offset #{ stride - 1}"
-      top = @retry_helper.with_retries do |retriable_connection|
-        retriable_connection.select_value(sql)
-      end
+      top = @connection.select_value(sql, should_retry: true, log_prefix: LOG_PREFIX)
 
       [top ? top.to_i : @limit, @limit].min
     end
@@ -100,5 +110,6 @@ module Lhm
       return if @chunk_finder.table_empty?
       @chunk_finder.validate
     end
+
   end
 end

data/lib/lhm/cleanup/current.rb

@@ -4,17 +4,15 @@ require 'lhm/sql_retry'
 module Lhm
   module Cleanup
     class Current
-      def initialize(run, origin_table_name, connection, options = {})
+
+      LOG_PREFIX = "Current"
+
+      def initialize(run, origin_table_name, connection, options={})
         @run = run
         @table_name = TableName.new(origin_table_name)
         @connection = connection
         @ddls = []
-        @retry_helper = SqlRetry.new(
-          @connection,
-          {
-            log_prefix: "Cleanup::Current"
-          }.merge!(options.fetch(:retriable, {}))
-        )
+        @retry_config = options[:retriable] || {}
       end
 
       attr_reader :run, :connection, :ddls
@@ -59,15 +57,14 @@ module Lhm
 
       def execute_ddls
         ddls.each do |ddl|
-          @retry_helper.with_retries do |retriable_connection|
-            retriable_connection.execute(ddl)
-          end
+            @connection.execute(ddl, should_retry: true, log_prefix: LOG_PREFIX)
         end
+        Lhm.logger.info("Dropped triggers on #{@lhm_triggers_for_origin.join(', ')}")
+        Lhm.logger.info("Dropped tables #{@lhm_triggers_for_origin.join(', ')}")
       end
 
       def report_ddls
-        puts "The following DDLs would be executed:"
-        ddls.each { |ddl| puts ddl }
+        Lhm.logger.info("The following DDLs would be executed: #{ddls}")
       end
     end
   end

data/lib/lhm/connection.rb

@@ -0,0 +1,108 @@
+require 'delegate'
+require 'forwardable'
+require 'lhm/sql_retry'
+
+module Lhm
+    # Lhm::Connection inherits from SingleDelegator. It will forward any unknown method calls to the ActiveRecord
+    # connection.
+  class Connection < SimpleDelegator
+    extend Forwardable
+
+    # Will delegate the following function to @sql_retry object, while leaving them accessible from the Lhm::Connection
+    # object
+    def_delegators :@sql_retry, :reconnect_with_consistent_host, :reconnect_with_consistent_host=, :retry_config=
+
+    alias ar_connection __getobj__
+
+    def initialize(connection:, options: {})
+      @sql_retry = Lhm::SqlRetry.new(
+        connection,
+        retry_options: options[:retriable] || {},
+        reconnect_with_consistent_host: options[:reconnect_with_consistent_host] || false
+      )
+
+      # Creates delegation for the ActiveRecord Connection
+      super(connection)
+    end
+
+    def ar_connection=(connection)
+      raise Lhm::Error.new("Lhm::Connection requires an active record connection to operate") if connection.nil?
+
+      @sql_retry.connection = connection
+      # Sets connection as the delegated object
+      __setobj__(connection)
+    end
+
+    # ActiveRecord::Base overridden methods to incorporate custom retry logic
+    # All other methods will be delegated
+    def execute(query, should_retry: false, log_prefix: nil)
+      if should_retry
+        exec_with_retries(:execute, query, log_prefix)
+      else
+        exec(:execute, query)
+      end
+    end
+
+    def update(query, should_retry: false, log_prefix: nil)
+      if should_retry
+        exec_with_retries(:update, query, log_prefix)
+      else
+        exec(:update, query)
+      end
+    end
+
+    def select_value(query, should_retry: false, log_prefix: nil)
+      if should_retry
+        exec_with_retries(:select_value, query, log_prefix)
+      else
+        exec(:select_value, query)
+      end
+    end
+
+    def select_values(query, should_retry: false, log_prefix: nil)
+      if should_retry
+        exec_with_retries(:select_values, query, log_prefix)
+      else
+        exec(:select_values, query)
+      end
+    end
+
+    def select_one(query, should_retry: false, log_prefix: nil)
+      if should_retry
+        exec_with_retries(:select_one, query, log_prefix)
+      else
+        exec(:select_one, query)
+      end
+    end
+
+    private
+
+    def exec(method, sql)
+      ar_connection.public_send(method, Lhm::ProxySQLHelper.tagged(sql))
+    end
+
+    def exec_with_retries(method, sql, log_prefix=nil)
+      effective_log_prefix = log_prefix || file
+      @sql_retry.with_retries(log_prefix: effective_log_prefix) do |conn|
+        conn.public_send(method, Lhm::ProxySQLHelper.tagged(sql))
+      end
+    end
+
+    # Returns camelized file name of caller (e.g. chunk_insert.rb -> ChunkInsert)
+    def file
+      # Find calling file and extract name
+      /[\/]*(\w+).rb:\d+:in/.match(relevant_caller)
+      name = $1&.camelize || "Connection"
+      "#{name}"
+    end
+
+    def relevant_caller
+      lhm_stack = caller.select { |x| x.include?("/lhm") }
+      first_candidate_index = lhm_stack.find_index { |line| !line.include?(__FILE__) }
+
+      # Find the file that called the `#execute` (fallbacks to current file)
+      return lhm_stack.first unless first_candidate_index
+      lhm_stack.at(first_candidate_index)
+    end
+  end
+end

data/lib/lhm/entangler.rb

@@ -4,6 +4,7 @@
 require 'lhm/command'
 require 'lhm/sql_helper'
 require 'lhm/sql_retry'
+require 'lhm/connection'
 
 module Lhm
   class Entangler
@@ -12,19 +13,15 @@ module Lhm
 
     attr_reader :connection
 
+    LOG_PREFIX = "Entangler"
+
     # Creates entanglement between two tables. All creates, updates and deletes
     # to origin will be repeated on the destination table.
-    def initialize(migration, connection = nil, options = {})
+    def initialize(migration, connection = nil)
       @intersection = migration.intersection
       @origin = migration.origin
       @destination = migration.destination
       @connection = connection
-      @retry_helper = SqlRetry.new(
-        @connection,
-        {
-          log_prefix: "Entangler"
-        }.merge!(options.fetch(:retriable, {}))
-      )
     end
 
     def entangle
@@ -90,18 +87,16 @@ module Lhm
 
     def before
       entangle.each do |stmt|
-        @retry_helper.with_retries do |retriable_connection|
-          retriable_connection.execute(stmt)
-        end
+        @connection.execute(stmt, should_retry: true, log_prefix: LOG_PREFIX)
       end
+      Lhm.logger.info("Created triggers on #{@origin.name}")
     end
 
     def after
       untangle.each do |stmt|
-        @retry_helper.with_retries do |retriable_connection|
-          retriable_connection.execute(stmt)
-        end
+        @connection.execute(stmt, should_retry: true, log_prefix: LOG_PREFIX)
       end
+      Lhm.logger.info("Dropped triggers on #{@origin.name}")
     end
 
     def revert

data/lib/lhm/invoker.rb

@@ -16,8 +16,8 @@ module Lhm
   class Invoker
     include SqlHelper
     LOCK_WAIT_TIMEOUT_DELTA = 10
-      INNODB_LOCK_WAIT_TIMEOUT_MAX=1073741824.freeze # https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_lock_wait_timeout
-      LOCK_WAIT_TIMEOUT_MAX=31536000.freeze # https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html
+    INNODB_LOCK_WAIT_TIMEOUT_MAX = 1073741824.freeze # https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_lock_wait_timeout
+    LOCK_WAIT_TIMEOUT_MAX = 31536000.freeze # https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html
 
     attr_reader :migrator, :connection
 
@@ -49,14 +49,14 @@ module Lhm
       normalize_options(options)
       set_session_lock_wait_timeouts
       migration = @migrator.run
-      entangler = Entangler.new(migration, @connection, options)
+      entangler = Entangler.new(migration, @connection)
 
       entangler.run do
         options[:verifier] ||= Proc.new { |conn| triggers_still_exist?(conn, entangler) }
         Chunker.new(migration, @connection, options).run
         raise "Required triggers do not exist" unless triggers_still_exist?(@connection, entangler)
         if options[:atomic_switch]
-          AtomicSwitcher.new(migration, @connection, options).run
+          AtomicSwitcher.new(migration, @connection).run
         else
           LockedSwitcher.new(migration, @connection).run
         end
@@ -90,6 +90,8 @@ module Lhm
         options[:throttler] = Lhm.throttler
       end
 
+      Lhm.connection.retry_config = options[:retriable] || {}
+
     rescue => e
       Lhm.logger.error "LHM run failed with exception=#{e.class} message=#{e.message}"
       raise

data/lib/lhm/locked_switcher.rb

@@ -22,6 +22,8 @@ module Lhm
 
     attr_reader :connection
 
+    LOG_PREFIX = "LockedSwitcher"
+
     def initialize(migration, connection = nil)
       @migration = migration
       @connection = connection

data/lib/lhm/migrator.rb

@@ -214,6 +214,8 @@ module Lhm
       replacement = %{CREATE TABLE `#{ @origin.destination_name }`}
       stmt = @origin.ddl.gsub(original, replacement)
       @connection.execute(tagged(stmt))
+
+      Lhm.logger.info("Created destination table #{@origin.destination_name}")
     end
 
     def destination_read

data/lib/lhm/printer.rb

@@ -12,26 +12,30 @@ module Lhm
       end
     end
 
-    class Percentage < Base
+    class Percentage
       def initialize
-        super
         @max_length = 0
       end
 
       def notify(lowest, highest)
         return if !highest || highest == 0
+
+        # The argument lowest represents the next_to_insert row id, and highest represents the 
+        # maximum id upto which chunker has to copy the data. 
+        # If all the rows are inserted upto highest, then lowest passed here from chunker was 
+        # highest + 1, which leads to the printer printing the progress > 100%.
+        return if lowest >= highest
+        
         message = "%.2f%% (#{lowest}/#{highest}) complete" % (lowest.to_f / highest * 100.0)
         write(message)
       end
 
       def end
         write('100% complete')
-        @output.write "\n"
       end
 
       def exception(e)
-        write("failed: #{e}")
-        @output.write "\n"
+        Lhm.logger.error("failed: #{e}")
       end
 
       private
@@ -42,7 +46,7 @@ module Lhm
           extra = 0
         end
 
-        @output.write "\r#{message}" + (' ' * extra)
+        Lhm.logger.info(message)
       end
     end
 

data/lib/lhm/proxysql_helper.rb

@@ -0,0 +1,10 @@
+module Lhm
+  module ProxySQLHelper
+    extend self
+    ANNOTATION = "/*maintenance:lhm*/"
+
+    def tagged(sql)
+      "#{sql} #{ANNOTATION}"
+    end
+  end
+end